DistHPO_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras MNIST classifier with _distributed_ random-search hyper-parameter optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# System imports\n",
    "from __future__ import print_function\n",
    "from __future__ import division\n",
    "from __future__ import absolute_import\n",
    "import os\n",
    "\n",
    "# External imports\n",
    "import ipyparallel as ipp\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "\n",
    "# Local imports\n",
    "from mnist import load_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Connect to cluster\n",
    "\n",
    "Here we use a shell command to query the slurm job ID running our cluster, then we connect using the corresponding cluster ID."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\n",
      "          24191498 interacti       sh sfarrell  R       1:23     16 nid00[088-093,218-227]\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "squeue -u sfarrell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Worker IDs: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]\n"
     ]
    }
   ],
   "source": [
    "# Cluster ID taken from job ID above\n",
    "job_id = 24191498\n",
    "cluster_id = 'cori_{}'.format(job_id)\n",
    "\n",
    "# Use default profile\n",
    "c = ipp.Client(timeout=60, cluster_id=cluster_id)\n",
    "print('Worker IDs:', c.ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (60000, 28, 28, 1)\n",
      "x_test shape: (10000, 28, 28, 1)\n",
      "y_train shape: (60000, 10)\n",
      "y_test shape: (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "x_train, y_train, x_test, y_test = load_data()\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configure the tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Temporarily making things reproducible for development\n",
    "np.random.seed(0)\n",
    "\n",
    "# Define the hyper-parameter search points\n",
    "n_hpo_trials = 32\n",
    "grid_h1 = np.random.choice([4, 8, 16, 32, 64], size=n_hpo_trials)\n",
    "grid_h2 = np.random.choice([4, 8, 16, 32, 64], size=n_hpo_trials)\n",
    "grid_h3 = np.random.choice([8, 16, 32, 64, 128], size=n_hpo_trials)\n",
    "grid_dropout = np.random.rand(n_hpo_trials)\n",
    "grid_optimizer = np.random.choice(['Adadelta', 'Adam', 'Nadam'], size=n_hpo_trials)\n",
    "\n",
    "# Training config\n",
    "batch_size = 128\n",
    "n_epochs = 16\n",
    "valid_frac = 0.17\n",
    "checkpoint_dir = os.path.join(os.environ['SCRATCH'],\n",
    "                              'cori-interactive-dl/mnist_hpo_%i' % job_id)\n",
    "os.makedirs(checkpoint_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run the trainings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_and_train(valid_frac, batch_size, n_epochs,\n",
    "                    h1, h2, h3, dropout, optimizer, verbose=0,\n",
    "                    checkpoint_file=None):\n",
    "    \"\"\"Run training for one set of hyper-parameters\"\"\"\n",
    "    import keras\n",
    "    from mnist import build_model, load_data\n",
    "    from mlextras import configure_session, IPyParallelLogger\n",
    "    # Load the data\n",
    "    x_train, y_train, _, _ = load_data()\n",
    "    # Thread settings\n",
    "    keras.backend.set_session(configure_session())\n",
    "    # Build the model\n",
    "    model = build_model(h1=h1, h2=h2, h3=h3,\n",
    "                        dropout=dropout, optimizer=optimizer)\n",
    "    callbacks = []\n",
    "    if checkpoint_file is not None:\n",
    "        callbacks.append(keras.callbacks.ModelCheckpoint(checkpoint_file))\n",
    "    # Train the model\n",
    "    history = model.fit(x_train, y_train,\n",
    "                        validation_split=valid_frac,\n",
    "                        batch_size=batch_size, epochs=n_epochs,\n",
    "                        verbose=verbose, callbacks=callbacks)\n",
    "    return history.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HP trial 0 hidden 64-8-128 dropout 0.020 opt Nadam\n",
      "HP trial 1 hidden 4-8-16 dropout 0.442 opt Nadam\n",
      "HP trial 2 hidden 32-8-32 dropout 0.980 opt Adadelta\n",
      "HP trial 3 hidden 32-4-128 dropout 0.359 opt Adadelta\n",
      "HP trial 4 hidden 32-16-64 dropout 0.481 opt Adadelta\n",
      "HP trial 5 hidden 8-64-128 dropout 0.689 opt Adam\n",
      "HP trial 6 hidden 32-32-128 dropout 0.880 opt Nadam\n",
      "HP trial 7 hidden 16-32-128 dropout 0.918 opt Adadelta\n",
      "HP trial 8 hidden 64-16-64 dropout 0.217 opt Adadelta\n",
      "HP trial 9 hidden 4-64-128 dropout 0.565 opt Adam\n",
      "HP trial 10 hidden 4-16-128 dropout 0.865 opt Adadelta\n",
      "HP trial 11 hidden 64-4-128 dropout 0.509 opt Nadam\n",
      "HP trial 12 hidden 16-4-8 dropout 0.917 opt Adam\n",
      "HP trial 13 hidden 8-64-128 dropout 0.921 opt Adam\n",
      "HP trial 14 hidden 4-4-64 dropout 0.083 opt Adam\n",
      "HP trial 15 hidden 8-64-32 dropout 0.278 opt Adadelta\n",
      "HP trial 16 hidden 8-8-8 dropout 0.009 opt Adadelta\n",
      "HP trial 17 hidden 4-64-16 dropout 0.842 opt Adadelta\n",
      "HP trial 18 hidden 8-8-16 dropout 0.647 opt Adam\n",
      "HP trial 19 hidden 64-16-64 dropout 0.841 opt Adam\n",
      "HP trial 20 hidden 32-16-8 dropout 0.265 opt Nadam\n",
      "HP trial 21 hidden 4-4-8 dropout 0.398 opt Adam\n",
      "HP trial 22 hidden 32-8-16 dropout 0.553 opt Adadelta\n",
      "HP trial 23 hidden 4-8-32 dropout 0.165 opt Adadelta\n",
      "HP trial 24 hidden 16-8-128 dropout 0.370 opt Adam\n",
      "HP trial 25 hidden 32-8-32 dropout 0.146 opt Nadam\n",
      "HP trial 26 hidden 4-32-8 dropout 0.570 opt Adam\n",
      "HP trial 27 hidden 8-32-64 dropout 0.704 opt Adam\n",
      "HP trial 28 hidden 32-16-32 dropout 0.288 opt Adam\n",
      "HP trial 29 hidden 32-32-32 dropout 0.433 opt Adadelta\n",
      "HP trial 30 hidden 32-4-8 dropout 0.756 opt Adadelta\n",
      "HP trial 31 hidden 4-32-16 dropout 0.396 opt Adadelta\n"
     ]
    }
   ],
   "source": [
    "# Load-balanced view\n",
    "lv = c.load_balanced_view()\n",
    "\n",
    "# Loop over hps\n",
    "results = []\n",
    "params = zip(grid_h1, grid_h2, grid_h3, grid_dropout, grid_optimizer)\n",
    "for ihp, (h1, h2, h3, dropout, optimizer) in enumerate(params):\n",
    "    print('HP trial %i hidden %i-%i-%i dropout %.3f opt %s' %\n",
    "          (ihp, h1, h2, h3, dropout, optimizer))\n",
    "    checkpoint_file = os.path.join(checkpoint_dir, 'model_%i.h5' % ihp)\n",
    "    result = lv.apply(build_and_train,\n",
    "                      valid_frac=valid_frac,\n",
    "                      batch_size=batch_size, n_epochs=n_epochs,\n",
    "                      h1=h1, h2=h2, h3=h3,\n",
    "                      dropout=dropout, optimizer=optimizer,\n",
    "                      checkpoint_file=checkpoint_file, verbose=2)\n",
    "    results.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tasks completed: 32 / 32\n"
     ]
    }
   ],
   "source": [
    "print('Tasks completed: %i / %i' % (np.sum([ar.ready() for ar in results]), len(results)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "ar = results[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 49800 samples, validate on 10200 samples\n",
      "Epoch 1/16\n",
      " - 5s - loss: 0.2147 - acc: 0.9342 - val_loss: 0.1541 - val_acc: 0.9574\n",
      "Epoch 2/16\n",
      " - 4s - loss: 0.0548 - acc: 0.9833 - val_loss: 0.0559 - val_acc: 0.9838\n",
      "Epoch 3/16\n",
      " - 4s - loss: 0.0344 - acc: 0.9887 - val_loss: 0.0492 - val_acc: 0.9864\n",
      "Epoch 4/16\n",
      " - 4s - loss: 0.0241 - acc: 0.9919 - val_loss: 0.0492 - val_acc: 0.9851\n",
      "Epoch 5/16\n",
      " - 4s - loss: 0.0197 - acc: 0.9936 - val_loss: 0.0434 - val_acc: 0.9875\n",
      "Epoch 6/16\n",
      " - 4s - loss: 0.0159 - acc: 0.9944 - val_loss: 0.0479 - val_acc: 0.9875\n",
      "Epoch 7/16\n",
      " - 4s - loss: 0.0132 - acc: 0.9957 - val_loss: 0.0504 - val_acc: 0.9875\n",
      "Epoch 8/16\n",
      " - 4s - loss: 0.0117 - acc: 0.9959 - val_loss: 0.0508 - val_acc: 0.9876\n",
      "Epoch 9/16\n",
      " - 4s - loss: 0.0111 - acc: 0.9962 - val_loss: 0.0566 - val_acc: 0.9876\n",
      "Epoch 10/16\n",
      " - 4s - loss: 0.0105 - acc: 0.9963 - val_loss: 0.0567 - val_acc: 0.9863\n",
      "Epoch 11/16\n",
      " - 4s - loss: 0.0103 - acc: 0.9964 - val_loss: 0.0856 - val_acc: 0.9793\n",
      "Epoch 12/16\n",
      " - 4s - loss: 0.0093 - acc: 0.9968 - val_loss: 0.0564 - val_acc: 0.9874\n",
      "Epoch 13/16\n",
      " - 4s - loss: 0.0078 - acc: 0.9972 - val_loss: 0.0693 - val_acc: 0.9849\n",
      "Epoch 14/16\n",
      " - 4s - loss: 0.0078 - acc: 0.9972 - val_loss: 0.0571 - val_acc: 0.9875\n",
      "Epoch 15/16\n",
      " - 4s - loss: 0.0074 - acc: 0.9976 - val_loss: 0.0496 - val_acc: 0.9903\n",
      "Epoch 16/16\n",
      " - 4s - loss: 0.0052 - acc: 0.9983 - val_loss: 0.0526 - val_acc: 0.9890\n",
      "\n",
      "Using TensorFlow backend.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(ar.stdout)\n",
    "\n",
    "print(ar.stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "histories = [ar.get() for ar in results]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model selection and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_scores = np.array([h['val_acc'][-1] for h in histories])\n",
    "best_scores = np.array([max(h['val_acc']) for h in histories])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hyperparameters: trial 24 hidden 16-8-128 dropout 0.370 opt Adam\n",
      "  Last validation accuracy 0.9909\n",
      "  Best validation accuracy 0.9916\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnUAAAEYCAYAAAA6dQBNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOzde3ycdZ33/9dnZpJMzukhPaWlByyHtpRSQhFBAWERUEERoQjLQVbEBXEX/e3iesAb5b5ZdRFXuRFU8IR0K9xqVQ6LiiC6SFOEYlugpRSaHtNz0zSHmfn8/riuJJM0adI0k0lm3s/HYx5zHef6JG2vvuf6Xtf3a+6OiIiIiIxskWwXICIiIiKHT6FOREREJAco1ImIiIjkAIU6ERERkRygUCciIiKSAxTqRERERHKAQp2ISIaY2f1mttXM/tbLejOz/zSzNWa23MzmD3WNIpI7FOpERDLnB8C5B1l/HjAzfF0H3DMENYlIjlKoExHJEHd/BthxkE0uBH7kgeeAKjObODTViUiuiWW7gMEyduxYnzZtWrbLEJEsWbZs2TZ3r852HYeoBlifNl8fLtvUfUMzu47gah6lpaUnHnPMMUNSoIgMLwc71+VMqJs2bRp1dXXZLkNEssTM3sx2DQNgPSzrcexGd78PuA+gtrbWdb6Tw+IOqSSk2iDZBqlE8Eq2BctSyWAbM7BI+Eqbpqfl1vM6rOdp6+mvf4akUuHP2hr8jMnW3qcTLT0vTyW6/T56+j308jvosn34e6D9rX3a+jVt09/Z67kuZ0KdiMgIVA9MSZufDGzMUi2Dq+M/0TAkJBNdA0T7f5Qehgc8iLOeCqe9/++e6hZK0t7Tj91leQ/beirt5d3me3p13yYZfnay8xi9zafX0X29GVg0CAGRaNcQccCy9u0iXZelf156UOuYTmTv70YXaSGn1xBonX/e0Pd0x5j2Tvv49tbzd6Wco1AnIpI9S4AbzWwRcDKw290PaHodEqkUtOyB5l2wf2f4SpvuWL6rc3lrY9eAlh4gPJWVH6PfLAKRGEQKIBoLpq2HANX9ykxf6yMxiBZAQXH4+e2vaA/zBT2sj6aFxWRHePRUglQqhaeSwctTeDJ8TyXxVLB9+7RbFI/ESFmUlMXwSEHHdCoSwy1Gsn2dxUhasG3wHq5zwJ1UeNUuOK53HCtYHx4vrDlYl8K9vR5IkcJT4fqO/YPplKct987ltG8XBrOkO6mUkQRS7iRTkPLglXQn5Ra+QzJ8pdxJOjiGY7R6Aa3ESBAjYTESVkDSYiStgITFSKXNJyMFJK2QZKSAVKQg+L1ZlFQqRTKZwj1JKpUilUySbP/dJFMkwz+vCCkMwveggkj4ag+Y6UHTuiyn67R13/5fe/1rrVAnIpIhZvYQcAYw1szqgVuBAgB3/w7wKHA+sAZoAq4Z1ALcoWkH7KmHPRthz4bwfSPsa+ga3pp3HTyIxYqheBQUVwXvo6dDYWlaKCoIwkwkBtHCtOmCbusKOveJFoZBqqerM9D1Ks5B3i3SNZwdMJ0WntqPH4mEvyInmXLakk5rMkUimSKRctqSKRJJJ5FK0ZZ0EkmnLRUuS6bCbdPWh+8tiRQtbcmu74kULYkkzW0pWlqStLQFy5o71ie7zCeTQYhJpJxUKpj2EX6hKRoxombBe8SIGMSiESJmxNqXRSAWiQTrIhEiUSMagWgkQkHEiEWNgmiEaMSIRSIURI1YNFgXjYTT0c51HcsiRiRiRFJOQcrDUNgeFNsDYvdladMOqZTjOLFIhFhYSzRtOvgZOo/bWWe4Lhrp+DnbG50tbH4Oglv7svA93Kp7C7WZsej/KNSJZEVbWxv19fU0Nzdnu5ScEY/HmTx5MgUFBdkupU/uflkf6x24YUAfnkpB07bOoLZ7Q9fQ1j6dbOm6XyQG5ROhtDoIZ6Omdwa14lEQT5tuXx6vgoL4gMocCPcgHO1tTtDYkqCxOcHeljYa2+dbEl3W7WtN0JZ02hIJWpOttCVTtCZStCWDoNU+3Zrsuqx9PpOBqSgWIV4QpSgWoaggQlEsSjx8L4pFKI/HOtfHohTGIkFgMCMa7RqEohaEk44QZEGgiHTbJtoeYgyiZpiF0+E+ljYdCddFImnT7csjB27XMd9RT7A9ySQ7GjbR1hr8fevMIjakt8515UCyj22s2/vhHq/9L1Nfx+3bQM51CnUiGVRfX095eTnTpk3r+FYmA+fubN++nfr6eqZPn57tcrJnywr4yrigmTNdpAAqJkFFDdScCMe+P5iurOlcXlodNPFlWCKZYk9zgl1Nreza38bupjZ27W9lV1Mbu5ra2L2/jV1Nreze39Y1pIVBLZHqO2nFIkZ5PEZJYYyiWISCaISCWHCVpDAaobQoFiwLr/AUxoLlBdHObYvC6VjadrGoURAJ3tuv9MS6Le/YNu2KUSxiFBV0hrjCaCRv/t2/8cYbjK6qZMyYMXnzM2fSQM91CnUiGdTc3KxAN4jMjDFjxtDQ0JDtUrKrsAzecUMQ0iomdQa2krEdzYqZsnNfK6u3NrJmayObdu8PQlpaQNvV1MbOplb2Nh/8RvyKeIyqkkIqimOUFxUwZXQJ5UUxyuIxysL3zvkCyopilKetKysKgpz+bQ0POtcNroGe6xTqRDJMJ7nBpd8nMGoqnP2ljH28u7NlTwtrtjayeute1oQhbs3WRrbva+3YLmJQVVJIVXEBlSUFjCkt5MjqMiqLC6gqKaCquICqkkIq06ariguoKC4gGtGfY67Rv83BNZDfp0KdiEieSqacDTv3dwS39itwr29tZG9L55W2iniMmePLOfvY8cwcX8aR48qYOa6MSZXFRBTORIaNnAl1r23Zy9qGRmZUl2W7FJFhY/v27Zx11lkAbN68mWg0SnV10BH5888/T2FhYZ+fcc0113DLLbdw9NFH97rN3XffTVVVFZdffvngFC4Zde/Tr/OLFzeytqGRlkTnE6/V5UXMHFfGB+fX8LZxZR2v6rIiXYWRYU3nukDOhLqWRIr6nfsV6kTSjBkzhhdffBGAL33pS5SVlfGZz3ymyzZB/1BOpJd7sR544IE+j3PDDQN7gFOG3veffYP/89gr1E4dxZWnTO0Mb9XlVJYM/yeKRXqic10gs3fUDrFNu/dnuwSREWHNmjXMmTOH66+/nvnz57Np0yauu+46amtrmT17NrfddlvHtqeddhovvvgiiUSCqqoqbrnlFo4//nhOOeUUtm7dCsDnP/957rrrro7tb7nlFhYsWMDRRx/Nn//8ZwD27dvHhz70IY4//nguu+wyamtrO07CMjQe/9smvvKblZw7ewL/9fFT+Nx7Z3HpSUdw4tTRCnSSk/LtXJczV+oANuxSX2AyfP2vX61g5cY9g/qZsyZVcOv7Zw9o35UrV/LAAw/wne98B4A77riD0aNHk0gkOPPMM7n44ouZNWtWl312797N6aefzh133MHNN9/M/fffzy233HLAZ7s7zz//PEuWLOG2227j8ccf51vf+hYTJkzgkUce4aWXXmL+/PkDqlsGZtmbO/nUoheZN6WKuxbO04MKkjE612XvXJczV+oKIhE27tKVOpH+OvLIIznppJM65h966CHmz5/P/PnzWbVqFStXrjxgn+LiYs477zwATjzxRNatW9fjZ1900UUHbPPss8+ycOFCAI4//nhmzx7YCVoO3Rvb9vEPP1zKxMo437uylnhB5vupExku8ulclzNX6gpipuZXGdYG+i0zU0pLSzumV69ezTe/+U2ef/55qqqquOKKK3ocBSP9ZuNoNEoi0XNfZEVFRQds4yN9nKMRantjC9c88Dxmxg+uWcCYsqJslyQ5Tue67J3rcudKXTTCRjW/igzInj17KC8vp6Kigk2bNvHEE08M+jFOO+00Fi9eDMDLL7/c47djGVzNbUn+4Ud1bNrdzHevrGXa2NK+dxLJYbl+rsudK3XRCBt27cfd9ei9yCGaP38+s2bNYs6cOcyYMYNTTz110I/xyU9+kiuvvJK5c+cyf/585syZQ2Vl5aAfRwLJlPOpRX/lxfW7uOfy+Zw4dVS2SxLJulw/11muNIlMO+Y45wN3UPf5sxmr5gUZJlatWsWxxx6b7TKGhUQiQSKRIB6Ps3r1as455xxWr15NLHbo3y17+r2a2TJ3rx2seoez2tpar6urO+g2/+tXK3jgT+v44vtm8dHT8nicXBkSOtd1yua5LqNX6szsXOCbQBT4nrvf0ct2FwM/A05y97pw2WeBa4EkcJO7H/QaaWE0QiuwaVezQp3IMNTY2MhZZ51FIpHA3bn33nsHdJKTvn3/2Td44E/ruPa06Qp0IkMsm+e6jB3FzKLA3cDfAfXAUjNb4u4ru21XDtwE/CVt2SxgITAbmAT81syOcvdkb8criAWhbsOu/Rw3WU06IsNNVVUVy5Yty3YZOe+xl4O+6M6bM4HPna8rJyJDLZvnukw+KLEAWOPua929FVgEXNjDdl8GvgqkP+VwIbDI3Vvc/Q1gTfh5vSqIBj+KujURkXy17M0d/NN/vcgJU6r4xqXzNC6rSJ7JZKirAdanzdeHyzqY2QnAFHf/9aHuG+5/nZnVmVndzu3bKIpF1K2JiOSloC+6uqAvuqtOUl90Inkok6Gup6+IHU9lmFkE+Abw6UPdt2OB+33uXuvutdXV1dRUFatbExHJO9sbW7g6rS+60aV9D14uIrknk3fu1QNT0uYnAxvT5suBOcAfwi5IJgBLzOyCfuzbo0lVxWxQ86uI5JH9rUmu/WEdm3c389B1b1dfdCJ5LJNX6pYCM81supkVEjz4sKR9pbvvdvex7j7N3acBzwEXhE+/LgEWmlmRmU0HZgLP93XAiZVx3VMnkuaMM844oHPNu+66i3/8x3/sdZ+ysjIANm7cyMUXX9zr5/bVpcZdd91FU1NTx/z555/Prl27+lu69EMy5fzTf/2Vl+p38c2FJzD/CPVFJ/lJ57pAxkKduyeAG4EngFXAYndfYWa3hVfjDrbvCmAxsBJ4HLjhYE++tptUVUxDYwutidTh/wAiOeCyyy5j0aJFXZYtWrSIyy67rM99J02axMMPPzzgY3c/0T366KNUVVUN+PPkQF/5zUqeWLGFL75vFufOmZDtckSyRue6QEaHCXP3R939KHc/0t1vD5d90d2X9LDtGe191IXzt4f7He3uj/XneDVVxbjDlj26r04E4OKLL+bXv/41LS0tAKxbt46NGzcyb948zjrrLObPn89xxx3HL3/5ywP2XbduHXPmzAFg//79LFy4kLlz53LppZeyf3/nFfFPfOIT1NbWMnv2bG699VYA/vM//5ONGzdy5plncuaZZwIwbdo0tm3bBsCdd97JnDlzmDNnDnfddVfH8Y499lg+9rGPMXv2bM4555wux5GuvvfHtR190V1zqvqik/ymc10gp3r+nFRVDAR91U0ZXZLlakS6eewW2Pzy4H7mhOPgvB779AZgzJgxLFiwgMcff5wLL7yQRYsWcemll1JcXMzPf/5zKioq2LZtG29/+9u54IILeh1i75577qGkpITly5ezfPly5s+f37Hu9ttvZ/To0SSTSc466yyWL1/OTTfdxJ133slTTz3F2LFju3zWsmXLeOCBB/jLX/6Cu3PyySdz+umnM2rUKFavXs1DDz3Ed7/7XS655BIeeeQRrrjiisH5XeWQ3fvbuP3RVeqLToYnneuA7JzrMnqlbqhNrIoDqFsTkTTpzRLtzRHuzr/9278xd+5czj77bDZs2MCWLVt6/Yxnnnmm44Qzd+5c5s6d27Fu8eLFzJ8/nxNOOIEVK1b0OXj1s88+ywc/+EFKS0spKyvjoosu4o9//CMA06dPZ968eQCceOKJrFu37nB+9Jy1fkeT+qIT6Ubnuly7UlcZXKlTtyYyLB3kW2YmfeADH+Dmm2/mhRdeYP/+/cyfP58f/OAHNDQ0sGzZMgoKCpg2bRrNzQf/d9PTN9s33niDr3/96yxdupRRo0Zx9dVX9/k5Bxtvuqioc4i/aDSq5tdeFEQj6otOhi+d64DsnOty6kpdcWGU0aWF6tZEJE1ZWRlnnHEGH/3oRztuGt69ezfjxo2joKCAp556ijfffPOgn/Gud72LBx98EIC//e1vLF++HIA9e/ZQWlpKZWUlW7Zs4bHHOm9/LS8vZ+/evT1+1i9+8QuamprYt28fP//5z3nnO985WD9uXpg+tlR90Yl0o3Ndjl2pg6Bbk00KdSJdXHbZZVx00UUdTROXX34573//+6mtrWXevHkcc8wxB93/E5/4BNdccw1z585l3rx5LFgQjNp3/PHHc8IJJzB79mxmzJjBqaee2rHPddddx3nnncfEiRN56qmnOpbPnz+fq6++uuMz/uEf/oETTjhBTa2HoDCWU9/HRQZNvp/r7GCXB0eS2tpar6ur42M/quOt7U088c/vynZJIqxatYpjj9WN7IOtp9+rmS1z99oslTSk2s93IsOFznWZcajnupz7uhcMFaYrdSIiIpJfci7UTayMs7clwZ7mtmyXIiIiIjJkci7UtfdVt0lPwMowkSu3OAwX+n2KDE/6tzm4BvL7zNlQpyZYGQ7i8Tjbt2/XyW6QuDvbt28nHo9nuxQRSaNz3eAa6Lku555+rUkbVUIk2yZPnkx9fT0NDQ3ZLiVnxONxJk+enO0yRCSNznWDbyDnupwLddXlRcQiplElZFgoKChg+nSNyykiuU3nuuEh55pfoxFjfEVco0qIiIhIXsm5UAdBE6yaX0VERCSf5GSom1gV14MSIiIikldyMtRNqipmy55mkik9hSMi2WVm55rZq2a2xsxu6WH9EWb2lJn91cyWm9n52ahTREa+nA11bUlnW2NLtksRkTxmZlHgbuA8YBZwmZnN6rbZ54HF7n4CsBD4v0NbpYjkipwMdTVVQb8uuq9ORLJsAbDG3de6eyuwCLiw2zYOVITTlcDGIaxPRHJIRkNdP5odrjezl83sRTN7tv0brJlNM7P94fIXzew7h3LciZUaVUJEhoUaYH3afH24LN2XgCvMrB54FPjk0JQmIrkmY6Gun80OP3X349x9HvBV4M60da+7+7zwdf2hHFujSojIMGE9LOt+s+9lwA/cfTJwPvBjMzvg3Gxm15lZnZnVqYNXEelJJq/U9dns4O570mZLOfBkNyAV8RhlRTE1v4pIttUDU9LmJ3Ng8+q1wGIAd/8fIA6M7f5B7n6fu9e6e211dXWGyhWRkSyToa4/zQ6Y2Q1m9jrBlbqb0lZND58Ge9rM3tnTAXr75mpmTKyMa1QJEcm2pcBMM5tuZoUED0Is6bbNW8BZAGZ2LEGo06U4ETlkmQx1/Wl2wN3vdvcjgX8leAoMYBNwRPg02M3AT82sood9e/3mOqmqWKNKiEhWuXsCuBF4AlhF8JTrCjO7zcwuCDf7NPAxM3sJeAi42jUquogMQCbHfu1Ps0O6RcA9AO7eArSE08vCK3lHAXX9PfikqmL+tmH3odYsIjKo3P1Rggcg0pd9MW16JXDqUNclIrknk1fq+mx2MLOZabPvBVaHy6vDBy0wsxnATGDtoRx8UmWc7ftaaW5LHsaPICIiIjIyZOxKnbsnzKy92SEK3N/e7ADUufsS4EYzOxtoA3YCV4W7vwu4zcwSQBK43t13HMrx25+A3bS7meljSwflZxIREREZrjLZ/NqfZodP9bLfI8Ajh3Ps9G5NFOpEREQk1+XkiBIANWGoU7cmIiIikg9yNtSNrywCNKqEiIiI5IecDXVFsSjV5UUaVUJERETyQs6GOgj7qlMHxCIiIpIHcjvUVcZ1T52IiIjkhdwOdVXFbNrVjDpnFxERkVyX86Fuf1uSXU1t2S5FREREJKNyOtTVVMUBdWsiIiIiuS+nQ93Eys5RJURERERyWU6HuvRRJURERERyWU6HujGlhRTGIgp1IiIikvNyOtRFIsbEyjgb1fwqIiIiOS6nQx3ApMpiXakTERGRnJf7oa5KoU5ERERyXx6Eujhb9jSTSKayXYqIiIhIxuRBqCsm5bBlb0u2SxERERHJmLwIdaBuTURERCS35Xyoax9VQqFOREREcllGQ52ZnWtmr5rZGjO7pYf115vZy2b2opk9a2az0tZ9NtzvVTN7z0BraB9VYuMudWsiIiIiuStjoc7MosDdwHnALOCy9NAW+qm7H+fu84CvAneG+84CFgKzgXOB/xt+3iErLYpRWVygK3UiIiKS0zJ5pW4BsMbd17p7K7AIuDB9A3ffkzZbCng4fSGwyN1b3P0NYE34eQOibk1EREQk18Uy+Nk1wPq0+Xrg5O4bmdkNwM1AIfDutH2f67ZvTQ/7XgdcB3DEEUf0WsikyjgbFOpEREQkh2XySp31sMwPWOB+t7sfCfwr8PlD3Pc+d69199rq6upeC5lUVcwmDRUmIiIiOSyToa4emJI2PxnYeJDtFwEfGOC+BzWpqpjd+9tobEkM9CNEREREhrVMhrqlwEwzm25mhQQPPixJ38DMZqbNvhdYHU4vARaaWZGZTQdmAs8PtJBJYbcmm9QEKyIiIjkqY/fUuXvCzG4EngCiwP3uvsLMbgPq3H0JcKOZnQ20ATuBq8J9V5jZYmAlkABucPfkQGvp6IB4dzMzx5cfzo8lIiIiMixl8kEJ3P1R4NFuy76YNv2pg+x7O3D7YNShUSVEREQk1+X8iBIA48uLiJhCnYiIiOSuvAh1sWiE8RVxjSohIiIiOSsvQh2oA2IRERHJbfkV6nYr1ImIiEhuyp9QVxln065mUqkD+jAWERERGfHyJ9RVFdOaTLF9X2u2SxEREREZdHkV6kBPwIqIiEhuyqNQF4wqoVAnIiIiuSh/Ql1l56gSIiJDxczONbNXzWyNmd3SyzaXmNlKM1thZj8d6hpFJDdkdESJ4aSqpIDigqiu1InIkDGzKHA38HdAPbDUzJa4+8q0bWYCnwVOdfedZjYuO9WKyEiXN1fqzIxJVXGFOhEZSguANe6+1t1bgUXAhd22+Rhwt7vvBHD3rUNco4jkiLwJdaAOiEVkyNUA69Pm68Nl6Y4CjjKzP5nZc2Z2bk8fZGbXmVmdmdU1NDRkqFwRGcnyK9RVFuueOhEZStbDsu6dZcaAmcAZwGXA98ys6oCd3O9z91p3r62urh70QkVk5MuvUFdVTMPeFloSyWyXIiL5oR6YkjY/GdjYwza/dPc2d38DeJUg5ImIHJK8CnUTw25NNutqnYgcAjO70cxGDWDXpcBMM5tuZoXAQmBJt21+AZwZHmcsQXPs2sOpV0TyU16FupqODogV6kTkkEwgeHJ1cdhFSU/Nqgdw9wRwI/AEsApY7O4rzOw2M7sg3OwJYLuZrQSeAv4/d9+egZ9BRHJc3nRpAhpVQkQGxt0/b2ZfAM4BrgG+bWaLge+7++t97Pso8Gi3ZV9Mm3bg5vAlIjJgeXWlbmKlRpUQkYEJw9fm8JUARgEPm9lXs1qYiEgoo6Gur57UzezmsBf15Wb2OzObmrYuaWYvhq/u96AMSLwgypjSQj0BKyKHxMxuMrNlwFeBPwHHufsngBOBD2W1OBGRUMaaX/vTkzrwV6DW3ZvM7BMEJ8xLw3X73X3eYNelvupEZADGAhe5+5vpC909ZWbvy1JNIiJdZPJKXZ89qbv7U+7eFM4+R/C4f0ZpVAkRGYBHgR3tM2ZWbmYnA7j7qqxVJSKSJpOhrj89qae7FngsbT4e9p7+nJl9oKcdBtLD+sTK4EpdcHuMiEi/3AM0ps3vC5eJiAwbmXz6tT89qQcbml0B1AKnpy0+wt03mtkM4Pdm9nL3p8zc/T7gPoDa2tp+pbSaqmL2tSbZ05ygsrigP7uIiJinfRMMm13zqvcAERn+Mnmlrj89qWNmZwOfAy5w95b25e6+MXxfC/wBOGEwilK3JiIyAGvDhyUKwtenUAfBIjLMZDLU9dmTupmdANxLEOi2pi0fZWZF4fRY4FQg/QGLAZtUpW5NROSQXQ+8A9hA8IX1ZOC6rFYkItJNxpoP3D1hZu09qUeB+9t7Ugfq3H0J8DWgDPhZ2EH7W+5+AXAscK+ZpQiC5x3dnpodsI4rderWRET6KfzSuTDbdYiIHEy/Qp2ZHQnUu3uLmZ0BzAV+5O67DrZfP3pSP7uX/f4MHNef2g5VdVkRBVHTlToR6TczixM8zDUbiLcvd/ePZq0oEZFu+tv8+giQNLO3Ad8HpgM/zVhVGRSJGBMq1a2JiBySHxOM//oe4GmCe4T3ZrUiEZFu+hvqUuHA1B8E7nL3fwYmZq6szGrv1kREpJ/e5u5fAPa5+w+B95Kh1gQRkYHqb6hrM7PLgKuAX4fLRmx/IDVVxWzcpXvqRKTf2sL3XWY2B6gEpmWvHBGRA/U31F0DnALc7u5vmNl04CeZKyuzJlXF2bynmWRKHRCLSL/cZ2ajgM8TPMW/Evj37JYkItJVvx6UCJ88vQmC7kaAcne/I5OFZdLEymKSKWfr3mYmVhZnuxwRGcbMLALscfedwDPAjCyXJCLSo35dqTOzP5hZhZmNBl4CHjCzOzNbWubUdHRArCZYETk4d08BN2a7DhGRvvS3+bXS3fcAFwEPuPuJQI/dkYwEGlVCRA7Rk2b2GTObYmaj21/ZLkpEJF1/Ox+OmdlE4BKCIb1GNI0qISKHqL0/uhvSljlqihWRYaS/oe42gpEh/uTuS81sBrA6c2VlVnm8gPKimEKdiPSLu0/Pdg0iIn3p74MSPwN+lja/FvhQpooaCpOqijVUmIj0i5ld2dNyd//RUNciItKb/g4TNhn4FnAqQZPDs8Cn3L0+g7Vl1KQqjSohIv12Utp0HDgLeAFQqBORYaO/za8PEAwL9uFw/opw2d9loqihMLGqmBfXH3ToWhERANz9k+nzZlZJMHSYiMiw0d+nX6vd/QF3T4SvHwDVGawr42qqitnZ1Mb+1mS2SxGRkacJmJntIkRE0vX3St02M7sCeCicvwzYnpmShkbHE7C793NkdVmWqxGR4czMfkVw6wkEX4ZnAYuzV5GIyIH6G+o+Cnwb+AbBie3PBEOHjViTKjv7qlOoE5E+fD1tOgG8OZLvKRaR3NTfp1/fAi5IX2Zm/wTclYmihkJ7B8SbNKqEiPTtLWCTuygeiB8AACAASURBVDcDmFmxmU1z93XZLUtEpFN/76nryc2DVkUWjK+IYwYb9ASsiPTtZ0AqbT5JWjdPIiLDweGEOhu0KrKgMBZhXHmRujURkf6IuXtr+0w4XZjFekREDnA4oc772sDMzjWzV81sjZnd0sP6m81spZktN7PfmdnUtHVXmdnq8HXVYdTZq4mVxWzcrVAnIn1qMLOOW1DM7EJgWxbrERE5wEHvqTOzvfQc3gwo7mPfKHA3QV929cBSM1vi7ivTNvsrUOvuTWb2CeCrwKXhQNm3ArXh8ZeF++7s58/VLzVVxazatGcwP1JEctP1wINm9u1wvh7ocZQJEZFsOWioc/fyw/jsBcCacEgxzGwRcCHQEerc/am07Z8j6NQY4D3Ak+6+I9z3SeBcOrtUGRSTquL8dtUW3B2zEd2aLCIZ5O6vA283szLA3H1vtmsSEenucJpf+1IDrE+brw+X9eZa4LFD2dfMrjOzOjOra2hoOOQCJ1YW05JIsWNfa98bi0jeMrP/bWZV7t7o7nvNbJSZfSXbdYmIpMtkqOvp0leP9+GFHRvXAl87lH3d/T53r3X32urqQx/goqNbk93q1kREDuo8d+8YVzC8FeT8LNYjInKATIa6emBK2vxkYGP3jczsbOBzwAXu3nIo+x6umjDUqVsTEelD1MyK2mfMrBgoOsj2IiJDLpOhbikw08ymm1khsBBYkr6BmZ0A3EsQ6LamrXoCOCds4hgFnBMuG1QdQ4Up1InIwf0E+J2ZXWtm1wJPAj/Mck0iIl30d5iwQ+buCTO7kSCMRYH73X2Fmd0G1Ln7EoLm1jLgZ+GDCm+5+wXuvsPMvkwQDAFua39oYjCNLi2kKBZRqBORg3L3r5rZcuBsgttDHgemHnwvEZGhlbFQB+DujwKPdlv2xbTpsw+y7/3A/ZmrDsyMSVXFbNQ9dSLSt80Eo0pcArwBPJLdckREuspk8+uIMKkqrit1ItIjMzvKzL5oZquAbxM8lW/ufqa7f7uP3ds/46CdsKdtd7GZuZnVDlL5IpJn8j7UTawsVqgTkd68ApwFvN/dT3P3bxGM+9ovaZ2wnwfMAi4zs1k9bFcO3AT8ZVCqFpG8lPehblJVMVv3ttCWTPW9sYjkmw8RNLs+ZWbfNbOzOLRxrzs6YQ/Hi23vhL27LxOMqKN7QURkwPI+1NVUxXGHzbqvTkS6cfefu/ulwDHAH4B/Bsab2T1mdk4/PqLPjtTDXgCmuPuvD/ZBh9vZuojkvrwPde0dEKsJVkR64+773P1Bd38fQb+ZLwK93h+X5qAdqZtZBPgG8Ol+1HBYna2LSO7L+1A3sVKjSohI/7n7Dne/193f3Y/N++pIvRyYA/zBzNYBbweW6GEJERmIvA917R0Qa1QJEcmAg3bC7u673X2su09z92nAcwSdsddlp1wRGcnyPtSVFMYYVVKg5lcRGXTungDaO2FfBSxu74TdzC7IbnUikmsy2vnwSKFuTUQkU/rqhL3b8jOGoiYRyU15f6UOgocldE+diIiIjGQKdQTdmuieOhERERnJFOqAiVXF7G1OsLe5LduliIiIiAyIQh2dfdWpCVZERERGKoU6guZXULcmIiIiMnIp1KFRJURERGTkU6gDxpXHiUZMoU5ERERGLIU6IBoxJlTE2bRL99SJiIjIyKRQF5qkbk1ERERkBMtoqDOzc83sVTNbY2a39LD+XWb2gpklzOzibuuSZvZi+FrSfd/BNrGymI27FepERERkZMpYqDOzKHA3cB4wC7jMzGZ12+wt4Grgpz18xH53nxe+Mj5G4qSqYjbvbiaV8kwfSkRERGTQZfJK3QJgjbuvdfdWYBFwYfoG7r7O3ZcDqQzW0S81VXHaks62xpZslyIiIiJyyDIZ6mqA9Wnz9eGy/oqbWZ2ZPWdmH+hpAzO7LtymrqGh4XBq7ejWRPfViYiIyEiUyVBnPSw7lLbNI9y9FvgIcJeZHXnAh7nf5+617l5bXV090DoBeNu4Mszg3qfXqglWRERERpxMhrp6YEra/GRgY393dveN4fta4A/ACYNZXHdTx5TyufOP5fEVm7nj8VcyeSgRERGRQZfJULcUmGlm082sEFgI9OspVjMbZWZF4fRY4FRgZcYqDV172nSuPGUq9z2zlh//z7pMH05ERERk0GQs1Ll7ArgReAJYBSx29xVmdpuZXQBgZieZWT3wYeBeM1sR7n4sUGdmLwFPAXe4e8ZDnZnxxffN4qxjxnHrkhX8/pUtmT6kiIiIyKAw99y4f6y2ttbr6uoG5bP2tSS49L7/YW3DPhZ//BTm1FQOyueKSOaY2bLwPtycN5jnOxEZWQ52rtOIEj0oLYpx/1UnUVVcwEd/sFRjwoqIiMiwp1DXi3EVcR64ZgH7W5N89AdL2dvclu2SRERERHqlUHcQR08o554rTmTN1kb+8cEXaEtmvY9kERERkR4p1PXhtJlj+d8fPI4/rt7GF37xN3LlHkQRERHJLbFsFzASXHLSFN7a0cS3n1rDlNEl3HDm27JdkoiIiEgXCnX99OlzjmL9zia+9sSrTBldwgXHT8p2SSIiIiIdFOr6ycz46sVz2bSrmc8sfomJlXFOmjY622WJiIiIALqn7pAUxaLcd+WJTB5VzMd+VMfahsZslyQiIiICKNQdsqqSQh645iQiZlzzg6Vsb2zJdkkiIiIiCnUDMXVMKd+9spbNu5v52I/qaG5LZrskERERyXMKdQN04tRRfOPSebzw1i4+vfglUil1dSIiIiLZo1B3GM4/biL/dv4x/OblTXz1iVezXY6IiIjkMT39epg+9s4ZvLWjie88/TpTRhdz+clTs12SiIiI5CGFusNkZnzp/bPZsHM/X/zlCmqqijnj6HHZLktERETyjJpfB0EsGuFbH5nP0ePLueHBF3j6tYZslyQiIiJ5RqFukJQVxbj/6pMYXxnnqvuf5+M/rmP9jqZslyUiIiJ5QqFuEE2ojPPYp97J//eeo3nmtW2cfefTfPO3q9XliYiIiGScQt0gK4pFueHMt/G7T5/O2bPG843fvsbZdz7Nf6/YjLu6PREREZHMyGioM7NzzexVM1tjZrf0sP5dZvaCmSXM7OJu664ys9Xh66pM1pkJk6qKufsj8/npx06mpDDKdT9extUPLNXQYiIiIpIRGQt1ZhYF7gbOA2YBl5nZrG6bvQVcDfy0276jgVuBk4EFwK1mNuqgB2zeMyh1D7Z3HDmW39z0Tr7wvlm88OZO3nPXM9zx2Cvsa0lkuzQRGQL9+HJ7s5mtNLPlZvY7M1O/SCIyIJm8UrcAWOPua929FVgEXJi+gbuvc/flQKrbvu8BnnT3He6+E3gSOPegR9vxOjx5KwzDJs6CaIRrT5vO7z5zOhccX8N3nn6ds/7jaZa8tFFNsiI5rJ9fbv8K1Lr7XOBh4KtDW6WI5IpMhroaYH3afH24bND2NbPrzKzOzOr2R8rgT3fBrz4FqeH5YMK48jj/ccnxPPKJdzC2vJCbHvorC+97jlc2D8+rjCJy2Prz5fYpd29/VP45YPIQ1ygiOSKToc56WNbfy1L92tfd73P3WnevLZ4wE975GXjhh/DwRyHRekjFDqUTp47ilzecxu0fnMOrW/by3v98li8tWcHu/W3ZLk1EBtehfrm9FngsoxWJSM7KZKirB6akzU8GNmZ037O+AOfcDit/AQ9dCq37+lvrkItGjMtPnspTnz6DyxZM4Yf/s453f/0PLK5bTyqlJlmRHNHvL7dmdgVQC3ytl/UdLRMNDergXEQOlMlQtxSYaWbTzawQWAgs6ee+TwDnmNmo8AGJc8JlfXvHjXDBt2HtH+BHH4D9OwdQ+tAZVVrIVz5wHL+68TSmjinhXx5ezkX3/Jk/vLqVpMKdyEjXry+oZnY28DngAndv6emD0lsmqqurM1KsiIxsGQt17p4AbiQIY6uAxe6+wsxuM7MLAMzsJDOrBz4M3GtmK8J9dwBfJgiGS4HbwmX9M//v4cM/hE0vwgPvhb2bB/Vny4Q5NZU8fP07+I8PH8+GXfu5+oGlnPbvv+frT7zKum3D94qjiBxUn19uzewE4F6CQLc1CzWKSI6wXHn6sra21uvq6rouXPsHeOgjUFYNf/8LGD09K7UdqpZEkt+t2sriuvU881oDKYcF00dzSe0Uzj9uAiWFsWyXKDLsmNkyd6/Ndh3dmdn5wF1AFLjf3W83s9uAOndfYma/BY4DNoW7vOXuFxzsM3s834lIXjjYuS63Qx1AfR08eDFEi+Dvfw7ju/cmMLxt3t3MIy/U87O69azb3kRpYZT3zZ3EJSdNZv4RozDr6ZYdkfwzXENdJijUieSv/A51AFtXwY8/CG374fKHYcpJQ1vcIHB36t7cyeKl6/nNy5toak0yo7qUD584hQ/Nr2FcRTzbJYpklUKdiOQDhTqAneuCBycat8LCn8CR7x6y2gbbvpYEv3l5Ez+rW8/SdTuJRowzjqrmw7WTefcx4ymMaUhfyT8KdSKSDw52rsufm7NGTYOPPgE/uQgevAQu/j7MurDP3Yaj0qIYl9RO4ZLaKaxtaOThZfU88kI9v/vJVkaXFvLBE2r4cO1kjplQke1SRUREZIjkz5W6dvt3wk8vhfql8P5vwvwrM1/cEEgkU/xx9TZ+tmw9T67cQlvSmTmujHcdVc3pR1WzYPpo4gXRbJcpkjG6Uici+UBX6tIVjwoemFh8JSz5ZBDyTv1Utqs6bLFohDOPGceZx4xjx75WfvniBn7/ylZ+/NybfP/ZNyiKRTh5xhhOP6qa048ay5HVZXrIQkREJIfkX6gDKCyFhQ/Bzz8OT34xCHZn3Qo5EnJGlxZyzanTuebU6exvTfLcG9t5+tUGnlndwJd/vZIvA5Mq45x+dDXvmlnNO942lsrigmyXLSIiIochP0MdQKwQPvQ9iFfCs9+A/bvgvf8BkdxqoiwujHLm0eM48+hxAKzf0cQzqxt45rUGfv3SJh56fj3RiHHClKqOptrjaiqJRHIj4IqIiOSL/A11EAS4930jaJJ99k5o3gUfuAcKirNdWcZMGV3C5SdP5fKTp9KWTPHXt3bxzGsNPP1aA3c++Rp3Pvkao0oKOG1mEPBOnj6amqpihTwREZFhLr9DHQRNrmffGgS7J78A9cvgPbfDse/PmebY3hREIyyYPpoF00fzmfcczfbGFp5dsy1sqt3Gr14KhqgsLYwyc3w5x0wo5+gJ5Rw9PngfU1aU5Z9ARERE2inUtTv1Jph0Ajz2r7D472H66XDev8O4Y7Nd2ZAZU1bEhfNquHBeDamUs3LTHpbX7+a1LXt5ZfMenlixmUVL13dsP7asiKMnlHH0+AqOmVDOURPKOWp8mYYxExERyQL975tu+jvh489A3f3w1FfgnlPh5I/D6f8KxVXZrm5IRSLGnJpK5tRUdixzdxoaW3h1815e3byXVzbv5bUte/np82/S3JYCgoubR4wu6biad/SEcmaMLWN8RRGjSgrVjCsiIpIhCnXdRWNw8nUw50Pw+y/Dc/fA8sVBE+28KyCSv6M1mBnjyuOMK4/zzpnVHcuTKeetHU0dYe/VLXt4ZfNefrtqC6m0bhALouH+FUWML48zvqKIcRVxxlcE0+Mr4owvj1NRHFN3KyIiIodIoa43pWPg/XdB7TXw6L8EfdrV3Q/nfW1Ejh2bSdGIMX1sKdPHlnLunAkdy5vbkqzZ2shbO5rYsqeZLXta2Lq3ma17Wni9oZE/v76NPc2JAz6vKBZhfEWcceVB0BtXUURNVTGzJ1Uya1KFul8RERHpgUJdXyYeDx99HF7+Gfz3F+D7Z8PxH4GzvwTl47Nd3bAWL4ge0ITb3f7WJFv3BoEvCH7NbN3bwtYwBK7avIenX2uhsaUz/B0xuoQ5NRXMnhR89uxJFYzVQxsiIpLnFOr6wwzmXgJHnwd//A/487dh1a/g9H+Bk68P+ryTASkujDJ1TClTx5QedLuGvS2s2LibFRv3sGLjbv62YQ+Pvry5Y/2EinhH0Js9qYI5NZVMrIyrGVdERPKGQt2hKCoPrtCd8Pfw+GeDLlBe+BGcewfMPDvb1eW06vIizjh6HGeEnSgD7N7fxsqOkBcEvt+/srXjPr7RpYXMntR+RS94QndsWREV8QI9sCEiIjlHoW4gxhwJly+G154Iwt2DH4Kjzw/6txs9I9vV5Y3K4gJOOXIMpxw5pmNZU2uCVZv2Blf1Nuzhbxt38/1n19KW7HxiIxoxRpUUMKqkkFGlhYwpTXsvKWRMWfA+urTzFS/IrZFGREQk92Q01JnZucA3gSjwPXe/o9v6IuBHwInAduBSd19nZtOAVcCr4abPufv1max1QI56D8w4I3hC9pmvwd0nwyk3Bs20ZeOhfALEdK/XUCopjHHi1FGcOHVUx7KWRJLVWxpZs7WRHftag1dTKzv3tbJ9X2vH8p1NrV2e1u36udEugW9USQGjSgsZHQbDICAWBCGwpJCqkkIKY/n7pLSIiAy9jIU6M4sCdwN/B9QDS81sibuvTNvsWmCnu7/NzBYC/w5cGq573d3nZaq+QRMrgtP+CeZeCr+9NRhu7Nk7O9cXjw7CXfkEKJ8Yhr2JacsmBMsU/jKmKNb3AxsAqZSzp7mN7fs6A9/OMADuaAzf97Wys6mNN7btY+e+Vva2HPj0bruyolgQ9MKQN7q0/QpgAWPLihhXUUR1WZzq8iLGlBVSEFUIFBGRgcvklboFwBp3XwtgZouAC4H0UHch8KVw+mHg2zZS72yvmAgX3Rd0VLxjLezdBHu3hO+boXEzNLwKjVsg1UMQKB7dNewVlkKkIOg3L1IA0cK06QKIxMJlBb1vN3oGVNYM/e9ihIpEjKowgFHd9/YArYkUu5qCoNd+tW9HGAZ3NrV1zje18npDIzv3tbKvNXnA55jB6JJCqsuLgldZUed0+BpXHoRA9eMnIiI9yWSoqwHWp83XAyf3to27J8xsN9B+g9R0M/srsAf4vLv/sfsBzOw64DqAI444YnCrH6gxRwav3qRS0LQ9CHuNW3oJf69AWxMkE5BshVQbeGpg9YyaBtNOg6mnwbRToWqY/J5yRGEswriKOOMq4v3ep7ktybbGFhr2hq9weuvezmVrG/bRsLeF1uSBf+6FsQjVZUVMG1vCkdVlna9xpUyo0BO/IiL5KpOhrqf/WbrfsdTbNpuAI9x9u5mdCPzCzGa7+54uG7rfB9wHUFtb28vdUMNMJAJl1cHrUKRSQbhLtkKyLbjalwzne5xugS0rYd2zsOrX8NefBJ9TeUQQ8qadClNPDUKfQsCQihdEmTyqhMmjSg66nbuzZ3+ChsbmLoGvobGFLbubeWN7Ez9/YUOXJuCSwigzqku7hL0Z1UHH0Jl42KMtmSIWMQVJEZFhIJOhrh6YkjY/GdjYyzb1ZhYDKoEd7u5AC4C7LzOz14GjgLoM1ju8RSIQKTq0e+9mnAGn/GMQCLeuhDf/FIS81U/ASz8NtqmoCa/knRq8j56hkDdMmBmVJQVUlhTwtnHlPW7TPh7v61v38XpDY/jaR926nfzyxY1pnwWTRxV3CXozxpYRjRj7WhPsa2l/JYP31vb3YHlTa5LG9G1aEzS1JGkNQ92YskLGlhWlvcL58uB9TGkwPbqkkJjuHRQRyYhMhrqlwEwzmw5sABYCH+m2zRLgKuB/gIuB37u7m1k1QbhLmtkMYCawNoO15rZIBCbMCV4nfxzcgybedc8GQe/1p2D5fwXblk0IruK1N9mOnamQN4ylj8eb3rULBKN1vLGta9hb29DIX9buYH/bgff1pSuMRigtilJSGKOsKEZJUZSyohjjy+Md0yWFMUoLo+wPm5O3NbayrbGF1Vv2sq2xtcem4/Z7B8eWFXUJgmVFUSIRI2pGJGLEIkY0YkQseO94pa1v3z4agWgej8ksItIuY6EuvEfuRuAJgi5N7nf3FWZ2G1Dn7kuA7wM/NrM1wA6C4AfwLuA2M0sASeB6d9+RqVrzjhmMOzZ4LfhYEPK2rYY3n4V1fwqC3t8eCbaNFEDxKCgZHTzMUTK6h/nu76OCBzgkq4oLo8yaVMGsSRVdlqdSzqY9zbzRsA/HKS0Kg1thZ1g73O5Y3J29LQm27e0Me9sbW2gIp4PlLby4fhfbGlto6uHhEREROTQWtHSOfLW1tV5Xl7+ts4PKPXiCd92zsPMNaNoB+3dA087wPZxPtvb+GUUVneEvXgXximBZvDJ8r+jlPVyvodfyTirlJN1JpsKXe7AsbT6ZclIpwukUyRTBMneOm1y1zN1rs/1zDAWd70Tyl5n1eq7TiBJyILO+n+J1h9Z9XUNe0w7Yv7Pb/A7Yvwv2bIDmPdCyJ3iyty+xeNewV1gaHNNTgAfThPMd0z0to3OfSDS4mlg6FkrGQskYKB0TTKcvKxkdbJsN7kFYTjRDogXa9gfvif1d55MtUFAMRZVdg3FByYhtLo9EjAiGBu8QERkYhToZGDMoKgteh9pNSrINWvZC8+4g5LWHvS7vu7vOJ5oB6wxbZmCRYJlZ+B5Jm+6+jODJ4KYdsPGvsG97cIyef7jgKmNpe8gb0xn64pVBSEy1BV3OpBJp022dTya3P4V8wHbh+kQztDWH4a256/wBD4kfgkgsGKO444po9yuj3UJgLN7Zz2G0sOfpjv4P2/tHDOezFXxFRKRHCnUy9KIFwdWwktHZrSPRGlxJ3LcNmraF79uDV8ey7bB9Daz/S7C8e3+BFu3sADoS7aEz6DAItYeh9mVF5UGgisWhIN45HYsHTzgXFAfvseJu8/FgWbQguGrXsicIxwcLyLve7BqUDyc0dv3hgzrilVBa3fVV1j49LnwfC2Xjgp9DREQyQqFO8lessHMEj/5IpaC1MQhvkfawNsKeumz/GdoDX7KlayfXXabb0q429jbdGgTKfdugcStsWBZMt+7t+fiFZUHA6x72okVhH4wtnX0uJtKm01+J1gOXJduG9vcoIjIMKdSJ9FckEjRZjmTtP0O8IugVMlNam8IrnQ3Q2BC872sIAt++rcH0rjehfmmwXfsV0GjYF2N7E2/6K5Y2XVDZ2SQcKwqmWZHBH0hEZPhTqBORwVdYAoVH9O9+y1QqCHWR6GE+5HHPYewrIjLyKdSJSHZFIsAIa8YWERmGdCYVERERyQEKdSIiIiI5QKFOREREJAco1ImIiIjkAIU6ERERkRygUCcikkFmdq6ZvWpma8zslh7WF5nZf4Xr/2Jm04a+ShHJBQp1IiIZYmZR4G7gPGAWcJmZzeq22bXATnd/G/AN4N+HtkoRyRUKdSIimbMAWOPua929FVgEXNhtmwuBH4bTDwNnmR1WL8wikqdypvPhZcuWNZrZq9muIzQW2JbtIlAd3Q2XOmD41JJLdUwdjEIGWQ2wPm2+Hji5t23cPWFmu4ExdPt9mNl1wHXhbIuZ/S0jFR+aXPr7M1iGSy2qo6tcqqPXc13OhDrgVXevzXYRAGZWNxxqUR3Dsw4YPrWojozr6YqbD2Ab3P0+4D4YPr8v1XGg4VKL6sjPOtT8KiKSOfXAlLT5ycDG3rYxsxhQCewYkupEJKco1ImIZM5SYKaZTTezQmAhsKTbNkuAq8Lpi4Hfu/sBV+pERPqSS82v92W7gDTDpRbV0dVwqQOGTy2qI4PCe+RuBJ4AosD97r7CzG4D6tx9CfB94MdmtobgCt3Cfnz0cPl9qY4DDZdaVEdXeVGH6QuhiIiIyMin5lcRERGRHKBQJyIiIpIDciLU9TUMzxDVMMXMnjKzVWa2wsw+lY060uqJmtlfzezXWa6jysweNrNXwt/NKVmq45/DP5e/mdlDZhYfouPeb2Zb0/sUM7PRZvakma0O30dlqY6vhX8uy83s52ZWlek6eqslbd1nzMzNbOxQ1DLSDIdzXViHzncH1pDX57rw2Drf9VFH2rqMnOtGfKjr5zA8QyEBfNrdjwXeDtyQpTrafQpYlcXjt/sm8Li7HwMcTxZqMrMa4Cag1t3nENyw3p+b0QfDD4Bzuy27Bfidu88EfhfOZ6OOJ4E57j4XeA347BDU0VstmNkU4O+At4aojhFlGJ3rQOe7nuT7uQ50vutPHRk91434UEf/huHJOHff5O4vhNN7Cf5B1wx1HQBmNhl4L/C9bBw/rY4K4F0ET/fh7q3uvitL5cSA4rAfsBIO7CssI9z9GQ7scyx9WKgfAh/IRh3u/t/unghnnyPoQy3jevmdQDDu6b/QQ8e7AgyTcx3ofNdDDXl/rgOd7/pTRyhj57pcCHU9DcOTlZNLOzObBpwA/CVLJdxF8BcmlaXjt5sBNAAPhE0j3zOz0qEuwt03AF8n+Fa0Cdjt7v891HWkGe/um8LaNgHjslhLu48Cj2Xr4GZ2AbDB3V/KVg0jwLA714HOdyGd63qn812aTJ/rciHU9WuInaFiZmXAI8A/ufueLBz/fcBWd1821MfuQQyYD9zj7icA+xiaS+9dhPdwXAhMByYBpWZ2xVDXMVyZ2ecImtMezNLxS4DPAV/MxvFHkGF1rgOd79LoXDdCZPN8NxTnulwIdf0ZhmdImFkBwQnuQXf/f9moATgVuMDM1hE0z7zbzH6SpVrqgXp3b/8G/zDBiW+onQ284e4N7t4G/D/gHVmoo90WM5sIEL5vzVYhZnYV8D7g8iyOYnAkwX9CL4V/bycDL5jZhCzVM1wNm3Md6HzXjc51vdP5rlPGz3W5EOr6MwxPxpmZEdxPscrd7xzq47dz98+6+2R3n0bwu/i9u2flm5q7bwbWm9nR4aKzgJVZKOUt4O3/f3v38ypVGcdx/P1BI66UG8MIou4icRGoiLhplwTRoo0Li4gIVy6qlfgPhOhOxCAMWgQhRCDuJHEhiGKLuGnSQpC7EAx0ESGGiHxbzHNxkLn+uM7cM3N6v+AwzzwzzHwPc/jyPc955jxJ1rXfaRfdTqoeXhbqM+BUF0EkeR84AHxY1I+xFgAAAm9JREFUVXe7iAGgqq5U1caqmm/H7Q1gezt+9NBU5Dow342Iw1y3PPNdsxq5buaLujbxcWkZnj+Bn6rqagehvAN8yuBMcaFtH3QQx7T5AvgxyWVgG3BwtQNoZ88/A78BVxgc96uyZEySE8BFYHOSG0n2AoeA95JcY/APqEMdxXEMeBk4047Xbycdx2Ni0RNMUa4D890o/+tcB+a7p4xjst/pMmGSJEmzb+ZH6iRJkmRRJ0mS1AsWdZIkST1gUSdJktQDFnWSJEk9YFGnqZPkwdBtEhaSjO3O7Enmk/wxrs+TpJUy12nc1nYdgDTCv1W1resgJGnCzHUaK0fqNDOSLCY5nOTXtr3V+t9McjbJ5fb4Rut/NcnJJL+3bWm5nDVJvktyNckvSeY62ylJeoS5TitlUadpNPfIJYk9Q6/9U1U7Gdwd/EjrOwb8UFVbGCzSfLT1HwXOVdVWBuswLt19fxPwTVW9DfwN7J7w/kjSKOY6jZUrSmjqJLlTVS+N6F8E3q2q620x8b+qakOS28BrVXW/9d+sqleS3AJer6p7Q58xD5ypqk3t+QHghar6evJ7JkkPmes0bo7UadbUMu3l3jPKvaH2A5xbKmn6mOv0zCzqNGv2DD1ebO0LwEet/QlwvrXPAvsAkqxJsn61gpSk52Su0zOzatc0mkuyMPT8dFUt/dX/xSSXGJyQfNz6vgS+T7IfuAV83vq/Ao4n2cvgLHUfcHPi0UvS0zHXaaycU6eZ0eaZ7Kiq213HIkmTYq7TSnn5VZIkqQccqZMkSeoBR+okSZJ6wKJOkiSpByzqJEmSesCiTpIkqQcs6iRJknrgP+9VF86xAjZkAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Find the best in terms of final validation set accuracy\n",
    "i = best_scores.argmax()\n",
    "h = histories[i]\n",
    "\n",
    "print('Hyperparameters: trial %i hidden %i-%i-%i dropout %.3f opt %s' %\n",
    "      (i, grid_h1[i], grid_h2[i], grid_h3[i], grid_dropout[i], grid_optimizer[i]))\n",
    "print('  Last validation accuracy %.4f' % last_scores[i])\n",
    "print('  Best validation accuracy %.4f' % best_scores[i])\n",
    "\n",
    "plt.figure(figsize=(9,4))\n",
    "\n",
    "# Loss\n",
    "plt.subplot(121)\n",
    "plt.plot(h['loss'], label='Training')\n",
    "plt.plot(h['val_loss'], label='Validation')\n",
    "plt.xlim(xmin=0, xmax=len(h['loss'])-1)\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend(loc=0)\n",
    "\n",
    "# Accuracy\n",
    "plt.subplot(122)\n",
    "plt.plot(h['acc'], label='Training')\n",
    "plt.plot(h['val_acc'], label='Validation')\n",
    "plt.xlim(xmin=0, xmax=len(h['loss'])-1)\n",
    "plt.ylim((0, 1))\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend(loc=0)\n",
    "\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Worst model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hyperparameters: trial 2 hidden 32-8-32 dropout 0.980 opt Adadelta\n",
      "  Last validation accuracy 0.1066\n",
      "  Best validation accuracy 0.1066\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnUAAAEYCAYAAAA6dQBNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3df5xcdX3v8dd7fye7CdndgCABE35USGJIli1qQYVCucCthF9XyBURsObiL6Bo702tVS/WPsBaShWuiBWoLRduBGlTRZAHNy1yrUhCYyCJNClGDYkSNoGQn/vrc/+Ys5vZ2dlkd7OzZ/bM+/l4TObMOd8589nZyXc/8/2ecz6KCMzMzMxsYqtKOwAzMzMzO3RO6szMzMwywEmdmZmZWQY4qTMzMzPLACd1ZmZmZhngpM7MzMwsA5zUmZmViKR7JL0i6YUhtkvSVyRtkLRaUtt4x2hm2eGkzsysdO4DzjvA9vOBE5PbYuBr4xCTmWWUkzozsxKJiKeAbQdoshD4VuT8GJgm6ajxic7MsqYm7QDGyvTp02PmzJlph2FmKVm5cuWrEXF42nGM0NHAr/Ieb0rWbSlsKGkxudE8GhsbTz3ppJPGJUAzKy8H6usyk9TNnDmTFStWpB2GmaVE0i/SjmEUVGRd0dqNEXE3cDdAe3t7uL8zq0wH6us8/Wpmlp5NwDF5j2cAm1OKxcwmOCd1ZmbpWQZclZwF+w7g9YgYNPVqZjYcmZl+NTMrN5IeAM4EpkvaBHwOqAWIiLuAR4ELgA3AbuCadCI1syxwUmdWQl1dXWzatIm9e/emHUpmNDQ0MGPGDGpra9MO5aAiYtFBtgfwsXEKx6xk3NeNvdH0dU7qzEpo06ZNTJkyhZkzZyIVOybeRiIi6OjoYNOmTcyaNSvtcMws4b5ubI22r/MxdWYltHfvXlpbW93JjRFJtLa2ejTArMy4rxtbo+3rnNSZlZg7ubHl99OsPPn/5tgazfvppM7MzMwsAzKT1P37b95gwys70w7DrKx0dHQwf/585s+fz5FHHsnRRx/d/7izs3NY+7jmmmt48cUXD9jmzjvv5P777x+LkM3MRsx9XU5mTpTY193L1jf2ccIRTWmHYlY2WltbWbVqFQCf//znaWpq4lOf+tSANhFBRFBVVfw73r333nvQ1/nYx3wCp5mlx31dTmZG6gC27RpeNm5W6TZs2MDcuXO57rrraGtrY8uWLSxevJj29nbmzJnDzTff3N/2jDPOYNWqVXR3dzNt2jSWLFnCKaecwjvf+U5eeeUVAD7zmc9w++2397dfsmQJp512Gm9961v50Y9+BMCuXbu49NJLOeWUU1i0aBHt7e39nbCZWSlUWl+XmZE6gG279qUdgtmQ/uc/rWHt5h1jus/Zb57K5947Z1TPXbt2Lffeey933XUXALfccgstLS10d3dz1llncdlllzF79uwBz3n99dd5z3vewy233MJNN93EPffcw5IlSwbtOyL4yU9+wrJly7j55pt57LHH+OpXv8qRRx7Jww8/zE9/+lPa2tpGFbeZlTf3den1dRkbqetKOwSzCeP444/nt3/7t/sfP/DAA7S1tdHW1sa6detYu3btoOdMmjSJ888/H4BTTz2VjRs3Ft33JZdcMqjN008/zRVXXAHAKaecwpw5o+ugzcxGopL6usyM1FVLHqmzsjbab5ml0tjY2L+8fv16/vqv/5qf/OQnTJs2jSuvvLLo9ZHq6ur6l6urq+nu7i667/r6+kFtcsUTzCzr3Nel19dlZqSuulp0+Jg6s1HZsWMHU6ZMYerUqWzZsoXHH398zF/jjDPOYOnSpQA8//zzRb8dm5mVUtb7usyM1NVUySdKmI1SW1sbs2fPZu7cuRx33HGcfvrpY/4an/jEJ7jqqquYN28ebW1tzJ07l8MOO2zMX8fMbChZ7+uUlSmRw2fNjlNvuIvHbnx32qGY9Vu3bh0nn3xy2mGUhe7ubrq7u2loaGD9+vWce+65rF+/npqakX+3LPa+SloZEe1jFW85a29vjxUrVqQdhlk/93X7pdnXZWakrrrK069m5Wznzp2cffbZdHd3ExF8/etfH1UnZ2ZWztLs6zLTo9ZUi+27OokI158zK0PTpk1j5cqVaYdhZlZSafZ1mTlRoqZKdPcGO/YWP0PFzMzMLMsyldSBq0qYmZlZZcpMUled1HLzterMzMysEmUmqesbqevY6ZE6MzMzqzyZSeqqqz39albozDPPHHRxzdtvv52PfvSjQz6nqakJgM2bN3PZZZcNud+DXVLj9ttvZ/fu3f2PL7jgAl577bXhhm5mNmzu63Iyk9TV9E2/7nZSZ9Zn0aJFPPjggwPWPfjggyxatOigz33zm9/MQw89NOrXLuzoHn30UaZNmzbq/ZmZDcV9XU5mkroqwaTaarZ5+tWs32WXXcZ3v/td9u3LHWu6ceNGNm/ezPz58zn77LNpa2vjbW97G//4j/846LkbN25k7ty5AOzZs4crrriCefPmcfnll7Nnz57+dh/5yEdob29nzpw5fO5znwPgK1/5Cps3b+ass87irLPOAmDmzJm8+uqrANx2223MnTuXuXPncvvtt/e/3sknn8yHP/xh5syZw7nnnjvgdczMhuK+Licz16kDaGms8/Srla/vL4FfPz+2+zzybXD+LUNubm1t5bTTTuOxxx5j4cKFPPjgg1x++eVMmjSJRx55hKlTp/Lqq6/yjne8gwsvvHDIazx+7WtfY/LkyaxevZrVq1fT1tbWv+2LX/wiLS0t9PT0cPbZZ7N69Wquv/56brvtNpYvX8706dMH7GvlypXce++9PPPMM0QEb3/723nPe95Dc3Mz69ev54EHHuAb3/gG73vf+3j44Ye58sorx+a9MrPx4b4OSKevy8xIHUBrU52rSpgVyJ+W6JuOiAg+/elPM2/ePM455xxefvllfvOb3wy5j6eeeqq/w5k3bx7z5s3r37Z06VLa2tpYsGABa9asOWjx6qeffpqLL76YxsZGmpqauOSSS/jhD38IwKxZs5g/fz4Ap556Khs3bjyUH93MKoj7ugyO1PnsVytbB/iWWUoXXXQRN910E8899xx79uyhra2N++67j61bt7Jy5Upqa2uZOXMme/fuPeB+in2z/fnPf86Xv/xlnn32WZqbm7n66qsPup8D1Zuur6/vX66urvb0q9lE5L4OSKevy9RIXctkT7+aFWpqauLMM8/k2muv7T9o+PXXX+eII46gtraW5cuX84tf/OKA+3j3u9/N/fffD8ALL7zA6tWrAdixYweNjY0cdthh/OY3v+H73/9+/3OmTJnCG2+8UXRf//AP/8Du3bvZtWsXjzzyCO9617vG6sc1swrlvi6DI3VO6swGW7RoEZdcckn/1MT73/9+3vve99Le3s78+fM56aSTDvj8j3zkI1xzzTXMmzeP+fPnc9pppwFwyimnsGDBAubMmcNxxx3H6aef3v+cxYsXc/7553PUUUexfPny/vVtbW1cffXV/fv4gz/4AxYsWOCpVjM7ZJXe1+lAw4MTSXt7e1z75Qf50mMvsu7m85hUV512SGasW7eOk08+Oe0wMqfY+yppZUS0pxTSuGpvb4+DXTvLbDy5ryuNkfZ1mZp+bW2sA6DDpcLMzMyswmQqqWtpzB146ClYMzMzqzQZS+r6Ruqc1Fn5yMohDuXC76dZefL/zbE1mvczk0mdq0pYuWhoaKCjo8Od3RiJCDo6OmhoaEg7FDPL475ubI22r8vc2a8A213/1crEjBkz2LRpE1u3bk07lMxoaGhgxowZaYdhZnnc14290fR1mUrqpjbUUFstT79a2aitrWXWrFlph2FmVlLu68pDpqZfJdE8uc7Tr2ZmZlZxSpbUSTpG0nJJ6yStkXRDkTYLJa2WtErSCklnFGyfKullSXcM93VbGl3/1czMzCpPKadfu4FPRsRzkqYAKyU9ERH5FXCfBJZFREiaBywF8i/3/AXgX0byoq1NdWzzderMzMyswpRspC4itkTEc8nyG8A64OiCNjtj/6kyjUD/aTOSTgXeBPxgJK/b7PqvZlZGJJ0n6UVJGyQtKbL92GRW49+SmYsL0ojTzCa+cTmmTtJMYAHwTJFtF0v6GfA94NpkXRXwl8AfjfS1Wl3/1czKhKRq4E7gfGA2sEjS7IJmnwGWRsQC4Argf41vlGaWFSVP6iQ1AQ8DN0bEjsLtEfFIRJwEXERuuhXgo8CjEfGrg+x7cXIs3oq+06hbGuvZsbebrp7eMf05zMxG4TRgQ0S8FBGdwIPAwoI2AUxNlg8DNo9jfGaWISW9pImkWnIJ3f0R8Z0DtY2IpyQdL2k68E7gXZI+CjQBdZJ2RsSSgufcDdwNuQLXAC1NybXqdnVyxFRfoNTMUnU0kP/ldBPw9oI2nwd+IOkT5A5DOWd8QjOzrCnl2a8Cvgmsi4jbhmhzQtIOSW1AHdAREe+PiGMjYibwKeBbhQndUFpdKszMyoeKrCu85P4i4L6ImAFcAPxdcgjKwB0VmZkwM8tXypG604EPAM9LWpWs+zRwLEBE3AVcClwlqQvYA1weh1hjpL9UmJM6M0vfJuCYvMczGDy9+iHgPICI+FdJDcB04JX8RsVmJszM8pUsqYuIpyn+LTW/za3ArQdpcx9w33Bft8UjdWZWPp4FTpQ0C3iZ3IkQ/7WgzS+Bs4H7JJ0MNAAeijOzEctURQnIG6nb6WvVmVm6IqIb+DjwOLnLOi2NiDWSbpZ0YdLsk8CHJf0UeAC4+lBnLMysMmWq9ivkrlMnwbbdXWmHYmZGRDwKPFqw7rN5y2vJHa5iZnZIMjdSV10lpk2qdVUJMzMzqyiZS+ogNwXrEyXMzMyskmQyqWttrKdjp5M6MzMzqxyZTOqaG2s9UmdmZmYVJZNJXUtjvZM6MzMzqyiZTOpaG+vYvruT3l5fFcDMzMwqQyaTupbGOnoDXt/jy5qYmZlZZchkUtfa5KoSZmZmVlkymdS5/quZmZlVmkwmdc2T+5I6X4DYzMzMKkMmkzpPv5qZmVmlyWRS1zf9ut1JnZmZmVWITCZ19TXVNNXXeKTOzMzMKkYmkzpw/VczMzOrLE7qzMzMzDIg00ldx04ndWZmZlYZMp3UeaTOzMzMKkVmk7rWxjq27e4kwvVfzczMLPsym9S1NNbR2d3Lrs6etEMxMzMzK7lMJ3UA23xcnZmZmVWAzCZ1+6tKuFSYmZmZZV9mk7r99V89UmdmZmbZl9mkrrWxHnD9VzMzM6sMmU3qWppc/9XMzMwqR2aTusa6aupqqjz9amZmZhUhs0mdJFob6zz9amZmZhUhs0kduKqEmZmZVY7MJ3UeqTMzM7NKkPmkbpuvU2dmZmYVIPNJ3fZdXWmHYWZmZlZymU7qWhvr2Lmvm33drv9qZmZm2ZbppK4luQCxT5YwMzOzrMt4UpfUf93ppM7MzMyyrSKSOo/UmZmZWdY5qTMzMzPLgJIldZKOkbRc0jpJayTdUKTNQkmrJa2StELSGcn6t0hamaxfI+m60cTQ6qTOzMzMKkRNCffdDXwyIp6TNAVYKemJiFib1+ZJYFlEhKR5wFLgJGAL8DsRsU9SE/CCpGURsXkkARw2qZbqKjmpMzMzs8wr2UhdRGyJiOeS5TeAdcDRBW12RkQkDxuBSNZ3RkTfVYPrRxtnVZVonlzrqhJmZmaWeeNyTJ2kmcAC4Jki2y6W9DPge8C1eeuPkbQa+BVwa7FROkmLk2nbFVu3bi362q4qYWZpknSepBclbZC0ZIg275O0Njnc5H+Pd4xmlg0lT+qS6dOHgRsjYkfh9oh4JCJOAi4CvpC3/lcRMQ84AfigpDcVee7dEdEeEe2HH3540ddvnlzn6VczS4WkauBO4HxgNrBI0uyCNicCfwycHhFzgBvHPVAzy4SSJnWSaskldPdHxHcO1DYingKOlzS9YP1mYA3wrtHE0NpU5+lXM0vLacCGiHgpIjqBB4GFBW0+DNwZEdsBIuKVcY7RzDKilGe/CvgmsC4ibhuizQlJOyS1AXVAh6QZkiYl65uB04EXRxNHrv6rkzozS8XR5A4h6bOJgmOLgd8CfkvS/5P0Y0nnFdvRcA43MbPKVsqzX08HPgA8L2lVsu7TwLEAEXEXcClwlaQuYA9weXIm7MnAX0oKQMCXI+L50QTR0ljPa3u66OkNqqt0iD+SmdmIFOt0ouBxDXAicCYwA/ihpLkR8dqAJ0XcDdwN0N7eXrgPM7PSJXUR8TTFO7T8NrcCtxZZ/wQwbyziaG2sIwK27+5kelP9WOzSzGy4NgHH5D2eARSe9LUJ+HFEdAE/l/QiuSTv2fEJ0cyyItMVJcBVJczs0En6eHIoyEg9C5woaZakOuAKYFlBm38AzkpeZzq56diXDiVeM6tMFZPUdex0Umdmo3Yk8KykpcklSoZ1LEdEdAMfBx4nd63OpRGxRtLNki5Mmj1O7ljitcBy4I8ioqMEP4OZZVwpj6krCx6pM7NDFRGfkfSnwLnANcAdkpYC34yI/zjIcx8FHi1Y99m85QBuSm5mZqOW+ZG6/vqvu53UmdnoJcnXr5NbN9AMPCTpS6kGZmaWyPxIXXNfUufpVzMbJUnXAx8EXgX+htwUaZekKmA98N/TjM/MDCogqautrmJqQ41LhZnZoZgOXBIRv8hfGRG9kn4/pZjMzAbI/PQrQGtTvatKmNmheBTY1vdA0hRJbweIiHWpRWVmlqcikrrmybU+UcLMDsXXgJ15j3cl68zMykZFJHUtjfVO6szsUCg5UQLITbtSAYevmNnEUhFJXWtjnadfzexQvCTpekm1ye0GfIFgMyszFZHUtTTVsX1XJ3lftM3MRuI64HeAl8mV9Xo7sDjViMzMClTE9EFrYx3dvcGOvd0cNqk27XDMbIKJiFfIlfgyMytbw0rqJB0PbIqIfZLOBOYB34qI10oZ3FjJryrhpM7MRkpSA/AhYA7Q0Lc+Iq5NLSgzswLDnX59GOiRdALwTWAW8L9LFtUY678Asa9VZ2aj83fk6r/+J+BfgBnAG6lGZGZWYLhJXW9SmPpi4PaI+EPgqNKFNbb6SoV1uKqEmY3OCRHxp8CuiPhb4D8Db0s5JjOzAYab1HVJWkSuTM53k3UTZh4zf/rVzGwUupL71yTNBQ4DZqYXjpnZYMNN6q4B3gl8MSJ+LmkW8PelC2tstTbWA7Btt5M6MxuVuyU1A58BlgFrgVvTDcnMbKBhnSgREWuB6wGSjm1KRNxSysDG0qS6aibVVrPN069mNkKSqoAdEbEdeAo4LuWQzMyKGtZInaR/ljRVUgvwU+BeSbeVNrSx1dJY5+lXMxuxpHrEx9OOw8zsYIY7/XpYROwALgHujYhTgXNKF9bYa3FVCTMbvSckfUrSMZJa+m5pB2Vmlm+4Fx+ukXQU8D7gT0oYT8l4pM7MDkHf9eg+lrcu8FSsmZWR4SZ1NwOPA/8vIp6VdBywvnRhjb3Wxjo2vLIz7TDMbAKKiFlpx2BmdjDDPVHi28C38x6/BFxaqqBKwSN1ZjZakq4qtj4ivjXesZiZDWW4ZcJmAF8FTic35fA0cENEbCphbGOqpamOPV097OnsYVJdddrhmNnE8tt5yw3A2cBzgJM6Mysbw51+vZdcWbD/kjy+Mln3e6UIqhT6q0rs2seMuskpR2NmE0lEfCL/saTDyJUOMzMrG8M9+/XwiLg3IrqT233A4SWMa8w1T3ZVCTMbM7uBE9MOwsws33BH6l6VdCXwQPJ4EdBRmpBKo7Wpb6TOSZ2ZjYykfyJ36AnkvgzPBpamF5GZ2WDDTequBe4A/opcx/YjcqXDJoyWvlJhriphZiP35bzlbuAXE+mYYjOrDMM9+/WXwIX56yTdCNxeiqBKoSU5pm6767+a2cj9EtgSEXsBJE2SNDMiNqYblpnZfsM9pq6Ym8YsinEwtaGG2mp5+tXMRuPbQG/e4x7yLvNkZlYODiWp05hFMQ4k0Ty5ztOvZjYaNRHR33kky3UpxmNmNsihJHVx8CblxfVfzWyUtkrqPwRF0kLg1RTjMTMb5IDH1El6g+LJm4BJJYmohHJVJfalHYaZTTzXAfdLuiN5vAkoWmXCzCwtB0zqImLKeAUyHloa63jh5dfTDsPMJpiI+A/gHZKaAEXEG2nHZGZW6FCmXyecVtd/NbNRkPTnkqZFxM6IeENSs6Q/SzsuM7N8FZXUtTTWs2NvN109vQdvbGa23/kR8Vrfg4jYDlyQYjxmZoOULKmTdIyk5ZLWSVoj6YYibRZKWi1plaQVks5I1s+X9K/J81ZLunwsYmpJqkps92idmY1MtaT6vgeSJgH1B2hvZjbuhltRYjS6gU9GxHOSpgArJT0REWvz2jwJLIuIkDSPXNmdk8jVVbwqItZLenPy3MfzvymPRsvk/aXCjpjacCi7MrPK8vfAk5LuTR5fA/xtivGYmQ1SsqQuIrYAW5LlNyStA44G1ua12Zn3lEaSM20j4t/z2myW9ApwOHBoSV1SVcLH1ZnZSETElyStBs4hd/b/Y8Bb0o3KzGygcTmmTtJMYAHwTJFtF0v6GfA9cjVmC7efRu4in/9RZNviZNp2xdatWw8aR2vT/pE6M7MR+jW5qhKXAmcD69INx8xsoJIndcklAB4GboyIHYXbI+KRiDgJuAj4QsFzjwL+DrgmIgad3RARd0dEe0S0H3744QeNpb/+q5M6MxsGSb8l6bPJTMMdwK/IXdLkrIi44yBP79vHeZJelLRB0pIDtLtMUkhqH6PwzazClDSpk1RLLqG7PyK+c6C2EfEUcLyk6clzp5IbvftMRPx4LOJpnlyH5JE6Mxu2n5EblXtvRJwREV8lV/d1WCRVA3cC5wOzgUWSZhdpNwW4niKzGWZmw1XKs18FfBNYFxG3DdHmhKQdktrITbN2SKoDHgG+FRFjVjS7ukpMm1TrqhJmNlyXkpt2XS7pG5LOZmR1r08DNkTES0m92AeBhUXafQH4ErD3UAM2s8pVypG604EPAL+bXLJklaQLJF0n6bqkzaXAC5JWkfs2e3lEBPA+4N3A1XnPnT8WQTX7AsRmNkzJ4SGXkzsr/5+BPwTeJOlrks4dxi6OJjdl22dTsq6fpAXAMRHx3QPtaKTHEJtZ5Snl2a9Pc5BvtBFxK3BrkfV/T+4SAmOutbGOjp1O6sxs+CJiF3A/ufqvLcB/AZYAPzjIU4v1gf31tCVVAX8FXD2MGO4G7gZob28vVpPbzCpcRVWUgNzJEh6pM7PRiohtEfH1iPjdYTTfBByT93gGsDnv8RRgLvDPkjYC7wCW+WQJMxuNCkzq6tm+20mdmY2LZ4ETJc1KjhW+AljWtzEiXo+I6RExMyJmAj8GLoyIFemEa2YTWcUlda2NdWzf3UVvr2cvzKy0IqIb+DjwOLnr2i2NiDWSbpZ0YbrRmVnWlLJMWFlqaayjpzd4fU8Xzcl168zMSiUiHgUeLVj32SHanjkeMZlZNlXcSF3fBYh9rTozMzPLkopN6nyyhJmZmWVJBSd1vgCxmZmZZUfFJXWtTX1JXVfKkZiZmZmNnYpL6jxSZ2ZmZllUcUldfU01TfU1PlHCzMzMMqXikjqA5sZanyhhZmZmmVKRSV1LY72TOjMzM8uUikzqWhvr6NjppM7MzMyyoyKTupbGOtd/NTMzs0ypyKSutbGOjl2dRLj+q5mZmWVDRSZ1LY11dHb3squzJ+1QzMzMzMZERSZ1zX3XqvNxdWZmZpYRFZnUtSZJXYcvQGxmZmYZUZFJ3f6qEh6pMzMzs2yoyKSutbEewFUlzMzMLDMqMqlracqN1G13UmdmZmYZUZFJXWNdNXU1VZ5+NTMzs8yoyKROEi2T6zz9amZmZplRkUkd5E6W8EidmZmZZUXFJnWtTR6pMzMzs+yo2KQuN1Ln69SZmZlZNlR0Urd9V1faYZiZmZmNiYpN6lob69i5r5t93a7/amZmZhNfxSZ1za4qYWZmZhlSsUldf/3XnU7qzMzMbOKr2KSuJSkV5pE6MzMzy4IKTuo8/WpmZmbZUbFJXauTOjMzM8uQik3qDptUS5Wc1JmZmVk2VGxSV1Ulml3/1czMzDKiYpM6cFUJMzMzyw4ndR6pMzMzswwoWVIn6RhJyyWtk7RG0g1F2iyUtFrSKkkrJJ2Rt+0xSa9J+m6pYmxt8vSrmZmZZUNNCffdDXwyIp6TNAVYKemJiFib1+ZJYFlEhKR5wFLgpGTbXwCTgf9WqgBz9V+d1JmZmdnEV7KRuojYEhHPJctvAOuAowva7IyISB42ApG37UngjVLFB7kLEL+2p4ue3jh4YzMzM7MyNi7H1EmaCSwAnimy7WJJPwO+B1w7wv0uTqZtV2zdunXEcbVMriUCtu/2aJ2ZmZlNbCVP6iQ1AQ8DN0bEjsLtEfFIRJwEXAR8YST7joi7I6I9ItoPP/zwEcfW0uRSYWZWWpLOk/SipA2SlhTZfpOktcnxxU9KeksacZrZxFfSpE5SLbmE7v6I+M6B2kbEU8DxkqaXMqZ8fVUlOnY6qTOzsSepGrgTOB+YDSySNLug2b8B7RExD3gI+NL4RmlmWVHKs18FfBNYFxG3DdHmhKQdktqAOqCjVDEVcv1XMyux04ANEfFSRHQCDwIL8xtExPKI2J08/DEwY5xjNLOMKOXZr6cDHwCel7QqWfdp4FiAiLgLuBS4SlIXsAe4vO/ECUk/JHcmbJOkTcCHIuLxsQywv/6rj6kzs9I4GvhV3uNNwNsP0P5DwPdLGpGZZVbJkrqIeBrQQdrcCtw6xLZ3lSKufM19SZ2nX82sNIr1gUVPt5d0JdAOvGeI7YuBxQDHHnvsWMVnZhlS0RUlaqurmNJQ41JhZlYqm4Bj8h7PADYXNpJ0DvAnwIURUbRDOtQTw8ws+yo6qYPcFKyrSphZiTwLnChplqQ64ApgWX4DSQuAr5NL6F5JIUYzy4iKT+pc/9XMSiUiuoGPA4+TuwD70ohYI+lmSRcmzf4CaAK+nZRMXDbE7szMDqiUJ0pMCC2N9WzavvvgDc3MRiEiHgUeLVj32bzlc8Y9KDPLpIofqWv1SJ2ZmZllQMUndS1NdWzf3cn+ErRmZmZmE4+Tusl1dPUEO/Z2px2KmZmZ2QRfc18AAA7ESURBVKg5qXNVCTMzM8sAJ3VNfUmdr1VnZmZmE1fFJ3V9pcI6XFXCzMzMJrCKT+r6pl+3u/6rmZmZTWAVn9S1NtYDuKqEmZmZTWgVn9RNqqumobaKbZ5+NTMzswms4pM6yI3W+exXMzMzm8ic1JE7rs7Tr2ZmZjaROakjl9R5pM7MzMwmMid1uP6rmZmZTXw1aQdQDjxSZ2ZZ0t3TS2dPL13dwb6eHrp6gs7uXjq7e+nq6WVfd29Z1buW0o7ALBuc1AHNjXXs6ephT2cPk+qq0w7HzOyA1r+yk3Nu+xe6enr7k7XOnv1JW2/55GtmNo6c1JFXVWLXPmbUTU45GjOzAzum92X+V9efUgWoWqgGqiSkgntAElUafG9mE1PzAbY5qWN/VYltuzqZ0eykzszKW0NNFb91xJS0wzCzMuOkDmht6hup83F1ZjYBTD8Rrvle2lGYWRquHXqo3We/Ai1JqbDtTurMzMxsgnJSx8DpVzMzM7OJyEkdMLWhhpoqefrVzMzMJiwndeTOBmturGPbTid1ZmZmNjE5qUu0uv6rmZmZTWBO6hK5qhL70g7DzMzMbFSc1CVcKszMzMwmsuxfp66nG3r2Qfc+6N6b3PYNuj+j699p2vUKrN566K+pKqiqBlUX3I9gPUD05t168pajYFverbdn//beruTn60zeg87970VPZ/Lz7xt6W08nIKhtgNpJUDMpWZ4MNcm6ousL2tQ0JLf6/fdV41iOLQJ6u3M/T08n9HQVWe4aYn3Bcm93sV94wcNi1xDSwO3VtVBdDzV1ufekb3nAurrce1W4biSFMiNyN2Lw54dIPi890Jt8xvofF6zv/2z1bct73LdPYuC+B6yLgnUFz6mqyX1u+j4rfZ+b2knJ5ya5d5FQM7MhZSep2/oz+MqCwQlb0T/Cg12R3PhOKYMsM9V1g5OJwkQjemF3B3Tthe490LVn/3LPIYxsVtUOTvT6b4XrG3KJ7oAka9/+ZKs7b3nA+s79zyFDxTD7fm+Ql0AVJPt96zJF+780FEv+zMwqXHaSuup6OPrUvGSgSGJwgPt/WrONv1q+kW9fdzqtTfWjjyPiwCMew1qf/DFW1eBb38ieVHy7qnIJkJQb/aipT0Z8GvKW60c+4lNMb08uyevem3e/Oy8BTB73J9mDR0jp3lswkrpv//72bM9LznuSZKYuGeVKfo7aSdBwWN62uiQ5LXarLbgfxXJV9cD3LYaRLBa2id79CWh3Z/IedBYfNR0wclqwDiWfAyXLVQM/G/3r8u4pWB7uSHLRbX2ftSH2PSCOvnUUWafcl6+uvpH0vQWfq337P0/994Xt9h7aZ9nMLAOyk9S1zIJL/2bUT6/q2MJL0ckrdTNobZ06hoFlWFU11DflbmZpO0DpHDOzSpCdpO4Q9VWV+PNH19HSWJcbcAN6k4XeiGRd0Nt/mFAQ5O57k/YRgSSqBdVVSpZFdZWoqhJVgmrlLVeJKuVu+5eHHgwqHBcauK34qJGS47n69qn85WRB/f8Mbj/U6xV9zeHOchYehlawYqhBxL7X73/dgXdEXoBRsK3Yyxd9HwoaFXs/DhRHXwyFr1/43vX9nqukZBBr/+++bxnlXr1KStZDVdX+QPo/e8lnc+DnMvncUmR7QG+RN6bw9zmcgcjhKPx9DvX77vtsKvm5+96XQdvZ/370vUdmZpXOSV3ipCOnMPuoqfyiYze/3La7/w8pyR/dvj8mVclfl/4/sgO259b1BvT2Br0R9OTdR0BP/nJv0BNBJOvy2xTK/6Olgr+QxRKRfkWSnsIko++PfX47kiSg6J/LQX+gCx4f5C/s4MSweMwDV+XFMvAu7w/+4ORrf5uBiVD+ywxK0BickJHXZjhxaNC2gUnjwC8DuXvylvMTs9FQwee2PwkakDwOkQwd9Pd74F9wFAR9sN9v4WdzQAKa1yB/3aG+P2ZmWeSkLtHcWMejN7wr7TDMBtk/sjZwBG7QFwr2j3JVmoig6ta0ozAzS1fJrlMn6RhJyyWtk7RG0g1F2iyUtFrSKkkrJJ2Rt+2DktYntw+WKk6zcqdkur6muora6irqaqpoqK2mvqaa2uoqaqqr+qf3KzGhg8pMZM3MCpVypK4b+GREPCdpCrBS0hMRsTavzZPAsogISfOApcBJklqAzwHt5GZbVkpaFhHbSxivmZmZ2YRVspG6iNgSEc8ly28A64CjC9rsjP0H4DSy//Ca/wQ8ERHbkkTuCeC8UsVqZmZmNtGNS5kwSTOBBcAzRbZdLOlnwPeAa5PVRwO/ymu2iYKEMHnu4mTadsXWrWNQCcLMzMxsgip5UiepCXgYuDEidhRuj4hHIuIk4CLgC31PK7KrwSfRRdwdEe0R0X744YePZdhmZmZmE0pJkzpJteQSuvsj4oAFuCLiKeB4SdPJjcwdk7d5BrC5ZIGamZmZTXClPPtVwDeBdRFx2xBtTkjaIakNqAM6gMeBcyU1S2oGzk3WmZlNKJLOk/SipA2SlhTZXi/p/yTbn0kOVzEzG7FSnv16OvAB4HlJq5J1nwaOBYiIu4BLgaskdQF7gMuTEye2SfoC8GzyvJsjYlsJYzUzG3OSqoE7gd8jNwPxbHImf/5VAD4EbI+IEyRdAdwKXD7+0ZrZRFeypC4inqb4sXH5bW4l14EV23YPcE8JQjMzGy+nARsi4iUASQ8CC4H8pG4h8Plk+SHgDknKuzKAmdmwZKaixMqVK3dKejHtOBLTgVfTDgLHUahc4oDyiSVLcbxlLAIZY8XO5H/7UG0iolvS60ArBe+HpMXA4uThPkkvlCTikcnS52eslEssjmOgLMUxZF+XmaQOeDEi2tMOAkDSinKIxXGUZxxQPrE4jpIbzpn8wz7bH7gbyuf9chyDlUssjqMy4xiX69SZmVWo4ZzJ399GUg1wGOBjiM1sxJzUmZmVzrPAiZJmSaoDrgCWFbRZBvTVt74M+L8+ns7MRiNL0693px1AnnKJxXEMVC5xQPnE4jhKKDlG7uPkLslUDdwTEWsk3QysiIhl5C799HeSNpAbobtiGLsul/fLcQxWLrE4joEqIg75C6GZmZnZxOfpVzMzM7MMcFJnZmZmlgGZSOoOVoZnnGI4RtJySeskrZF0Qxpx5MVTLenfJH035TimSXpI0s+S9+adKcXxh8nv5QVJD0hqGKfXvUfSK/nXFJPUIukJSeuT++aU4viL5PeyWtIjkqaVOo6hYsnb9ilJkdSAtgLl0Nclcbi/GxxDRfd1yWu7vztIHHnbStLXTfikLq8Mz/nAbGCRpNkphNINfDIiTgbeAXwspTj63ACsS/H1+/w18FhEnAScQgoxSToauB5oj4i55A5YH87B6GPhPuC8gnVLgCcj4kTgyeRxGnE8AcyNiHnAvwN/PA5xDBULko4hV07rl+MUx4RSRn0duL8rptL7OnB/N5w4StrXTfikjrwyPBHRCfSV4RlXEbElIp5Llt8g9x/66PGOA0DSDOA/A3+TxuvnxTEVeDe5s/uIiM6IeC2lcGqAScl1wCYz+FphJRERTzH4mmMLgb9Nlv8WuCiNOCLiBxHRnTz8MblrqJXcEO8JwF8B/50iF941oEz6OnB/VySGiu/rwP3dcOJIlKyvy0JSV6wMTyqdSx9JM4EFwDMphXA7uQ9Mb0qv3+c4YCtwbzI18jeSGsc7iIh4GfgyuW9FW4DXI+IH4x1HnjdFxJYkti3AESnG0uda4PtpvbikC4GXI+KnacUwAZRdXwfu7xLu64bm/i5Pqfu6LCR1wyqxM14kNQEPAzdGxI4UXv/3gVciYuV4v3YRNUAb8LWIWADsYnyG3gdIjuFYCMwC3gw0SrpyvOMoV5L+hNx02v0pvf5k4E+Az6bx+hNIWfV14P4uj/u6CSLN/m48+rosJHXDKcMzLiTVkuvg7o+I76QRA3A6cKGkjeSmZ35X0t+nFMsmYFNE9H2Df4hcxzfezgF+HhFbI6IL+A7wOynE0ec3ko4CSO5fSSsQSR8Efh94f4pVDI4n90fop8nndgbwnKQjU4qnXJVNXwfu7wq4rxua+7v9St7XZSGpG04ZnpKTJHLHU6yLiNvG+/X7RMQfR8SMiJhJ7r34vxGRyje1iPg18CtJb01WnQ2sTSGUXwLvkDQ5+T2dTboHVeeXhfog8I9pBCHpPOB/ABdGxO40YgCIiOcj4oiImJl8bjcBbcnnx/Yri74O3N8VicN93dDc3yXGo6+b8EldcuBjXxmedcDSiFiTQiinAx8g901xVXK7IIU4ys0ngPslrQbmA38+3gEk354fAp4Dnif3uR+XkjGSHgD+FXirpE2SPgTcAvyepPXkzoC6JaU47gCmAE8kn9e7Sh3HAWKxgyijvg7c3xVT0X0duL8bZhylfU2XCTMzMzOb+Cb8SJ2ZmZmZOakzMzMzywQndWZmZmYZ4KTOzMzMLAOc1JmZmZllgJM6KzuSevIuk7BK0phdmV3STEkvjNX+zMxGy32djbWatAMwK2JPRMxPOwgzsxJzX2djyiN1NmFI2ijpVkk/SW4nJOvfIulJSauT+2OT9W+S9Iiknya3vnI51ZK+IWmNpB9ImpTaD2VmVsB9nY2WkzorR5MKpiQuz9u2IyJOI3d18NuTdXcA34qIeeSKNH8lWf8V4F8i4hRydRj7rr5/InBnRMwBXgMuLfHPY2ZWjPs6G1OuKGFlR9LOiGgqsn4j8LsR8VJSTPzXEdEq6VXgqIjoStZviYjpkrYCMyJiX94+ZgJPRMSJyeP/AdRGxJ+V/iczM9vPfZ2NNY/U2UQTQywP1aaYfXnLPfjYUjMrP+7rbMSc1NlEc3ne/b8myz8CrkiW3w88nSw/CXwEQFK1pKnjFaSZ2SFyX2cj5qzdytEkSavyHj8WEX2n+tdLeobcF5JFybrrgXsk/RGwFbgmWX8DcLekD5H7lvoRYEvJozczGx73dTamfEydTRjJcSbtEfFq2rGYmZWK+zobLU+/mpmZmWWAR+rMzMzMMsAjdWZmZmYZ4KTOzMzMLAOc1JmZmZllgJM6MzMzswxwUmdmZmaWAf8fey42RUCIEWYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Find the best in terms of final validation set accuracy\n",
    "i = best_scores.argmin()\n",
    "h = histories[i]\n",
    "\n",
    "print('Hyperparameters: trial %i hidden %i-%i-%i dropout %.3f opt %s' %\n",
    "      (i, grid_h1[i], grid_h2[i], grid_h3[i], grid_dropout[i], grid_optimizer[i]))\n",
    "print('  Last validation accuracy %.4f' % last_scores[i])\n",
    "print('  Best validation accuracy %.4f' % best_scores[i])\n",
    "\n",
    "plt.figure(figsize=(9,4))\n",
    "\n",
    "# Loss\n",
    "plt.subplot(121)\n",
    "plt.plot(h['loss'], label='Training')\n",
    "plt.plot(h['val_loss'], label='Validation')\n",
    "plt.xlim(xmin=0, xmax=len(h['loss'])-1)\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Loss')\n",
    "plt.legend(loc=0)\n",
    "\n",
    "# Accuracy\n",
    "plt.subplot(122)\n",
    "plt.plot(h['acc'], label='Training')\n",
    "plt.plot(h['val_acc'], label='Validation')\n",
    "plt.xlim(xmin=0, xmax=len(h['loss'])-1)\n",
    "plt.ylim((0, 1))\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend(loc=0)\n",
    "\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test set evaluation\n",
    "\n",
    "Here we load the best selected model and evaluate final performance on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/common/software/tensorflow/intel-tensorflow/1.13.1-py36/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/common/software/tensorflow/intel-tensorflow/1.13.1-py36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /usr/common/software/tensorflow/intel-tensorflow/1.13.1-py36/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "i = best_scores.argmax()\n",
    "model_file = os.path.join(checkpoint_dir, 'model_%i.h5' % i)\n",
    "model = keras.models.load_model(model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loss, test_acc = model.evaluate(x_test, y_test, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test loss: 0.0248\n",
      "Test accuracy: 0.9920\n"
     ]
    }
   ],
   "source": [
    "print('Test loss: %.4f' % test_loss)\n",
    "print('Test accuracy: %.4f' % test_acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow-intel(cpu)/1.13.1-py36",
   "language": "python",
   "name": "tensorflow_intel_1.13.1_py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}