mnist_neuralGAN_vs_cnnGAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"191pt\" viewBox=\"0.00 0.00 157.00 191.00\" width=\"157pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 187)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-187 153,-187 153,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2099373095904 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2099373095904</title>\n",
       "<polygon fill=\"none\" points=\"11.5,-146.5 11.5,-182.5 137.5,-182.5 137.5,-146.5 11.5,-146.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"74.5\" y=\"-160.8\">input_3: InputLayer</text>\n",
       "</g>\n",
       "<!-- 2099357224408 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2099357224408</title>\n",
       "<polygon fill=\"none\" points=\"0,-73.5 0,-109.5 149,-109.5 149,-73.5 0,-73.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"74.5\" y=\"-87.8\">sequential_7: Sequential</text>\n",
       "</g>\n",
       "<!-- 2099373095904&#45;&gt;2099357224408 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2099373095904-&gt;2099357224408</title>\n",
       "<path d=\"M74.5,-146.313C74.5,-138.289 74.5,-128.547 74.5,-119.569\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"78.0001,-119.529 74.5,-109.529 71.0001,-119.529 78.0001,-119.529\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099356332888 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2099356332888</title>\n",
       "<polygon fill=\"none\" points=\"0,-0.5 0,-36.5 149,-36.5 149,-0.5 0,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"74.5\" y=\"-14.8\">sequential_8: Sequential</text>\n",
       "</g>\n",
       "<!-- 2099357224408&#45;&gt;2099356332888 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2099357224408-&gt;2099356332888</title>\n",
       "<path d=\"M74.5,-73.3129C74.5,-65.2895 74.5,-55.5475 74.5,-46.5691\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"78.0001,-46.5288 74.5,-36.5288 71.0001,-46.5289 78.0001,-46.5288\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epochs: 50\n",
      "Batch size: 128\n",
      "Batches per epoch: 468\n",
      "--------------- Epoch 1 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|                                                                                          | 0/468 [00:00<?, ?it/s]\n",
      "Exception in thread Thread-8:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Anaconda3\\envs\\tf-gpu\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Anaconda3\\envs\\tf-gpu\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n",
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:11<00:00,  6.56it/s]\n",
      "C:\\Anaconda3\\Lib\\site-packages\\matplotlib\\pyplot.py:524: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 2 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 3 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 4 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 5 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 6 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 7 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 8 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 9 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 10 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 11 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 12 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 13 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 14 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 15 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 16 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 17 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 18 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 19 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 20 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 21 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 22 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 23 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 24 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 25 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 26 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 27 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 28 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 29 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 30 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 31 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 32 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 33 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 34 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 35 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 36 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 37 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 38 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 39 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 40 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 41 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 42 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 43 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 44 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 45 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 46 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 47 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 48 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 49 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------- Epoch 50 ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 468/468 [01:08<00:00,  6.83it/s]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import SVG, display\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import os\n",
    "from keras.layers import Input\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers.core import Reshape, Dense, Dropout, Flatten\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.layers.convolutional import Conv2D, UpSampling2D\n",
    "from keras.datasets import mnist\n",
    "from keras.optimizers import Adam\n",
    "from keras import backend as K\n",
    "from keras import initializers\n",
    "import cv2\n",
    "K.set_image_dim_ordering('th')\n",
    "\n",
    "np.random.seed(1000)\n",
    "\n",
    "randomDim = 100\n",
    "\n",
    "# Load MNIST data\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = (X_train.astype(np.float32) - 127.5)/127.5\n",
    "X_train = X_train[:, np.newaxis, :, :]\n",
    "\n",
    "# Optimizer\n",
    "adam = Adam(lr=0.0002, beta_1=0.5)\n",
    "\n",
    "# Generator\n",
    "generator = Sequential()\n",
    "generator.add(Dense(128*7*7, input_dim=randomDim, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Reshape((128, 7, 7)))\n",
    "generator.add(UpSampling2D(size=(2, 2)))\n",
    "generator.add(Conv2D(16, kernel_size=(3, 3), padding='same'))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(UpSampling2D(size=(2, 2)))\n",
    "generator.add(Conv2D(1, kernel_size=(3, 3), padding='same', activation='tanh'))\n",
    "generator.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "# Discriminator\n",
    "discriminator = Sequential()\n",
    "discriminator.add(Conv2D(16, kernel_size=(3, 3), strides=(1, 1), padding='same', input_shape=(1, 28, 28), kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.2))\n",
    "# discriminator.add(Conv2D(128, kernel_size=(3, 3), strides=(1, 1), padding='same'))\n",
    "# discriminator.add(LeakyReLU(0.2))\n",
    "# discriminator.add(Dropout(0.2))\n",
    "discriminator.add(Conv2D(32, kernel_size=(3, 3), strides=(1, 1), padding='same'))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.2))\n",
    "discriminator.add(Flatten())\n",
    "discriminator.add(Dense(1, activation='sigmoid'))\n",
    "discriminator.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "discriminator.trainable = False\n",
    "ganInput = Input(shape=(randomDim,))\n",
    "x = generator(ganInput)\n",
    "ganOutput = discriminator(x)\n",
    "gan = Model(inputs=ganInput, outputs=ganOutput)\n",
    "# gan.layers[1].name = \"Generator_Network\"\n",
    "# gan.layers[2].name = \"Discriminator_Network\"\n",
    "gan.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "display(SVG(model_to_dot(gan).create(prog='dot', format='svg')))\n",
    "\n",
    "dLosses = []\n",
    "gLosses = []\n",
    "\n",
    "def plotLoss(epoch):\n",
    "    plt.figure(figsize=(10, 8))\n",
    "    plt.plot(dLosses, label='Discriminitive loss')\n",
    "    plt.plot(gLosses, label='Generative loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    plt.savefig('images/gan_cnn/dcgan_%d_loss_epoch.png' % epoch)\n",
    "\n",
    "def plotGeneratedImages(epoch, examples=100, dim=(1, 1), figsize=(2, 2)):\n",
    "    randomDim = 100\n",
    "    examples=1\n",
    "    \n",
    "    for counter in range(0,50):\n",
    "        noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "        generatedImages = generator.predict(noise)\n",
    "        generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "        # generatedImages.shape\n",
    "        plt.subplot(2, 2, 1)\n",
    "        plt.figure(figsize=(2,2))\n",
    "        plt.imshow(generatedImages[0], cmap = 'gray_r', interpolation = 'bicubic')\n",
    "        plt.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis  \n",
    "        plt.tight_layout()\n",
    "        fol = 'images/gan_cnn/digits/epoch_'+str(epoch)+'/'\n",
    "        if not os.path.exists(fol):\n",
    "            os.makedirs(fol)\n",
    "        plt.savefig(fol+'random_%d.png' % counter)\n",
    "    \n",
    "    \n",
    "def saveModels(epoch):\n",
    "    fol = 'models/gan_cnn/'\n",
    "    if not os.path.exists(fol):\n",
    "        os.makedirs(fol)\n",
    "    generator.save(fol+'dcgan_generator_epoch_%d.h5' % epoch)\n",
    "    discriminator.save(fol+'dcgan_discriminator_epoch_%d.h5' % epoch)\n",
    "\n",
    "def train(epochs=1, batchSize=128):\n",
    "    batchCount = X_train.shape[0] / batchSize\n",
    "    print ('Epochs:', epochs)\n",
    "    print ('Batch size:', batchSize)\n",
    "    print ('Batches per epoch:', int(batchCount))\n",
    "\n",
    "    for e in range(1, epochs+1):\n",
    "        print ('-'*15, 'Epoch %d' % e, '-'*15)\n",
    "        for _ in tqdm(range(int(batchCount))):\n",
    "            # Get a random set of input noise and images\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            imageBatch = X_train[np.random.randint(0, X_train.shape[0], size=batchSize)]\n",
    "\n",
    "            # Generate fake MNIST images\n",
    "            generatedImages = generator.predict(noise)\n",
    "            X = np.concatenate([imageBatch, generatedImages])\n",
    "#             print(X[0].reshape((28,28,1)).shape)\n",
    "#             cv2.imshow(\"img\",X[0].reshape((28,28,1)))\n",
    "#             cv2.waitKey(0)\n",
    "            \n",
    "#             cv2.imshow(\"img\",X[batchSize+1].reshape((28,28,1)))\n",
    "#             cv2.waitKey(0)\n",
    "            # Labels for generated and real data\n",
    "            yDis = np.zeros(2*batchSize)\n",
    "            # One-sided label smoothing\n",
    "            yDis[:batchSize] = 0.9\n",
    "\n",
    "            # Train discriminator\n",
    "            discriminator.trainable = True\n",
    "            dloss = discriminator.train_on_batch(X, yDis)\n",
    "\n",
    "            # Train generator\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            yGen = np.ones(batchSize)\n",
    "            discriminator.trainable = False\n",
    "            gloss = gan.train_on_batch(noise, yGen)\n",
    "\n",
    "        # Store loss of most recent batch from this epoch\n",
    "        dLosses.append(dloss)\n",
    "        gLosses.append(gloss)\n",
    "\n",
    "        if e == 1 or e % 5 == 0:\n",
    "            plotGeneratedImages(e)\n",
    "            saveModels(e)\n",
    "\n",
    "    # Plot losses from every epoch\n",
    "    plotLoss(e)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    print(\"\")\n",
    "    train(50, 128)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "discriminator.predict(X_train[0].reshape((1,1,28,28)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Discriminator Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"719pt\" viewBox=\"0.00 0.00 359.00 719.00\" width=\"359pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 715)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-715 355,-715 355,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2099368171952 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2099368171952</title>\n",
       "<polygon fill=\"none\" points=\"0.5,-664.5 0.5,-710.5 350.5,-710.5 350.5,-664.5 0.5,-664.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"91.5\" y=\"-683.8\">conv2d_15_input: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"182.5,-664.5 182.5,-710.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"210.5\" y=\"-695.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"182.5,-687.5 238.5,-687.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"210.5\" y=\"-672.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"238.5,-664.5 238.5,-710.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"294.5\" y=\"-695.3\">(None, 1, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"238.5,-687.5 350.5,-687.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"294.5\" y=\"-672.3\">(None, 1, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099362753952 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2099362753952</title>\n",
       "<polygon fill=\"none\" points=\"21,-581.5 21,-627.5 330,-627.5 330,-581.5 21,-581.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88\" y=\"-600.8\">conv2d_15: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"155,-581.5 155,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"183\" y=\"-612.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"155,-604.5 211,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"183\" y=\"-589.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"211,-581.5 211,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"270.5\" y=\"-612.3\">(None, 1, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"211,-604.5 330,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"270.5\" y=\"-589.3\">(None, 16, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099368171952&#45;&gt;2099362753952 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2099368171952-&gt;2099362753952</title>\n",
       "<path d=\"M175.5,-664.366C175.5,-656.152 175.5,-646.658 175.5,-637.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-637.607 175.5,-627.607 172,-637.607 179,-637.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099353435496 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2099353435496</title>\n",
       "<polygon fill=\"none\" points=\"0,-498.5 0,-544.5 351,-544.5 351,-498.5 0,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88\" y=\"-517.8\">leaky_re_lu_14: LeakyReLU</text>\n",
       "<polyline fill=\"none\" points=\"176,-498.5 176,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"204\" y=\"-529.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"176,-521.5 232,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"204\" y=\"-506.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"232,-498.5 232,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"291.5\" y=\"-529.3\">(None, 16, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"232,-521.5 351,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"291.5\" y=\"-506.3\">(None, 16, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099362753952&#45;&gt;2099353435496 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2099362753952-&gt;2099353435496</title>\n",
       "<path d=\"M175.5,-581.366C175.5,-573.152 175.5,-563.658 175.5,-554.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-554.607 175.5,-544.607 172,-554.607 179,-554.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099368107032 -->\n",
       "<g class=\"node\" id=\"node4\"><title>2099368107032</title>\n",
       "<polygon fill=\"none\" points=\"23,-415.5 23,-461.5 328,-461.5 328,-415.5 23,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88\" y=\"-434.8\">dropout_6: Dropout</text>\n",
       "<polyline fill=\"none\" points=\"153,-415.5 153,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"181\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"153,-438.5 209,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"181\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"209,-415.5 209,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"268.5\" y=\"-446.3\">(None, 16, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"209,-438.5 328,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"268.5\" y=\"-423.3\">(None, 16, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099353435496&#45;&gt;2099368107032 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>2099353435496-&gt;2099368107032</title>\n",
       "<path d=\"M175.5,-498.366C175.5,-490.152 175.5,-480.658 175.5,-471.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-471.607 175.5,-461.607 172,-471.607 179,-471.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099368106920 -->\n",
       "<g class=\"node\" id=\"node5\"><title>2099368106920</title>\n",
       "<polygon fill=\"none\" points=\"21,-332.5 21,-378.5 330,-378.5 330,-332.5 21,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88\" y=\"-351.8\">conv2d_16: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"155,-332.5 155,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"183\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"155,-355.5 211,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"183\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"211,-332.5 211,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"270.5\" y=\"-363.3\">(None, 16, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"211,-355.5 330,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"270.5\" y=\"-340.3\">(None, 32, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099368107032&#45;&gt;2099368106920 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>2099368107032-&gt;2099368106920</title>\n",
       "<path d=\"M175.5,-415.366C175.5,-407.152 175.5,-397.658 175.5,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-388.607 175.5,-378.607 172,-388.607 179,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099355393944 -->\n",
       "<g class=\"node\" id=\"node6\"><title>2099355393944</title>\n",
       "<polygon fill=\"none\" points=\"0,-249.5 0,-295.5 351,-295.5 351,-249.5 0,-249.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88\" y=\"-268.8\">leaky_re_lu_15: LeakyReLU</text>\n",
       "<polyline fill=\"none\" points=\"176,-249.5 176,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"204\" y=\"-280.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"176,-272.5 232,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"204\" y=\"-257.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"232,-249.5 232,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"291.5\" y=\"-280.3\">(None, 32, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"232,-272.5 351,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"291.5\" y=\"-257.3\">(None, 32, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099368106920&#45;&gt;2099355393944 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>2099368106920-&gt;2099355393944</title>\n",
       "<path d=\"M175.5,-332.366C175.5,-324.152 175.5,-314.658 175.5,-305.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-305.607 175.5,-295.607 172,-305.607 179,-305.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099360827768 -->\n",
       "<g class=\"node\" id=\"node7\"><title>2099360827768</title>\n",
       "<polygon fill=\"none\" points=\"23,-166.5 23,-212.5 328,-212.5 328,-166.5 23,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88\" y=\"-185.8\">dropout_7: Dropout</text>\n",
       "<polyline fill=\"none\" points=\"153,-166.5 153,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"181\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"153,-189.5 209,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"181\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"209,-166.5 209,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"268.5\" y=\"-197.3\">(None, 32, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"209,-189.5 328,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"268.5\" y=\"-174.3\">(None, 32, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099355393944&#45;&gt;2099360827768 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>2099355393944-&gt;2099360827768</title>\n",
       "<path d=\"M175.5,-249.366C175.5,-241.152 175.5,-231.658 175.5,-222.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-222.607 175.5,-212.607 172,-222.607 179,-222.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099360826536 -->\n",
       "<g class=\"node\" id=\"node8\"><title>2099360826536</title>\n",
       "<polygon fill=\"none\" points=\"33.5,-83.5 33.5,-129.5 317.5,-129.5 317.5,-83.5 33.5,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88\" y=\"-102.8\">flatten_3: Flatten</text>\n",
       "<polyline fill=\"none\" points=\"142.5,-83.5 142.5,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"170.5\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"142.5,-106.5 198.5,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"170.5\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"198.5,-83.5 198.5,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258\" y=\"-114.3\">(None, 32, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"198.5,-106.5 317.5,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258\" y=\"-91.3\">(None, 25088)</text>\n",
       "</g>\n",
       "<!-- 2099360827768&#45;&gt;2099360826536 -->\n",
       "<g class=\"edge\" id=\"edge7\"><title>2099360827768-&gt;2099360826536</title>\n",
       "<path d=\"M175.5,-166.366C175.5,-158.152 175.5,-148.658 175.5,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-139.607 175.5,-129.607 172,-139.607 179,-139.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099355434736 -->\n",
       "<g class=\"node\" id=\"node9\"><title>2099355434736</title>\n",
       "<polygon fill=\"none\" points=\"47,-0.5 47,-46.5 304,-46.5 304,-0.5 47,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"99\" y=\"-19.8\">dense_7: Dense</text>\n",
       "<polyline fill=\"none\" points=\"151,-0.5 151,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"151,-23.5 207,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"207,-0.5 207,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"255.5\" y=\"-31.3\">(None, 25088)</text>\n",
       "<polyline fill=\"none\" points=\"207,-23.5 304,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"255.5\" y=\"-8.3\">(None, 1)</text>\n",
       "</g>\n",
       "<!-- 2099360826536&#45;&gt;2099355434736 -->\n",
       "<g class=\"edge\" id=\"edge8\"><title>2099360826536-&gt;2099355434736</title>\n",
       "<path d=\"M175.5,-83.3664C175.5,-75.1516 175.5,-65.6579 175.5,-56.7252\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"179,-56.6068 175.5,-46.6068 172,-56.6069 179,-56.6068\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "display(SVG(model_to_dot(discriminator,show_layer_names=True, show_shapes=True).create(prog='dot', format='svg')))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generator Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"719pt\" viewBox=\"0.00 0.00 398.00 719.00\" width=\"398pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 715)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-715 394,-715 394,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2099357266664 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2099357266664</title>\n",
       "<polygon fill=\"none\" points=\"42.5,-664.5 42.5,-710.5 347.5,-710.5 347.5,-664.5 42.5,-664.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"125.5\" y=\"-683.8\">dense_6_input: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"208.5,-664.5 208.5,-710.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"236.5\" y=\"-695.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"208.5,-687.5 264.5,-687.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"236.5\" y=\"-672.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"264.5,-664.5 264.5,-710.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"306\" y=\"-695.3\">(None, 100)</text>\n",
       "<polyline fill=\"none\" points=\"264.5,-687.5 347.5,-687.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"306\" y=\"-672.3\">(None, 100)</text>\n",
       "</g>\n",
       "<!-- 2099369723776 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2099369723776</title>\n",
       "<polygon fill=\"none\" points=\"70,-581.5 70,-627.5 320,-627.5 320,-581.5 70,-581.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"122\" y=\"-600.8\">dense_6: Dense</text>\n",
       "<polyline fill=\"none\" points=\"174,-581.5 174,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"202\" y=\"-612.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"174,-604.5 230,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"202\" y=\"-589.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"230,-581.5 230,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"275\" y=\"-612.3\">(None, 100)</text>\n",
       "<polyline fill=\"none\" points=\"230,-604.5 320,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"275\" y=\"-589.3\">(None, 6272)</text>\n",
       "</g>\n",
       "<!-- 2099357266664&#45;&gt;2099369723776 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2099357266664-&gt;2099369723776</title>\n",
       "<path d=\"M195,-664.366C195,-656.152 195,-646.658 195,-637.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-637.607 195,-627.607 191.5,-637.607 198.5,-637.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099369723832 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2099369723832</title>\n",
       "<polygon fill=\"none\" points=\"34,-498.5 34,-544.5 356,-544.5 356,-498.5 34,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"122\" y=\"-517.8\">leaky_re_lu_12: LeakyReLU</text>\n",
       "<polyline fill=\"none\" points=\"210,-498.5 210,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"238\" y=\"-529.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"210,-521.5 266,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"238\" y=\"-506.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"266,-498.5 266,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"311\" y=\"-529.3\">(None, 6272)</text>\n",
       "<polyline fill=\"none\" points=\"266,-521.5 356,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"311\" y=\"-506.3\">(None, 6272)</text>\n",
       "</g>\n",
       "<!-- 2099369723776&#45;&gt;2099369723832 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2099369723776-&gt;2099369723832</title>\n",
       "<path d=\"M195,-581.366C195,-573.152 195,-563.658 195,-554.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-554.607 195,-544.607 191.5,-554.607 198.5,-554.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099357268456 -->\n",
       "<g class=\"node\" id=\"node4\"><title>2099357268456</title>\n",
       "<polygon fill=\"none\" points=\"47,-415.5 47,-461.5 343,-461.5 343,-415.5 47,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"111\" y=\"-434.8\">reshape_4: Reshape</text>\n",
       "<polyline fill=\"none\" points=\"175,-415.5 175,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"203\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"175,-438.5 231,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"203\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"231,-415.5 231,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"287\" y=\"-446.3\">(None, 6272)</text>\n",
       "<polyline fill=\"none\" points=\"231,-438.5 343,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"287\" y=\"-423.3\">(None, 128, 7, 7)</text>\n",
       "</g>\n",
       "<!-- 2099369723832&#45;&gt;2099357268456 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>2099369723832-&gt;2099357268456</title>\n",
       "<path d=\"M195,-498.366C195,-490.152 195,-480.658 195,-471.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-471.607 195,-461.607 191.5,-471.607 198.5,-471.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099357267504 -->\n",
       "<g class=\"node\" id=\"node5\"><title>2099357267504</title>\n",
       "<polygon fill=\"none\" points=\"0,-332.5 0,-378.5 390,-378.5 390,-332.5 0,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"104.5\" y=\"-351.8\">up_sampling2d_7: UpSampling2D</text>\n",
       "<polyline fill=\"none\" points=\"209,-332.5 209,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"237\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"209,-355.5 265,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"237\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"265,-332.5 265,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"327.5\" y=\"-363.3\">(None, 128, 7, 7)</text>\n",
       "<polyline fill=\"none\" points=\"265,-355.5 390,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"327.5\" y=\"-340.3\">(None, 128, 14, 14)</text>\n",
       "</g>\n",
       "<!-- 2099357268456&#45;&gt;2099357267504 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>2099357268456-&gt;2099357267504</title>\n",
       "<path d=\"M195,-415.366C195,-407.152 195,-397.658 195,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-388.607 195,-378.607 191.5,-388.607 198.5,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099357267560 -->\n",
       "<g class=\"node\" id=\"node6\"><title>2099357267560</title>\n",
       "<polygon fill=\"none\" points=\"37.5,-249.5 37.5,-295.5 352.5,-295.5 352.5,-249.5 37.5,-249.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"104.5\" y=\"-268.8\">conv2d_13: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.5,-249.5 171.5,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.5\" y=\"-280.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.5,-272.5 227.5,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.5\" y=\"-257.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.5,-249.5 227.5,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"290\" y=\"-280.3\">(None, 128, 14, 14)</text>\n",
       "<polyline fill=\"none\" points=\"227.5,-272.5 352.5,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"290\" y=\"-257.3\">(None, 16, 14, 14)</text>\n",
       "</g>\n",
       "<!-- 2099357267504&#45;&gt;2099357267560 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>2099357267504-&gt;2099357267560</title>\n",
       "<path d=\"M195,-332.366C195,-324.152 195,-314.658 195,-305.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-305.607 195,-295.607 191.5,-305.607 198.5,-305.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099358227704 -->\n",
       "<g class=\"node\" id=\"node7\"><title>2099358227704</title>\n",
       "<polygon fill=\"none\" points=\"19.5,-166.5 19.5,-212.5 370.5,-212.5 370.5,-166.5 19.5,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"107.5\" y=\"-185.8\">leaky_re_lu_13: LeakyReLU</text>\n",
       "<polyline fill=\"none\" points=\"195.5,-166.5 195.5,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"223.5\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"195.5,-189.5 251.5,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"223.5\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"251.5,-166.5 251.5,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"311\" y=\"-197.3\">(None, 16, 14, 14)</text>\n",
       "<polyline fill=\"none\" points=\"251.5,-189.5 370.5,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"311\" y=\"-174.3\">(None, 16, 14, 14)</text>\n",
       "</g>\n",
       "<!-- 2099357267560&#45;&gt;2099358227704 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>2099357267560-&gt;2099358227704</title>\n",
       "<path d=\"M195,-249.366C195,-241.152 195,-231.658 195,-222.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-222.607 195,-212.607 191.5,-222.607 198.5,-222.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099354467296 -->\n",
       "<g class=\"node\" id=\"node8\"><title>2099354467296</title>\n",
       "<polygon fill=\"none\" points=\"3,-83.5 3,-129.5 387,-129.5 387,-83.5 3,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"107.5\" y=\"-102.8\">up_sampling2d_8: UpSampling2D</text>\n",
       "<polyline fill=\"none\" points=\"212,-83.5 212,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"240\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"212,-106.5 268,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"240\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"268,-83.5 268,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"327.5\" y=\"-114.3\">(None, 16, 14, 14)</text>\n",
       "<polyline fill=\"none\" points=\"268,-106.5 387,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"327.5\" y=\"-91.3\">(None, 16, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099358227704&#45;&gt;2099354467296 -->\n",
       "<g class=\"edge\" id=\"edge7\"><title>2099358227704-&gt;2099354467296</title>\n",
       "<path d=\"M195,-166.366C195,-158.152 195,-148.658 195,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-139.607 195,-129.607 191.5,-139.607 198.5,-139.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2099369812488 -->\n",
       "<g class=\"node\" id=\"node9\"><title>2099369812488</title>\n",
       "<polygon fill=\"none\" points=\"40.5,-0.5 40.5,-46.5 349.5,-46.5 349.5,-0.5 40.5,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"107.5\" y=\"-19.8\">conv2d_14: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"174.5,-0.5 174.5,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"202.5\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"174.5,-23.5 230.5,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"202.5\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"230.5,-0.5 230.5,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"290\" y=\"-31.3\">(None, 16, 28, 28)</text>\n",
       "<polyline fill=\"none\" points=\"230.5,-23.5 349.5,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"290\" y=\"-8.3\">(None, 1, 28, 28)</text>\n",
       "</g>\n",
       "<!-- 2099354467296&#45;&gt;2099369812488 -->\n",
       "<g class=\"edge\" id=\"edge8\"><title>2099354467296-&gt;2099369812488</title>\n",
       "<path d=\"M195,-83.3664C195,-75.1516 195,-65.6579 195,-56.7252\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"198.5,-56.6068 195,-46.6068 191.5,-56.6069 198.5,-56.6068\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from keras.utils.vis_utils import plot_model\n",
    "# plot_model(generator, to_file='gen_model.png')\n",
    "display(SVG(model_to_dot(generator, show_layer_names=True, show_shapes=True).create(prog='dot', format='svg')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import pyplot as pl\n",
    "from keras.layers import Input\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers.core import Reshape, Dense, Dropout, Flatten\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.layers.convolutional import Convolution2D, UpSampling2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.datasets import mnist\n",
    "from keras.optimizers import Adam\n",
    "from keras import backend as K\n",
    "from keras import initializers\n",
    "\n",
    "K.set_image_dim_ordering('th')\n",
    "\n",
    "np.random.seed(1000)\n",
    "\n",
    "randomDim = 100\n",
    "\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = (X_train.astype(np.float32) - 127.5)/127.5\n",
    "X_train = X_train.reshape(60000, 784)\n",
    "\n",
    "adam = Adam(lr=0.0002, beta_1=0.5)\n",
    "\n",
    "generator = Sequential()\n",
    "generator.add(Dense(256, input_dim=randomDim, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(512))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(1024))\n",
    "generator.add(LeakyReLU(0.2))\n",
    "generator.add(Dense(784, activation='tanh'))\n",
    "generator.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "discriminator = Sequential()\n",
    "discriminator.add(Dense(1024, input_dim=784, kernel_initializer=initializers.RandomNormal(stddev=0.02)))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(512))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(256))\n",
    "discriminator.add(LeakyReLU(0.2))\n",
    "discriminator.add(Dropout(0.3))\n",
    "discriminator.add(Dense(1, activation='sigmoid'))\n",
    "discriminator.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "# Combined network\n",
    "discriminator.trainable = False\n",
    "ganInput = Input(shape=(randomDim,))\n",
    "x = generator(ganInput)\n",
    "ganOutput = discriminator(x)\n",
    "gan = Model(inputs=ganInput, outputs=ganOutput)\n",
    "gan.compile(loss='binary_crossentropy', optimizer=adam)\n",
    "\n",
    "dLosses = []\n",
    "gLosses = []\n",
    "\n",
    "def plotLoss(epoch):\n",
    "    plt.figure(figsize=(10, 8))\n",
    "    plt.plot(dLosses, label='Discriminitive loss')\n",
    "    plt.plot(gLosses, label='Generative loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.legend()\n",
    "    plt.savefig('images/gan_neuralnet/gan_%d_loss_epoch_.png' % epoch)\n",
    "\n",
    "def plotGeneratedImages(epoch, examples=100, dim=(10, 10), figsize=(10, 10)):\n",
    "    noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "    generatedImages = generator.predict(noise)\n",
    "    generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "\n",
    "    plt.figure(figsize=figsize)\n",
    "    for i in range(generatedImages.shape[0]):\n",
    "        plt.subplot(dim[0], dim[1], 1)\n",
    "        plt.imshow(generatedImages[i], interpolation='nearest', cmap='gray_r')\n",
    "        plt.axis('off')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('images/gan_neuralnet/gan_%d_generated_image_epoch.png' % epoch)\n",
    "\n",
    "def saveModels(epoch):\n",
    "    generator.save('models/gan_neuralnet/gan_generator_epoch_%d.h5' % epoch)\n",
    "    discriminator.save('models/gan_neuralnet/gan_discriminator_epoch_%d.h5' % epoch)\n",
    "\n",
    "def train(epochs=1, batchSize=128):\n",
    "    batchCount = int(X_train.shape[0] / batchSize)\n",
    "    print( 'Epochs:', epochs)\n",
    "    print( 'Batch size:', batchSize)\n",
    "    print( 'Batches per epoch:', batchCount)\n",
    "\n",
    "    for e in range(1, epochs+1):\n",
    "        print ('-'*15, 'Epoch %d' % e, '-'*15)\n",
    "        for _ in tqdm(range(batchCount)):\n",
    "            # Get a random set of input noise and images\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            imageBatch = X_train[np.random.randint(0, X_train.shape[0], size=batchSize)]\n",
    "\n",
    "            # Generate fake MNIST images\n",
    "            generatedImages = generator.predict(noise)\n",
    "            # print np.shape(imageBatch), np.shape(generatedImages)\n",
    "            X = np.concatenate([imageBatch, generatedImages])\n",
    "\n",
    "            # Labels for generated and real data\n",
    "            yDis = np.zeros(2*batchSize)\n",
    "            # One-sided label smoothing\n",
    "            yDis[:batchSize] = 0.9\n",
    "\n",
    "            # Train discriminator\n",
    "            discriminator.trainable = True\n",
    "            dloss = discriminator.train_on_batch(X, yDis)\n",
    "\n",
    "            # Train generator\n",
    "            noise = np.random.normal(0, 1, size=[batchSize, randomDim])\n",
    "            yGen = np.ones(batchSize)\n",
    "            discriminator.trainable = False\n",
    "            gloss = gan.train_on_batch(noise, yGen)\n",
    "\n",
    "        dLosses.append(dloss)\n",
    "        gLosses.append(gloss)\n",
    "\n",
    "        if e == 1 or e % 20 == 0:\n",
    "            plotGeneratedImages(e)\n",
    "            saveModels(e)\n",
    "\n",
    "    plotLoss(e)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    print(\"\")\n",
    "    #     train(200, 128)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(SVG(model_to_dot(discriminator,show_layer_names=True, show_shapes=True).create(prog='dot', format='svg')))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(SVG(model_to_dot(generator, show_layer_names=True, show_shapes=True).create(prog='dot', format='svg')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "randomDim = 100\n",
    "examples=1\n",
    "noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "\n",
    "generatedImages = generator.predict(noise)\n",
    "generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "# generatedImages.shape\n",
    "pl.subplot(2, 2, 1)\n",
    "pl.figure(figsize=(2,2))\n",
    "pl.imshow(generatedImages[0], cmap = 'gray', interpolation = 'bicubic')\n",
    "pl.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis    \n",
    "\n",
    "counter=counter-1\n",
    "pl.tight_layout()\n",
    "pl.savefig('images/gan_neuralnet/digits/epoch_200/random_%d.png' % counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "discriminator.predict(generatedImages.reshape(1,784))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "counter = 995\n",
    "generator = keras.models.load_model(\"models/gan_cnn/gan_generator_epoch_50.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "randomDim = 100\n",
    "examples=1\n",
    "noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "\n",
    "generatedImages = generator.predict(noise)\n",
    "generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "# generatedImages.shape\n",
    "pl.subplot(2, 2, 1)\n",
    "pl.figure(figsize=(2,2))\n",
    "pl.imshow(generatedImages[0], cmap = 'gray', interpolation = 'bicubic')\n",
    "pl.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis    \n",
    "\n",
    "counter=counter-1\n",
    "pl.tight_layout()\n",
    "pl.savefig('images/gan_cnn/digits/epoch_50/random_%d.png' % counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import os, sys\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "os.chdir('/images/gan_cnn/epoch_50')\n",
    "list_im =  ['0.png', '3.png', '6.png','7.png','8.png']\n",
    "imgs    = [ Image.open(i) for i in list_im ]\n",
    "# pick the image which is the smallest, and resize the others to match it (can be arbitrary image shape here)\n",
    "min_shape = sorted( [(np.sum(i.size), i.size ) for i in imgs])[0][1]\n",
    "imgs_comb = np.hstack( (np.asarray( i.resize(min_shape) ) for i in imgs ) )\n",
    "\n",
    "# save that beautiful picture\n",
    "imgs_comb = Image.fromarray( imgs_comb)\n",
    "imgs_comb.save( '/images/gan_cnn/digits/epoch_50.jpg' )    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(28, 28, 1)\n",
      "(?, 10, 16)\n",
      "(?, 10)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def margin_loss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    Margin loss for Eq.(4). When y_true[i, :] contains not just one `1`, this loss should work too. Not test it.\n",
    "    :param y_true: [None, n_classes]\n",
    "    :param y_pred: [None, num_capsule]\n",
    "    :return: a scalar loss value.\n",
    "    \"\"\"\n",
    "    L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \\\n",
    "        0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))\n",
    "\n",
    "    return K.mean(K.sum(L, 1))\n",
    "from IPython.display import SVG, display\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from keras import layers, models, optimizers\n",
    "from keras import backend as K\n",
    "K.set_image_data_format('channels_last')\n",
    "import tensorflow as tf\n",
    "from keras.utils import to_categorical\n",
    "from capsule.capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import multi_gpu_model\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "x_train = x_train.reshape(-1, 28, 28, 1).astype('float32') / 255.\n",
    "x_test = x_test.reshape(-1, 28, 28, 1).astype('float32') / 255.\n",
    "y_train = to_categorical(y_train.astype('float32'))\n",
    "y_test = to_categorical(y_test.astype('float32'))\n",
    "    \n",
    "input_shape=x_train.shape[1:]\n",
    "n_class=len(np.unique(np.argmax(y_train, 1)))\n",
    "num_routing=3\n",
    "lam_recon=0.392\n",
    "lr=0.001\n",
    "print(input_shape)\n",
    "\n",
    "with tf.device('/gpu:0'):\n",
    "    x = layers.Input(shape=input_shape)\n",
    "    conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x)\n",
    "    primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid')\n",
    "    digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, num_routing=num_routing,\n",
    "                             name='digitcaps')(primarycaps)\n",
    "    print(digitcaps.shape)\n",
    "    # # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.\n",
    "    # # If using tensorflow, this will not be necessary. :)\n",
    "    out_caps = Length(name='capsnet')(digitcaps)\n",
    "    print(out_caps.shape)\n",
    "    # Decoder network.\n",
    "    y = layers.Input(shape=(n_class,))\n",
    "    masked_by_y = Mask()([digitcaps, y])  # The true label is used to mask the output of capsule layer. For training\n",
    "    masked = Mask()(digitcaps)  # Mask using the capsule with maximal length. For prediction\n",
    "\n",
    "    # Shared Decoder model in training and prediction\n",
    "    decoder = models.Sequential(name='decoder')\n",
    "    decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class))\n",
    "    decoder.add(layers.Dense(1024, activation='relu'))\n",
    "    decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))\n",
    "    decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))\n",
    "\n",
    "    # Models for training and evaluation (prediction)\n",
    "    train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])\n",
    "    eval_model = models.Model(x, [out_caps, decoder(masked)])\n",
    "\n",
    "    \n",
    "train_model.compile(optimizer=optimizers.Adam(lr=lr),\n",
    "                  loss=[margin_loss, 'mse'],\n",
    "                  loss_weights=[1., lam_recon],\n",
    "                  metrics={'capsnet': 'accuracy'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"636pt\" viewBox=\"0.00 0.00 661.00 636.00\" width=\"661pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 632)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-632 657,-632 657,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2099357294664 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2099357294664</title>\n",
       "<polygon fill=\"none\" points=\"324,-581.5 324,-627.5 618,-627.5 618,-581.5 324,-581.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"387\" y=\"-600.8\">input_4: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"450,-581.5 450,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"478\" y=\"-612.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"450,-604.5 506,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"478\" y=\"-589.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"506,-581.5 506,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"562\" y=\"-612.3\">(None, 28, 28, 1)</text>\n",
       "<polyline fill=\"none\" points=\"506,-604.5 618,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"562\" y=\"-589.3\">(None, 28, 28, 1)</text>\n",
       "</g>\n",
       "<!-- 2103679241520 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2103679241520</title>\n",
       "<polygon fill=\"none\" points=\"328,-498.5 328,-544.5 614,-544.5 614,-498.5 328,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"380.5\" y=\"-517.8\">conv1: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"433,-498.5 433,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"461\" y=\"-529.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"433,-521.5 489,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"461\" y=\"-506.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"489,-498.5 489,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"551.5\" y=\"-529.3\">(None, 28, 28, 1)</text>\n",
       "<polyline fill=\"none\" points=\"489,-521.5 614,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"551.5\" y=\"-506.3\">(None, 20, 20, 256)</text>\n",
       "</g>\n",
       "<!-- 2099357294664&#45;&gt;2103679241520 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2099357294664-&gt;2103679241520</title>\n",
       "<path d=\"M471,-581.366C471,-573.152 471,-563.658 471,-554.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"474.5,-554.607 471,-544.607 467.5,-554.607 474.5,-554.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103679135024 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2103679135024</title>\n",
       "<polygon fill=\"none\" points=\"289,-415.5 289,-461.5 653,-461.5 653,-415.5 289,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"380.5\" y=\"-434.8\">primarycap_conv2d: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"472,-415.5 472,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"500\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"472,-438.5 528,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"500\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"528,-415.5 528,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"590.5\" y=\"-446.3\">(None, 20, 20, 256)</text>\n",
       "<polyline fill=\"none\" points=\"528,-438.5 653,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"590.5\" y=\"-423.3\">(None, 6, 6, 256)</text>\n",
       "</g>\n",
       "<!-- 2103679241520&#45;&gt;2103679135024 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2103679241520-&gt;2103679135024</title>\n",
       "<path d=\"M471,-498.366C471,-490.152 471,-480.658 471,-471.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"474.5,-471.607 471,-461.607 467.5,-471.607 474.5,-471.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103678708536 -->\n",
       "<g class=\"node\" id=\"node4\"><title>2103678708536</title>\n",
       "<polygon fill=\"none\" points=\"294.5,-332.5 294.5,-378.5 647.5,-378.5 647.5,-332.5 294.5,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"387\" y=\"-351.8\">primarycap_reshape: Reshape</text>\n",
       "<polyline fill=\"none\" points=\"479.5,-332.5 479.5,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"507.5\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"479.5,-355.5 535.5,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"507.5\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"535.5,-332.5 535.5,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"591.5\" y=\"-363.3\">(None, 6, 6, 256)</text>\n",
       "<polyline fill=\"none\" points=\"535.5,-355.5 647.5,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"591.5\" y=\"-340.3\">(None, 1152, 8)</text>\n",
       "</g>\n",
       "<!-- 2103679135024&#45;&gt;2103678708536 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>2103679135024-&gt;2103678708536</title>\n",
       "<path d=\"M471,-415.366C471,-407.152 471,-397.658 471,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"474.5,-388.607 471,-378.607 467.5,-388.607 474.5,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103642207120 -->\n",
       "<g class=\"node\" id=\"node5\"><title>2103642207120</title>\n",
       "<polygon fill=\"none\" points=\"301.5,-249.5 301.5,-295.5 640.5,-295.5 640.5,-249.5 301.5,-249.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"391\" y=\"-268.8\">primarycap_squash: Lambda</text>\n",
       "<polyline fill=\"none\" points=\"480.5,-249.5 480.5,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"508.5\" y=\"-280.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"480.5,-272.5 536.5,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"508.5\" y=\"-257.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"536.5,-249.5 536.5,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"588.5\" y=\"-280.3\">(None, 1152, 8)</text>\n",
       "<polyline fill=\"none\" points=\"536.5,-272.5 640.5,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"588.5\" y=\"-257.3\">(None, 1152, 8)</text>\n",
       "</g>\n",
       "<!-- 2103678708536&#45;&gt;2103642207120 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>2103678708536-&gt;2103642207120</title>\n",
       "<path d=\"M471,-332.366C471,-324.152 471,-314.658 471,-305.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"474.5,-305.607 471,-295.607 467.5,-305.607 474.5,-305.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103642208688 -->\n",
       "<g class=\"node\" id=\"node6\"><title>2103642208688</title>\n",
       "<polygon fill=\"none\" points=\"316,-166.5 316,-212.5 626,-212.5 626,-166.5 316,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"391\" y=\"-185.8\">digitcaps: CapsuleLayer</text>\n",
       "<polyline fill=\"none\" points=\"466,-166.5 466,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"494\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"466,-189.5 522,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"494\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"522,-166.5 522,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"574\" y=\"-197.3\">(None, 1152, 8)</text>\n",
       "<polyline fill=\"none\" points=\"522,-189.5 626,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"574\" y=\"-174.3\">(None, 10, 16)</text>\n",
       "</g>\n",
       "<!-- 2103642207120&#45;&gt;2103642208688 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>2103642207120-&gt;2103642208688</title>\n",
       "<path d=\"M471,-249.366C471,-241.152 471,-231.658 471,-222.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"474.5,-222.607 471,-212.607 467.5,-222.607 474.5,-222.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103642207176 -->\n",
       "<g class=\"node\" id=\"node8\"><title>2103642207176</title>\n",
       "<polygon fill=\"none\" points=\"0,-83.5 0,-129.5 328,-129.5 328,-83.5 0,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"49\" y=\"-102.8\">mask_1: Mask</text>\n",
       "<polyline fill=\"none\" points=\"98,-83.5 98,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"126\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"98,-106.5 154,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"126\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"154,-83.5 154,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"241\" y=\"-114.3\">[(None, 10, 16), (None, 10)]</text>\n",
       "<polyline fill=\"none\" points=\"154,-106.5 328,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"241\" y=\"-91.3\">(None, 160)</text>\n",
       "</g>\n",
       "<!-- 2103642208688&#45;&gt;2103642207176 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>2103642208688-&gt;2103642207176</title>\n",
       "<path d=\"M387.516,-166.473C347.244,-155.848 298.788,-143.063 257.302,-132.117\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"258.029,-128.689 247.467,-129.522 256.243,-135.458 258.029,-128.689\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103687007144 -->\n",
       "<g class=\"node\" id=\"node9\"><title>2103687007144</title>\n",
       "<polygon fill=\"none\" points=\"346,-83.5 346,-129.5 604,-129.5 604,-83.5 346,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"398\" y=\"-102.8\">capsnet: Length</text>\n",
       "<polyline fill=\"none\" points=\"450,-83.5 450,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"478\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"450,-106.5 506,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"478\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"506,-83.5 506,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"555\" y=\"-114.3\">(None, 10, 16)</text>\n",
       "<polyline fill=\"none\" points=\"506,-106.5 604,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"555\" y=\"-91.3\">(None, 10)</text>\n",
       "</g>\n",
       "<!-- 2103642208688&#45;&gt;2103687007144 -->\n",
       "<g class=\"edge\" id=\"edge8\"><title>2103642208688-&gt;2103687007144</title>\n",
       "<path d=\"M472.093,-166.366C472.499,-158.152 472.968,-148.658 473.409,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"476.911,-139.767 473.908,-129.607 469.919,-139.422 476.911,-139.767\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103641993168 -->\n",
       "<g class=\"node\" id=\"node7\"><title>2103641993168</title>\n",
       "<polygon fill=\"none\" points=\"34.5,-166.5 34.5,-212.5 293.5,-212.5 293.5,-166.5 34.5,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"97.5\" y=\"-185.8\">input_5: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"160.5,-166.5 160.5,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"188.5\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"160.5,-189.5 216.5,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"188.5\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"216.5,-166.5 216.5,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"255\" y=\"-197.3\">(None, 10)</text>\n",
       "<polyline fill=\"none\" points=\"216.5,-189.5 293.5,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"255\" y=\"-174.3\">(None, 10)</text>\n",
       "</g>\n",
       "<!-- 2103641993168&#45;&gt;2103642207176 -->\n",
       "<g class=\"edge\" id=\"edge7\"><title>2103641993168-&gt;2103642207176</title>\n",
       "<path d=\"M164,-166.366C164,-158.152 164,-148.658 164,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"167.5,-139.607 164,-129.607 160.5,-139.607 167.5,-139.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2103686168248 -->\n",
       "<g class=\"node\" id=\"node10\"><title>2103686168248</title>\n",
       "<polygon fill=\"none\" points=\"17,-0.5 17,-46.5 311,-46.5 311,-0.5 17,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"80\" y=\"-19.8\">decoder: Sequential</text>\n",
       "<polyline fill=\"none\" points=\"143,-0.5 143,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"171\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"143,-23.5 199,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"171\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"199,-0.5 199,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"255\" y=\"-31.3\">(None, 160)</text>\n",
       "<polyline fill=\"none\" points=\"199,-23.5 311,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"255\" y=\"-8.3\">(None, 28, 28, 1)</text>\n",
       "</g>\n",
       "<!-- 2103642207176&#45;&gt;2103686168248 -->\n",
       "<g class=\"edge\" id=\"edge9\"><title>2103642207176-&gt;2103686168248</title>\n",
       "<path d=\"M164,-83.3664C164,-75.1516 164,-65.6579 164,-56.7252\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"167.5,-56.6068 164,-46.6068 160.5,-56.6069 167.5,-56.6068\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import SVG, display\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "display(SVG(model_to_dot(train_model,show_layer_names=True, show_shapes=True).create(prog='dot', format='svg')))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import keras\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from keras import callbacks\n",
    "dim=(1, 1)\n",
    "figsize=(2, 2)\n",
    "class SaveImage(keras.callbacks.Callback):\n",
    "    def __init__(self):\n",
    "        self.epoch_num=5\n",
    "        self.letter = np.zeros((10,10))\n",
    "        for i in range(10):\n",
    "            self.letter[i][i] = 1\n",
    "    def on_epoch_end(self, batch,epoch, logs={}):\n",
    "#         print(type(str(epoch['batch'])))\n",
    "        \n",
    "        plt.figure(figsize=figsize)\n",
    "        for i in range(50):            \n",
    "            out = train_model.predict([x_train[i].reshape(-1,28,28,1),self.letter[0].reshape(-1,10)])\n",
    "            out_num = np.argmax(out[0])\n",
    "#             print(out_num)\n",
    "#             print(self.letter[out_num].reshape(-1,10))\n",
    "            out = train_model.predict([x_train[i].reshape(-1,28,28,1),self.letter[out_num].reshape(-1,10)])\n",
    "            \n",
    "            plt.subplot(dim[0], dim[1], 1)\n",
    "#             print(\"------\"+str(out[1][0].shape))\n",
    "            plt.imshow(out[1][0].reshape(28,28), interpolation='nearest', cmap='gray_r')\n",
    "            plt.xticks([]), plt.yticks([]) \n",
    "            save_dir_epoch = 'images/reconstruct_capsulenet/epoch_'+str(self.epoch_num)+''\n",
    "#             if not os.path.exists(save_dir_epoch):\n",
    "#                 os.makedirs(save_dir_epoch)\n",
    "            plt.savefig(save_dir_epoch+'/caps_'+str(i)+'_image.png')\n",
    "        self.epoch_num = self.epoch_num+1\n",
    "\n",
    "\n",
    "batch_size = 100\n",
    "epochs = 20\n",
    "save_dir = './train_results'\n",
    "debug = 0\n",
    "\n",
    "saveimg = SaveImage()\n",
    "if not os.path.exists(save_dir):\n",
    "        os.makedirs(save_dir)\n",
    "log = callbacks.CSVLogger(save_dir + '/log.csv')\n",
    "tb = callbacks.TensorBoard(log_dir=save_dir + '/tensorboard-logs',\n",
    "                               batch_size=batch_size, histogram_freq=debug)\n",
    "checkpoint = callbacks.ModelCheckpoint(save_dir + '/weights-{epoch:02d}.h5', monitor='val_capsnet_acc',\n",
    "                                       save_best_only=True, save_weights_only=True, verbose=1)\n",
    "lr_decay = callbacks.LearningRateScheduler(schedule=lambda epoch: lr * (0.9 ** epoch))\n",
    "\n",
    "model_hist = train_model.fit([x_train, y_train], [y_train, x_train], batch_size=batch_size, epochs=epochs,\n",
    "              validation_data=[[x_test, y_test], [y_test, x_test]],verbose=1,initial_epoch = 5,  callbacks=[log, saveimg, checkpoint, lr_decay])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1e9c5805d30>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAJQCAYAAABM/CoCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl0VFW+9vHvrqqMBEKQIYySIDMhYQiDCDTYMtgIQhvS\nikOw1cttBBRb4H3BvtGG9zaCtkNzTTuAYnObyQFsVFobMCiKBjrILIRBwhimkBBCUlXn/aOSEOYk\nVKgMz2etWlXnnH32+VUVa9XDPjvnGMuyEBEREZEbZ/N1ASIiIiJVhYKViIiIiJcoWImIiIh4iYKV\niIiIiJcoWImIiIh4iYKViIiIiJcoWImIiIh4iYKViIiIiJcoWImIiIh4icNXB65bt67VvHlzXx1e\nREREpMQ2bNhw3LKsetdr57Ng1bx5c1JSUnx1eBEREZESM8bsL0k7nQoUERER8RIFKxEREREvUbAS\nERER8RIFKxEREREvUbASERER8RIFKxEREREvUbASERER8RIFKxEREREvUbASERER8RIFKxEREREv\nUbASERER8RIFKxEREREvUbASERER8RIFKxEREREvUbASERER8RIFKxEREREvUbASERER8ZLrBitj\nzFxjzDFjzJarbDfGmNeMMbuNMT8aYzp7v0wRERGRiq8kI1bvAoOusX0w0LLg8QTwxo2XJSIiIlL5\nXDdYWZaVDJy8RpNhwHzL4zugtjGmobcKFBEREaksvDHHqjFwoNhyesG6yxhjnjDGpBhjUjIyMrxw\naBEREZGK46ZOXrcs603LsrpaltW1Xr16N/PQIiIiIuXO4YU+DgJNiy03KVgnIiIiFYhlWTjdFk6X\nhdPtxuX2LBc+O11unG4Ly7JwucHltnBbhQ/PsmebZ7lwW1E7N7isC/tf2LfYstvCVdCf231hf8ui\nYL1VsP7i47vcXOPYFOu32P7WJfUWHq+Ux3ZbJf+MvRGslgNPGmMWAt2BTMuyDnuhXxERkXLjcrnJ\nd7pxOi3yXS7y893kOz3hIt/p8qx3unG5LPKdLlwuz7aLly1cBW2cLjdul4XLZeEqaOd2u3G6LNyF\nj4IQU/ja7XbjdltYLoqWLTdYbs92z2s3luVZZ7mBgrBQ+JqCwIFFwcPCFDwXrjMWGMsCDDbAZoHB\nc9qq6NkCGwbjm6/jugxgDNivsP7SF5e+B3PR9kua4+n4ivsUW/hXCeu8brAyxvwd+AVQ1xiTDvwX\n4AdgWVYS8ClwN7AbyAFGl/DYIiJSybjdFu6CAHHhcWHZdY1tl+97cVuXy40z301evpu8fBf5+QVh\nJ78gAOW7cTrdnnZOz/6FocYq6L/wGbeF5faEDiwL474QLgpDhd0HEcJQ8hENC3AXlGgZz6PwNcYU\nPHteex6e18ZO0bKxGYwBbKbotSl8bTPYCtbZbAZjN9iMZx0GTEHY8HRlih8CY0zR+zGm+LbL96Fw\nW2Efxbd5SvNsLb6t2Nu6LBFVcNf9fi3Luv862y1grNcqEpEqoehH1HnxD67LWbDO7cZVsM3tdF/e\nptgPr8tZ/Ef4wn5FP+LF9rfcvn7nFVDBKRq3y13wvVwpHF0h9Lgt3M5in7O7YATkJnBj4caTi1yG\nYq8vrL+wzcIyBmye0OAJEWAcNmx2U/Sw2z3LdocNh8OAw4bdbisKGDa759l+ybPNbsNRuL/Ns7/d\nZrA7DA67DYfdht1+YdnPbis6hsPuefZz2PFzeNp51tkKwk1BqCkMOsYTcIqHF6kgHipZM2+cChSR\ncmZZnh84p9ONK7/gf+kFz1cOGMVDTcEpiSsGlsL9Lu7DVYJQU9TvZTV4tlnl/ANsc3h+8OyFP5oF\nP6Km8L/OlUTh52QVLFhFr8EqSDEXt/Gsv/C62DPWxW3x/NspfO02BQ/AReGzhQsLpwVOyyLf8rzO\nt9zkWxZ5bssTZhwF+1wl5Fy8zcJl8AQLh90TNvw8YcLPz4a/nx1/fxsB/nbPaz87gQF2AvxtBPo7\nCAywE+RvJ8DPTpCfncCiZxuBBcuFrwu3222V6EuXKk3BSuQ6LLcngDgvCTRF6/IvrLu0jTP/kiBU\nuP7S5aLXhcdyXVguaFMeTOH/yu0GW0EwKfxf/YXlggDjMPgF2AmoceF/7Vfaz14QeGyXBB5bsdGC\nwvbGDsZmw7JZnmcDlo2i0xzuglMYhYHAAtw2UzD51DOp1OnyzFlxFUzKdbmKT8Z1F23PL5yoW3y5\nWNtLl4vv6yzjcmF/hZOCL2zztPPeF3mV15ewGYqCyJXCSfHgEnpZkPEsX6lt0BW2BzhsGnGRaknB\nSio0y7KuHF6KhZVLR3Eu2n5p2CnY7naWfD+388Z/AG02g63gf+x2P09IsfvZcfh5ThnY/Qx+Af4F\ny8bTxs+Ow2EDe0GYMODEc+ojHwunZXnChikMHpYnjOAJIS7AbcPz1y9cOGXixLPNhRtnwV/GFP+r\noOLLrmIhw/PaffE2p4Xz3OUBx1UsWFxYLt7fheNUBHabwW4z+BU8O+y2C8t2g8Nmw1G0zWC32Yra\nBvjZCC627LhCe4fN05+joD+/YssO++Vtr7bvRcv2gnVXWrYXey8Fy/52G36FI3oiUm4UrMRr3C43\n+Xlu8nNd5J93kn/eVbJHbuHrK+9zo3M6bPaCoOKwFQsyxZb9bPgHOq7c5jr7Fa5z+NmwbJBXcCrl\nvGVx3mVxzu3ivNsix+ki57yLnDwnZ/M8zzl5BevyXeScd3I27zzn8lyczfEse9a7yHOVz2hV0Y93\nwfwSz/KFH/XLt12y3lwIFXbDFfe9bJ9ix7HbrryPzRSGl0uXbdjN5X1fqUaHzVbUh19BsHDYLg4e\njmL7KGyIiLcoWFVDluU5vVSywFPygOTKL3kAsNkMfoF2/AKKPQLthIQFXryu4FEYXorCzFXCzWXh\np2CCaHFOl5uzeS5PiMlzFgWenILl03mFQcfFuTynp03WhQCUk3chIBXvozQByG4zBPvbqeHvINjf\nTnCAnWB/B2E1/GkSVrDO305wgIMa/naC/D3PwQEOgv087YP87J5JstcMMwVhxF4spBQEERER8T4F\nqwrO7bZwXhZinOTlXh5sLm93aVC6EJJKM7HYUTzk+Hue/YMc1KgdcMUQVBiS/AIcV91ud5Tsov9u\nt8Xxs+fJynVyLs9FZvFgc85JzukLIztn85wFQacgEBULTIWhKSfPRV4p5isVBqCiEBRgJ9jvQgAK\n8rdfHnyKt78oHF3ow9+u+SciIlWRgpWPnT/n5OjeTI7sOcPRvZlknzqPM+/CiJGzFKNAxmbwv3QU\nKMBOjVB//ALsFwekgEuCzxX2KwxSl474eItlWZw8m8fhzFwOnT7nec48x+HTuRwpeH30TC75ruun\nQJvhQvApCDQ1/B3UDvancdiFdcH+xUJOwMXrLg1FQf6agCsiIqWjYHUTWW6LU0dzOLInk6N7Mjmy\n9wwnD5/1zCEyUKdhDULrBeEfeHngcfhfOhp04eFfEJBsjoozV8SyLDLP5XPodC6HMz2h6XBBaDpU\ntJx72eiRn90QHhpIw9Agut4aRsPaQTQMDSQ0yK9YOPKEoiC/C+FIAUhERCoCBatydP6ck2N7z3Bk\nb6YnTO09w/kcJwABwQ4aRNTiti71CY8IpX5ELQKCKs/XkZWbf9FI0+HTF8JS4ajTuXzXRfvYbYbw\nWoE0DA2kY5PaDGzved0wNIhGtT3Pt9Tw1/wfERGptCrPL3kFV5LRqBad6tEgMpTwyFDCGgSX2ym2\nG5WT5+RQsdNxhwtGnQ4VBKgjmblknXdetI8xUL9mAA1Dg2gTXpN+revTMDSQRrWDCA8NpFFoEPVq\nBugifiIiUqUpWJVR3jknRyvhaFRuvuuywFQ00lQw6pR5Lv+y/eqGBNCodiCR9WrQ67a6npGm2kE0\nKniuXzMAP3vJJqSLiIhUVRXj176Cs9wWp495RqOO7DnDkT2ZFXI0Ks/p5uiZ3KL5TIXzmwqfj2Tm\ncuJs3mX7hQX70TA0iCZhQcQ2r0PD2p4RpsKRpgahAQQ4Lr2fuIiIiFxKweoKLh6N8vy13pVGoxpE\n1KJBROhNGY1yutwcyzp/UVDyzG26cJruePb5yy6jUCvQQaOCCeAdm9QuGmEqfG4YGkign0KTiIiI\nN1T7YFWRRqNOZJ/n+70ni+YyFU4EP5KZy9EzuVx6948a/vaicNQmvNbFI00Fk8FrBFT7r1hEROSm\nqXa/unnnnBzdd6YoSF02GtW8Fi061yc88uaNRoFn7tOwOd+QfuocAIF+NhqFBtGwdiC3t6hbFJQK\nw1PD2oHUDHDoEgMiIiIVSJUOVpZlcfpozkUh6sShijc3CuBv3+0n/dQ5Xv1NDH1a1qN2sJ9Ck4iI\nSCVTpYLVtUaj/IMchEfUIrLTzR+Nup4zufnMWb2b3i3rMiymsa/LERERkTKqGMmiDC6MRnkmmR/d\nc/loVGSneoRHhhIeEUpYeMW9btTbyXs4lZPPpIFtfF2KiIiI3IBKE6xKNRrVvBYBwX4+rrhkMrLO\n8/bXe/lVx4ZENQn1dTkiIiJyAypksLrmaBQQVolGo67n9VW7yHO6+f2A1r4uRURERG5QhQhWebme\n0aijhZc82JvJ+bOVfzTqevafOMv/rv+Z+NimRNSt4etyRERE5Ab5LFjlns1nzYIdHNlzhpOHsosu\nbBnWsAaRMVVjNOp6Xv7iJxx2w/g7W/q6FBEREfECnwWrM8dz2ZVyjAYRtYiMaU54ZCgNIqrOaNT1\nbD2UybLUQ/zuFy1oUCvQ1+WIiIiIF/gsWNVpVIPHXupdZUejrufFz3cSGuTHf/Rt4etSRERExEts\nvjqww89WbUPVt2kn+OqnDMb2a0FoUPUYoRMREakOfBasqivLspj5+Q4ahgbycM/mvi5HREREvEjB\n6iZbufUoqQdO89QvWxLoZ/d1OSIiIuJFClY3kdPlZtbKHbSoV4Nfd27i63JERETEyxSsbqIPNqaT\nlnGWZwe2wWHXRy8iIlLV6Nf9JsnNd/HKl7uIaVqbge0b+LocERERKQcKVjfJ/G/3cTgzl8mD2mBM\n9fxrSBERkapOweomyDyXz5zVafRtVY+eLW7xdTkiIiJSThSsboK/fpVG5rl8Jg3SjZZFRESqMgWr\ncnbsTC5zv9nL0OhGtG8U6utyREREpBwpWJWzV/+1C6fL4pkBrXxdioiIiJQzBatytPf4WRb+cIAH\nujfj1ltq+LocERERKWcKVuVo9j93EuCwMa5/S1+XIiIiIjeBglU52ZyeyYofD/PbOyKoVzPA1+WI\niIjITaBgVU5eXLmDsGA/nugT6etSRERE5CZRsCoH3+w+ztpdxxnb7zZqBvr5uhwRERG5SRSsvMyy\nLGZ+voPGtYN4sMetvi5HREREbiIFKy/7bMsRfkzP5KlftiTQz+7rckREROQmUrDyIqfLzeyVO2nV\nIIQRnZv4uhwRERG5yRSsvGhxSjp7jp/l2YFtsNt0o2UREZHqRsHKS87luXjly5/ocmsYv2xb39fl\niIiIiA8oWHnJu+v2cSzrPJMHtcEYjVaJiIhURwpWXpCZk88ba3bTv019ukXU8XU5IiIi4iMKVl7w\nP1/tJuu8k0mDWvu6FBEREfEhBasbdDjzHO9+s4/hMY1pE17L1+WIiIiIDylY3aDX/rULt2Xx9F2t\nfF2KiIiI+JiC1Q1Iy8hmcUo6o7rfStM6wb4uR0RERHxMweoGzF65k0CHjSf73+brUkRERKQCULAq\no9QDp/lsyxEe7xNJ3ZAAX5cjIiIiFYCCVRlYlsXMz3ZwSw1/Husd6etyREREpIJQsCqDtbuO8+2e\nEzzZ/zZCAhy+LkdEREQqCAWrUnK7LWZ+voMmYUE80L2Zr8sRERGRCkTBqpT+sfkwWw+d4ZkBrQhw\n2H1djoiIiFQgClalkO9y89I/d9ImvCZDoxv7uhwRERGpYBSsSmHhDwfYfyKHSYNaY7fpRssiIiJy\nMQWrEsrJc/Lav3bRrXkd+rWu7+tyREREpAJSsCqhuV/vJSPrPJMHt8YYjVaJiIjI5RSsSuDU2Tz+\n+tUeftm2AV1urePrckRERKSCUrAqgf9Zs5uzeU4mDWrt61JERESkAlOwuo6Dp8/x3rf7GdG5Ca0a\n1PR1OSIiIlKBKVhdxytf/AQWPH1XK1+XIiIiIhWcgtU17DqaxQcb03mo5600rh3k63JERESkglOw\nuoZZK3dSw9/B2H63+boUERERqQQUrK5iw/5T/HPbUZ7oE0mdGv6+LkdEREQqAQWrK7Asz42W64YE\n8OgdEb4uR0RERCoJBasrWPNTBt/vPcn4O2+jRoDD1+WIiIhIJaFgdQm32+LFz3fSrE4wv4lt5uty\nREREpBJRsLrE8k2H2H74DM8MaIW/Qx+PiIiIlJySQzF5TjcvfbGTdg1rcU/HRr4uR0RERCoZBati\n/v79zxw4eY5Jg1pjs+lGyyIiIlI6ClYFzp538vqqXfSIrEPfVvV8XY6IiIhUQgpWBd5eu5fj2XlM\nHtQGYzRaJSIiIqWnYAWcyD7Pm8lpDGofTqdmYb4uR0RERCopBStgzuo0zuW7+P1A3WhZREREyq7a\nB6v0Uzn87bv9xHVpym31a/q6HBEREanEqn2wevmLnzAGnrqrpa9LERERkUquWgerHUfO8NG/D5Jw\ne3Mahgb5uhwRERGp5Kp1sJq9cichAQ7+8xctfF2KiIiIVAHVNlj9sO8kX24/xpi+Lagd7O/rckRE\nRKQKqJbByrIsZn62g/o1A3i0V4SvyxEREZEqoloGq39tP0bK/lNM+GVLgvztvi5HREREqohqF6xc\nbotZK3cSUbcGI7s29XU5IiIiUoVUu2D18b8PsvNoFs8MaIWfvdq9fRERESlH1SpZnHe6ePmLn4hq\nHMrdHRr6uhwRERGpYqpVsPrbdz9z8PQ5Jg9qg82mGy2LiIiId1WbYJWVm8+c1bvpddst3NGyrq/L\nERERkSqo2gSrt9bu5eTZPCYPauPrUkRERKSKKlGwMsYMMsbsNMbsNsZMucL2UGPMJ8aYTcaYrcaY\n0d4vtewyss7z9to9/CqqIR2b1PZ1OSIiIlJFXTdYGWPswBxgMNAOuN8Y0+6SZmOBbZZlRQO/AF4y\nxlSYy5n/ZdUuzjvdPDOgla9LERERkSqsJCNW3YDdlmXtsSwrD1gIDLukjQXUNMYYIAQ4CTi9WmkZ\n/Xwih//9/mdGdm1KZL0QX5cjIiIiVVhJglVj4ECx5fSCdcX9BWgLHAI2AxMsy3J7pcIb9PIXO7Hb\nDE/9sqWvSxEREZEqzluT1wcCqUAjIAb4izGm1qWNjDFPGGNSjDEpGRkZXjr01W07dIZlmw4xulcE\nDWoFlvvxREREpHorSbA6CBS/90uTgnXFjQY+tDx2A3uBy/78zrKsNy3L6mpZVtd69eqVteYSe3Hl\nDmoF+jGmb4tyP5aIiIhISYLVD0BLY0xEwYT03wDLL2nzM3AngDGmAdAa2OPNQkvruz0nWLMzg//8\nRQtCg/x8WYqIiIhUE47rNbAsy2mMeRJYCdiBuZZlbTXGjCnYngT8EXjXGLMZMMBky7KOl2Pd16uZ\nmZ/vILxWIAm3N/dVGSIiIlLNXDdYAViW9Snw6SXrkoq9PgQM8G5pZffPbUf598+n+dOIKAL97L4u\nR0RERKqJKnfldafLzayVO4msV4P7ujTxdTkiIiJSjVS5YPXhvw+y+1g2zw5ojcNe5d6eiIiIVGBV\nKnnk5rt45YufiG5am0Edwn1djoiIiFQzVSpYvf/tfg5l5jJ5UGs8F4EXERERuXmqTLA6k5vPnDW7\n6dOqHre3qOvrckRERKQaqjLB6s2v9nA6J59JA1v7uhQRERGppqpEsDp2Jpd3vt7LPdGN6NA41Nfl\niIiISDVVJYLVa6t2ke9y88xdrXxdioiIiFRjlT5Y7Tt+loXfH+D+bs1oXreGr8sRERGRaqzSB6uX\nvvgJP7uNcXfe5utSREREpJqr1MFqy8FMPtl0iN/eEUH9moG+LkdERESquUodrGZ+voOwYD+e6Bvp\n61JEREREKm+wWrf7OGt3HWdsv9uoFejn63JEREREKmewsiyLmZ/voFFoIA/2uNXX5YiIiIgAlTRY\nfb7lCJvSM3nqrlYE+tl9XY6IiIgIUAmDldPlZtY/d9Kyfgi/7tzE1+WIiIiIFKl0wWrJhnT2ZJzl\n2YGtsdt0o2URERGpOCpVsMrNd/HKlz/RuVlt7mrXwNfliIiIiFykUgWrd9ft4+iZ80we1AZjNFol\nIiIiFUulCVaZOfn8z+rd9Gtdj+6Rt/i6HBEREZHLVJpg9cZXaWSddzJpUBtflyIiIiJyRZUiWB3J\nzGXeN3sZFt2Itg1r+bocERERkSuqFMHq1X/twm1ZPDOgta9LEREREbmqCh+s0jKyWZxygFHdb6Vp\nnWBflyMiIiJyVRU+WL30z50EOGw82f82X5ciIiIick0VOlhtOnCaTzcf4bHekdQNCfB1OSIiIiLX\nVKGD1Ysrd1Cnhj+P947wdSkiIiIi11Vhg9XaXRl8s/sET/a7jZqBfr4uR0REROS6KmSwcrstZn6+\ngyZhQYzq0czX5YiIiIiUSIUMVp9uOcyWg2eYeFcrAhx2X5cjIiIiUiIVLljlu9zMXrmT1g1qMiym\nsa/LERERESmxChesFv1wgH0ncpg0qDV2m260LCIiIpVHhQpWOXlOXv3XLmKbh9G/TX1flyMiIiJS\nKhUqWM37Zh8ZWeeZPKgNxmi0SkRERCqXChOsTufkkfRVGr9sW5+uzev4uhwRERGRUqswwep/1qSR\nfd7JswPb+LoUERERkTKpEMHq0OlzvLtuHyM6NaF1eE1flyMiIiJSJhUiWL365S6w4Om7Wvq6FBER\nEZEy83mw2n0siyUbDvBgj1tpEhbs63JEREREysznwWrWyp0E+zt4sv9tvi5FRERE5Ib4NFht/PkU\nK7ce5Yk+kdSp4e/LUkRERERumE+D1czPdlA3xJ/f3hHhyzJEREREvMJnwSor18n6vScZ178lNQIc\nvipDRERExGt8FqyOnMmlWZ1g7u/WzFcliIiIiHiVz4JVbr6LZwa0wt/h8/nzIiIiIl7hs1QT7G/n\nno6NfHV4EREREa/zWbBqUS8Em003WhYREZGqQ+fhRERERLxEwUpERETESxSsRERERLxEwUpERETE\nSxSsRERERLxEwUpERETESxSsRERERLxEwUpERETESxSsRERERLxEwUpERETESxSsRERERLxEwUpE\nRETESxSsRERERLxEwUpERETESxSsRERERLxEwUpERETESxSsRERERLxEwUpERETESxSsRERERLxE\nwUpERETESxSsRERERLxEwUpERETESxSsRERERLxEwUpERETESxSsRERERLxEwUpERETESxSsRERE\nRLxEwUpERETESxSsRERERLxEwUpERETESxSsRERERLxEwUpERETESxy+LkBERKQyyc/PJz09ndzc\nXF+XIuUgMDCQJk2a4OfnV6b9FaxERERKIT09nZo1a9K8eXOMMb4uR7zIsixOnDhBeno6ERERZepD\npwJFRERKITc3l1tuuUWhqgoyxnDLLbfc0GikgpWIiEgpKVRVXTf63SpYiYiIiHiJgpWIiIiU2Zo1\na1i3bp2vy6gwFKxERESkzBSsLqZgJSIiUgnNnz+fjh07Eh0dzUMPPcQnn3xC9+7d6dSpE7/85S85\nevQoAImJiTz00EP07NmTli1b8tZbbwFw+PBh+vTpQ0xMDB06dGDt2rUAhISEMHXqVKKjo+nRo0dR\nPxkZGfz6178mNjaW2NhYvvnmG/bt20dSUhJ//vOfiYmJKerjUlerLTs7m9GjRxMVFUXHjh354IMP\nAPj888/p3Lkz0dHR3HnnneX6OXqbsSzLJwfu2rWrlZKS4pNji4iIlNX27dtp27YtAM9/spVth854\ntf92jWrxX/e0v2abrVu3Mnz4cNatW0fdunU5efIkxhhq166NMYa3336b7du389JLL5GYmMhHH33E\nd999x9mzZ+nUqRPr16/n73//O7m5uUydOhWXy0VOTg41a9bEGMPy5cu55557mDRpErVq1WLatGk8\n8MAD/O53v+OOO+7g559/ZuDAgWzfvp3ExERCQkL4/e9/f9V6T506dcXaJk+ezPnz53nllVeK2jmd\nTjp37kxycjIRERGcPHmSOnXqePUzvp7i33EhY8wGy7K6Xm9fXcdKRESkklm1ahVxcXHUrVsXgDp1\n6rB582bi4+M5fPgweXl5F12HadiwYQQFBREUFES/fv34/vvviY2N5dFHHyU/P597772XmJgYAPz9\n/RkyZAgAXbp04YsvvgDgyy+/ZNu2bUV9njlzhuzs7BLVm56efsXavvzySxYuXFjULiwsjE8++YQ+\nffoUtbnZoepGKViJiIiU0fVGlm6mcePGMXHiRIYOHcqaNWtITEws2nbpJQSMMfTp04fk5GRWrFhB\nQkICEydO5OGHH8bPz6+ovd1ux+l0AuB2u/nuu+8IDAz0am1VjeZYiYiIVDL9+/dnyZIlnDhxAoCT\nJ0+SmZlJ48aNAXjvvfcuar9s2TJyc3M5ceIEa9asITY2lv3799OgQQMef/xxHnvsMTZu3HjNYw4Y\nMIDXX3+9aDk1NRWAmjVrkpWVdc19r1bbXXfdxZw5c4qWT506RY8ePUhOTmbv3r1F760yKVGwMsYM\nMsbsNMbsNsZMuUqbXxhjUo0xW40xX3m3TBERESnUvn17pk6dSt++fYmOjmbixIkkJiYSFxdHly5d\nik4RFurYsSP9+vWjR48ePPfcczRq1Ig1a9YQHR1Np06dWLRoERMmTLjmMV977TVSUlLo2LEj7dq1\nIykpCYB77rmHjz766JqT169W27Rp0zh16hQdOnQgOjqa1atXU69ePd58801GjBhBdHQ08fHxN/hp\n3VzXnbxujLEDPwF3AenAD8D9lmVtK9amNrAOGGRZ1s/GmPqWZR27Vr+avC4iIpXRlSY2V2QlmVwu\nF7uRyeslGbHqBuy2LGuPZVl5wEJg2CVtHgA+tCzrZ4DrhSoRERGRqqgkk9cbAweKLacD3S9p0wrw\nM8asAWp56I7xAAAgAElEQVQCr1qWNd8rFYqIiEiZ3cyJ4jNmzGDJkiUXrYuLi2Pq1Kk3rQZf89Zf\nBTqALsCdQBDwrTHmO8uyfireyBjzBPAEQLNmzbx0aBEREakIpk6dWq1C1JWU5FTgQaBpseUmBeuK\nSwdWWpZ11rKs40AyEH1pR5ZlvWlZVlfLsrrWq1evrDWLiIiIVEglCVY/AC2NMRHGGH/gN8DyS9os\nA+4wxjiMMcF4ThVu926pIiIiIhXbdU8FWpblNMY8CawE7MBcy7K2GmPGFGxPsixruzHmc+BHwA28\nbVnWlvIsXERERKSiKdEcK8uyPgU+vWRd0iXLs4BZ3itNREREpHLRlddFREREvETBSkRERMpszZo1\nrFu37pptEhMTmT179k2qyLcUrERERKTMShKsqhNvXcdKRESk+vlsChzZ7N0+w6Ng8J+u22z+/PnM\nnj0bYwwdO3Zk5MiRTJ8+nby8PG655RYWLFhAgwYNSExMJC0tjd27d3P8+HEmTZrE448/zuHDh4mP\nj+fMmTM4nU7eeOMNevfuTUhICBMmTOAf//gHQUFBLFu2jAYNGpCRkcGYMWP4+eefAXjllVdo3Lgx\nSUlJ2O12/va3v/H666/Tu3fva9admprKmDFjyMnJoUWLFsydO5ewsDBee+01kpKScDgctGvXjoUL\nF/LVV18V3cPQGENycjI1a9a88c+4HClYiYiIVDJbt25l+vTprFu3jrp163Ly5EmMMXz33XcYY3j7\n7bd58cUXeemllwD48ccf+e677zh79iydOnXiV7/6FX//+98ZOHAgU6dOxeVykZOTA8DZs2fp0aMH\nM2bMYNKkSbz11ltMmzaNCRMm8PTTT3PHHXfw888/M3DgQLZv386YMWNKdS/Chx9+mNdff52+ffvy\nhz/8geeff55XXnmFP/3pT+zdu5eAgABOnz4NwOzZs5kzZw69evUiOzubwMDA8vlAvUjBSkREpKxK\nMLJUHlatWkVcXBx169YFoE6dOmzevJn4+HgOHz5MXl4eERERRe2HDRtGUFAQQUFB9OvXj++//57Y\n2FgeffRR8vPzuffee4mJiQHA39+fIUOGANClSxe++OILAL788ku2bdtW1OeZM2fIzs4uVd2ZmZmc\nPn2avn37AvDII48QFxcHQMeOHRk1ahT33nsv9957LwC9evVi4sSJjBo1ihEjRtCkSZOyfFw3leZY\niYiIVAHjxo3jySefZPPmzfz1r38lNze3aJsx5qK2xhj69OlDcnIyjRs3JiEhgfnzPbf49fPzK2pv\nt9txOp0AuN1uvvvuO1JTU0lNTeXgwYOEhIR4rf4VK1YwduxYNm7cSGxsLE6nkylTpvD2229z7tw5\nevXqxY4dO7x2vPKiYCUiIlLJ9O/fnyVLlnDixAkATp48SWZmJo0bNwbgvffeu6j9smXLyM3N5cSJ\nE6xZs4bY2Fj2799PgwYNePzxx3nsscfYuHHjNY85YMAAXn/99aLl1NRUAGrWrElWVlaJ6g4NDSUs\nLIy1a9cC8P7779O3b1/cbjcHDhygX79+zJw5k8zMTLKzs0lLSyMqKorJkycTGxtbKYKVTgWKiIhU\nMu3bt2fq1Kn07dsXu91Op06dSExMJC4ujrCwMPr378/evXuL2nfs2JF+/fpx/PhxnnvuORo1asR7\n773HrFmz8PPzIyQkpGjE6mpee+01xo4dS8eOHXE6nfTp04ekpCTuuece7rvvPpYtW1aiyevvvfde\n0eT1yMhI5s2bh8vl4sEHHyQzMxPLshg/fjy1a9fmueeeY/Xq1dhsNtq3b8/gwYO98vmVJ2NZlk8O\n3LVrVyslJcUnxxYRESmr7du307ZtW1+XUWKJiYmlmlwuV/6OjTEbLMvqer19dSpQRERExEt0KlBE\nRKQKS0xMvGnHmjFjBkuWLLloXVxcHFOnTr1pNfiagpWIiIh4xdSpU6tViLoSnQoUERER8RIFKxER\nEREvUbASERER8RIFKxEREREvUbASERGp5BITE5k9e3aF7vvdd9/lySef9EJFFZuClYiIiHhN4b0F\nqytdbkFERKSMZn4/kx0nvXv/ujZ12jC52+TrtpsxYwbvvfce9evXp2nTpnTp0oW0tDTGjh1LRkYG\nwcHBvPXWW7Rp04ajR48yZswY9uzZA8Abb7zB7bffzssvv8zcuXMBeOyxx3jqqaeu2jdw1f4TEhII\nDAzk3//+N7169eLll1++Zu379u3j0Ucf5fjx49SrV4958+bRrFkzlixZwvPPP4/dbic0NJTk5GS2\nbt3K6NGjycvLw+1288EHH9CyZcsb+YjLlYKViIhIJbNhwwYWLlxIamoqTqeTzp0706VLF5544gmS\nkpJo2bIl69ev53e/+x2rVq1i/Pjx9O3bl48++giXy0V2djYbNmxg3rx5rF+/Hsuy6N69e9ENka/U\nN3DV/gHS09NZt24ddrv9uvWPGzeORx55hEceeYS5c+cyfvx4Pv74Y1544QVWrlxJ48aNOX36NABJ\nSUlMmDCBUaNGkZeXh8vlKr8P1gsUrERERMqoJCNL5WHt2rUMHz6c4OBgAIYOHUpubi7r1q0jLi6u\nqN358+cBWLVqVdFNlgtHg77++muGDx9OjRo1ABgxYgRr167F7XZf1jdAdnb2VfsHzxXWSxKqAL79\n9ls+/PBDAB566CEmTZoEQK9evUhISGDkyJGMGDECgJ49ezJjxgzS09MZMWJEhR6tAgUrERGRKsHt\ndlO7dm1SU1N90n9hQLsRSUlJrF+/nhUrVtClSxc2bNjAAw88QPfu3VmxYgV33303f/3rX+nfv/8N\nH6u8aPK6iIhIJdOnTx8+/vhjzp07R1ZWFp988gnBwcFEREQU3avPsiw2bdoEwJ133skbb7wBgMvl\nIjMzk969e/Pxxx+Tk5PD2bNn+eijj+jdu/cV+waoVavWVfsvrdtvv52FCxcCsGDBAnr37g145nB1\n796dF154gXr16nHgwAH27NlDZGQk48ePZ9iwYfz4449l/+BuAgUrERGRSqZz587Ex8cTHR3N4MGD\niY2NBTwh5Z133iE6Opr27duzbNkyAF599VVWr15NVFQUXbp0Ydu2bXTu3JmEhAS6detG9+7deeyx\nx+jUqdNV+75W/6X1+uuvM2/ePDp27Mj777/Pq6++CsCzzz5LVFQUHTp04Pbbbyc6OprFixfToUMH\nYmJi2LJlCw8//PANfnrly1iW5ZMDd+3a1UpJSfHJsUVERMpq+/bttG3b1tdlSDm60ndsjNlgWVbX\n6+2rESsRERERL9HkdREREfGaefPmFZ3aK9SrVy/mzJnjo4puLgUrERER8ZrRo0czevRoX5fhMzoV\nKCIiIuIlClYiIiIiXqJgJSIiIuIlClYiIiKVTEhIiK9LkKtQsBIRERHxEgUrERGRSsqyLJ599lk6\ndOhAVFQUixYtAuDw4cP06dOHmJgYOnTowNq1a3G5XCQkJBS1/fOf/+zj6qsmXW5BRESkjI78v//H\n+e07vNpnQNs2hP/f/1uith9++CGpqals2rSJ48ePExsbS58+ffjf//1fBg4cyNSpU3G5XOTk5JCa\nmsrBgwfZsmULAKdPn/Zq3eKhESsREZFK6uuvv+b+++/HbrfToEED+vbtyw8//EBsbCzz5s0jMTGR\nzZs3U7NmTSIjI9mzZw/jxo3j888/p1atWr4uv0rSiJWIiEgZlXRk6Wbr06cPycnJrFixgoSEBCZO\nnMjDDz/Mpk2bWLlyJUlJSSxevJi5c+f6utQqRyNWIiIilVTv3r1ZtGgRLpeLjIwMkpOT6datG/v3\n76dBgwY8/vjjPPbYY2zcuJHjx4/jdrv59a9/zfTp09m4caOvy6+SNGIlIiJSSQ0fPpxvv/2W6Oho\njDG8+OKLhIeH89577zFr1iz8/PwICQlh/vz5HDx4kNGjR+N2uwH47//+bx9XXzUZy7J8cuCuXbta\nKSkpPjm2iIhIWW3fvp22bdv6ugwpR1f6jo0xGyzL6nq9fXUqUERERMRLFKxEREREvETBSkRERMRL\nFKxEREREvETBSkRERMRLFKxEREREvETBSkREpIoLCQnxdQkl9vHHH7Nt2zZfl1FmClYiIiJSYShY\niYiIyE01ZcoU5syZU7ScmJjI9OnTufPOO+ncuTNRUVEsW7asxP3NnDmTqKgooqOjmTJlCgBvvfUW\nsbGxREdH8+tf/5qcnBwAEhISGDNmDF27dqVVq1b84x//AGDr1q1069aNmJgYOnbsyK5du9i3bx9t\n27bl8ccfp3379gwYMIBz584BkJaWxqBBg+jSpQu9e/dmx44drFu3juXLl/Pss88SExNDWlraFeu9\nWm1Hjx5l+PDhREdHEx0dzbp16wCYP38+HTt2JDo6moceeqiUn3bp6MrrIiIipVD8qtxrF//E8QPZ\nXu2/btMQeo9sdc02//73v3nqqaf46quvAGjXrh0rV64kNDSUWrVqcfz4cXr06MGuXbswxhASEkJ2\n9pXr/Oyzz/jjH//Il19+SXBwMCdPnqROnTqcOHGCW265BYBp06bRoEEDxo0bR0JCAkeOHOHTTz8l\nLS2Nfv36sXv3bp599ll69OjBqFGjyMvLw+VycfToUW677TZSUlKIiYlh5MiRDB06lAcffJA777yT\npKQkWrZsyfr16/k//+f/sGrVKhISEhgyZAj33XffVd//1WqLj4+nZ8+ePPXUU7hcLrKzs0lPT2f4\n8OGsW7eOunXrFr2/a7mRK6/rXoEiIiKVTKdOnTh27BiHDh0iIyODsLAwwsPDefrpp0lOTsZms3Hw\n4EGOHj1KeHj4Nfv68ssvGT16NMHBwQBFoWPLli1MmzaN06dPk52dzcCBA4v2GTlyJDabjZYtWxIZ\nGcmOHTvo2bMnM2bMID09nREjRtCyZUsAIiIiiImJAaBLly7s27eP7Oxs1q1bR1xcXFGf58+fL/H7\nv1ptq1atYv78+QDY7XZCQ0OZP38+cXFx1K1b96L3V14UrERERMroeiNL5SkuLo6lS5dy5MgR4uPj\nWbBgARkZGWzYsAE/Pz+aN29Obm5umftPSEjg448/Jjo6mnfffZc1a9YUbTPGXNTWGMMDDzxA9+7d\nWbFiBXfffTd//etfiYyMJCAgoKid3W7n3LlzuN1uateuTWpqqtdr8zXNsRIREamE4uPjWbhwIUuX\nLiUuLo7MzEzq16+Pn58fq1evZv/+/SXq56677mLevHlF85ROnjwJQFZWFg0bNiQ/P58FCxZctM+S\nJUtwu92kpaWxZ88eWrduzZ49e4iMjGT8+PEMGzaMH3/88arHrFWrFhERESxZsgQAy7LYtGkTADVr\n1iQrK+uaNV+ttjvvvJM33ngDAJfLRWZmJv3792fJkiWcOHHiovdXXhSsREREKqH27duTlZVF48aN\nadiwIaNGjSIlJYWoqCjmz59PmzZtStTPoEGDGDp0KF27diUmJobZs2cD8Mc//pHu3bvTq1evy/pq\n1qwZ3bp1Y/DgwSQlJREYGMjixYvp0KEDMTExbNmyhYcffviax12wYAHvvPMO0dHRtG/fvmiy/W9+\n8xtmzZpFp06drjp5/Wq1vfrqq6xevZqoqCi6dOnCtm3baN++PVOnTqVv375ER0czceLEEn0uZaXJ\n6yIiIqVwpYnN1UlJJpdXdjcyeV0jViIiIiJeosnrIiIi1cDmzZsvu4ZTQEAA69evL1U/7777rher\nuraxY8fyzTffXLRuwoQJjB49+qbVUFoKViIiItVAVFRUmf8Kz1eKXwS1stCpQBEREREvUbASERER\n8RIFKxEREREvUbASERER8RIFKxERkSouJCTE1yWU2Mcff8y2bduu2SYhIYGlS5fepIpKR8FKRERE\nKoySBKuKTJdbEBERKaPV777Jsf17vNpn/Vsj6ZfwxDXbTJkyhaZNmzJ27FgAEhMTcTgcrF69mlOn\nTpGfn8/06dMZNmxYiY45c+ZM/va3v2Gz2Rg8eDB/+tOfeOutt3jzzTfJy8vjtttu4/333yc4OJiE\nhAQCAwNJSUnhzJkzvPzyywwZMoStW7cyevRo8vLycLvdfPDBB/j5+TF48GDuuOMO1q1bR+PGjVm2\nbBlBQUGkpaUxduxYMjIyCA4O5q233uLkyZMsX76cr776iunTp/PBBx/QokWLa9b+r3/9i9///vc4\nnU5iY2N54403CAgIYMqUKSxfvhyHw8GAAQOYPXs2S5Ys4fnnn8dutxMaGkpycnLJvpRS0IiViIhI\nJRMfH8/ixYuLlhcvXswjjzzCRx99xMaNG1m9ejXPPPMMJblt3WeffcayZctYv349mzZtYtKkSQCM\nGDGCH374gU2bNtG2bVveeeedon327dvH999/z4oVKxgzZgy5ubkkJSUxYcIEUlNTSUlJoUmTJgDs\n2rWLsWPHsnXrVmrXrs0HH3wAwBNPPMHrr7/Ohg0bmD17Nr/73e+4/fbbGTp0KLNmzSI1NfW6oSo3\nN5eEhAQWLVrE5s2bcTqdvPHGG5w4cYKPPvqIrVu38uOPPzJt2jQAXnjhBVauXMmmTZtYvnx56T70\nEtKIlYiISBldb2SpvHTq1Iljx45x6NAhMjIyCAsLIzw8nKeffprk5GRsNhsHDx7k6NGjhIeHX7Ov\nL7/8ktGjRxMcHAxAnTp1ANiyZQvTpk3j9OnTZGdnM3DgwKJ9Ro4cic1mo2XLlkRGRrJjxw569uzJ\njBkzSE9PZ8SIEbRs2RKAiIgIYmJiAOjSpQv79u0jOzubdevWERcXV9Tn+fPnS/057Ny5k4iICFq1\nagXAI488wpw5c3jyyScJDAzkt7/9LUOGDGHIkCEA9OrVi4SEBEaOHMmIESNKfbyS0IiViIhIJRQX\nF8fSpUtZtGgR8fHxLFiwgIyMDDZs2EBqaioNGjQgNze3zP0nJCTwl7/8hc2bN/Nf//VfF/VljLmo\nrTGGBx54gOXLlxMUFMTdd9/NqlWrAM9tcwrZ7XacTidut5vatWuTmppa9Ni+fXuZa72Uw+Hg+++/\n57777uMf//gHgwYNAiApKYnp06dz4MABunTpwokTJ7x2zEIKViIiIpVQfHw8CxcuZOnSpcTFxZGZ\nmUn9+vXx8/Nj9erV7N+/v0T93HXXXcybN4+cnBwATp48CUBWVhYNGzYkPz+fBQsWXLTPkiVLcLvd\npKWlsWfPHlq3bs2ePXuIjIxk/PjxDBs2jB9//PGqx6xVqxYREREsWbIEAMuy2LRpEwA1a9YkKyur\nRLW3bt2affv2sXv3bgDef/99+vbtS3Z2NpmZmdx99938+c9/Luo7LS2N7t2788ILL1CvXj0OHDhQ\nouOUhoKViIhIJdS+fXuysrJo3LgxDRs2ZNSoUaSkpBAVFcX8+fNp06ZNifoZNGgQQ4cOpWvXrsTE\nxDB79mwA/vjHP9K9e3d69ep1WV/NmjWjW7duDB48mKSkJAIDA1m8eDEdOnQgJiaGLVu28PDDD1/z\nuAsWLOCdd94hOjqa9u3bs2zZMgB+85vfMGvWLDp16kRaWto1+wgMDGTevHnExcURFRWFzWZjzJgx\nZGVlMWTIEDp27Mgdd9zByy+/DMCzzz5LVFQUHTp04Pbbbyc6OrpEn1FpmJJMbCsPXbt2tVJSUnxy\nbBERkbLavn07bdu29XUZPpOQkMCQIUO47777fF1KubnSd2yM2WBZVtfr7asRKxEREREv0V8FioiI\nVAObN2/moYceumhdQEAA69evL1U/7777rheruraxY8fyzTffXLRuwoQJjB49+qbVUFoKViIiItVA\nVFQUqampvi6jVObMmePrEkpNpwJFREREvETBSkRERMRLFKxEREREvETBSkRERMRLFKxERESquJCQ\nkBK3TUhIYOnSpeVWi7f6T0xMLLqYaUWiYCUiIiIVltPp9HUJpaLLLYiIiJTR6U/SyDt01qt9+jeq\nQe17WlyzzZQpU2jatCljx44FPKM3DoeD1atXc+rUKfLz85k+fTrDhg277vEsy2LcuHF88cUXNG3a\nFH9//6JtGzZsYOLEiWRnZ1O3bl3effddGjZsyO7duxkzZgwZGRnY7XaWLFlCZGQkkyZN4rPPPsMY\nw7Rp04iPjy9T/7/4xS+IiYnh66+/5v777+eZZ5655ntITU1lzJgx5OTk0KJFC+bOnUtYWBivvfYa\nSUlJOBwO2rVrx8KFC/nqq6+YMGEC4Ll5dHJyMjVr1rzu51RSClYiIiKVTHx8PE899VRRsFq8eDEr\nV65k/Pjx1KpVi+PHj9OjRw+GDh2KMeaafX300Ufs3LmTbdu2cfToUdq1a8ejjz5Kfn4+48aNY9my\nZdSrV49FixYxdepU5s6dy6hRo5gyZQrDhw8nNzcXt9vNhx9+SGpqKps2beL48ePExsbSp08fvv32\n21L3D5CXl0dJb3338MMP8/rrr9O3b1/+8Ic/8Pzzz/PKK6/wpz/9ib179xIQEMDp06cBmD17NnPm\nzKFXr15kZ2cTGBh4A9/E5RSsREREyuh6I0vlpVOnThw7doxDhw6RkZFBWFgY4eHhPP300yQnJ2Oz\n2Th48CBHjx4lPDz8mn0lJydz//33Y7fbadSoEf379wdg586dbNmyhbvuugsAl8tFw4YNycrK4uDB\ngwwfPhygKJgUji7Z7XYaNGhA3759+eGHH0rdf6H4+PgSfRaZmZmcPn2avn37AvDII48QFxcHQMeO\nHRk1ahT33nsv9957LwC9evVi4sSJjBo1ihEjRtCkSZMSHaekFKxEREQqobi4OJYuXcqRI0eIj49n\nwYIFZGRksGHDBvz8/GjevDm5ubll7t+yLNq3b8+333570fqsrKwbLf2a/ReqUaPGDR9jxYoVJCcn\n88knnzBjxgw2b97MlClT+NWvfsWnn35Kr169WLlyJW3atLnhYxXS5HUREZFKKD4+noULF7J06VLi\n4uLIzMykfv36+Pn5sXr1avbv31+ifvr06cOiRYtwuVwcPnyY1atXA9C6dWsyMjKKgk9+fj5bt26l\nZs2aNGnShI8//hiA8+fPk5OTQ+/evYv6ycjIIDk5mW7dupW6/9IKDQ0lLCyMtWvXAvD+++/Tt29f\n3G43Bw4coF+/fsycOZPMzEyys7NJS0sjKiqKyZMnExsby44dO0p9zGvRiJWIiEgl1L59e7Kysmjc\nuDENGzZk1KhR3HPPPURFRdG1a9cSj8IMHz6cVatW0a5dO5o1a0bPnj0B8Pf3Z+nSpYwfP57MzEyc\nTidPPfUU7du35/333+c//uM/+MMf/oCfnx9Llixh+PDhfPvtt0RHR2OM4cUXXyQ8PLxM/ZfWe++9\nVzR5PTIyknnz5uFyuXjwwQfJzMzEsizGjx9P7dq1ee6551i9ejU2m4327dszePDgUh/vWoxlWV7t\nsKS6du1qlXRSmoiISEWxfft22rZt6+sypBxd6Ts2xmywLKvr9fbVqUARERERL9GpQBERkWpg8+bN\nPPTQQxetCwgIYP369T6qqGRmzJjBkiVLLloXFxfH1KlTfVTRtelUoIiISCnoVGDVV+6nAo0xg4wx\nO40xu40xU67RLtYY4zTG3FeSfkVERESqkusGK2OMHZgDDAbaAfcbY9pdpd1M4J/eLlJERESkMijJ\niFU3YLdlWXssy8oDFgJXuvnQOOAD4JgX6xMRERGpNEoSrBoDB4otpxesK2KMaQwMB97wXmkiIiIi\nlYu3LrfwCjDZsiz3tRoZY54wxqQYY1IyMjK8dGgRERG5lpCQkKtu27dvHx06dLiJ1VRtJbncwkGg\nabHlJgXriusKLCy4g3Zd4G5jjNOyrI+LN7Is603gTfD8VWBZixYRERGpiEoSrH4AWhpjIvAEqt8A\nDxRvYFlWROFrY8y7wD8uDVUiIiJVzWeffcaRI0e82md4ePh1b7MyZcoUmjZtytixYwFITEzE4XCw\nevVqTp06RX5+PtOnT2fYsCtNib663Nxc/vM//5OUlBQcDgcvv/wy/fr1Y+vWrYwePZq8vDzcbjcf\nfPABjRo1YuTIkaSnp+NyuXjuueeIj48v8/uuKq4brCzLchpjngRWAnZgrmVZW40xYwq2J5VzjSIi\nIlJMfHw8Tz31VFGwWrx4MStXrmT8+PHUqlWL48eP06NHD4YOHUrB2aQSmTNnDsYYNm/ezI4dOxgw\nYAA//fQTSUlJTJgwgVGjRpGXl4fL5eLTTz+lUaNGrFixAoDMzMxyea+VTYmuvG5Z1qfAp5esu2Kg\nsiwr4cbLEhERqfi8fQPfkurUqRPHjh3j0KFDZGRkEBYWRnh4OE8//TTJycnYbDYOHjzI0aNHCQ8P\nL3G/X3/9NePGjQOgTZs23Hrrrfz000/07NmTGTNmkJ6ezogRI2jZsiVRUVE888wzTJ48mSFDhtC7\nd+/yeruViu4VKCIiUgnFxcWxdOlSFi1aRHx8PAsWLCAjI4MNGzaQmppKgwYNyM3N9cqxHnjgAZYv\nX05QUBB33303q1atolWrVmzcuJGoqCimTZvGCy+84JVjVXa6V6CIiEglFB///9u78zg56jr/469v\nHX3MmYScJBAhCSABFA0RQXRFFBFW1HU9Vlx113U9WcQD1p97uY9VRMVrxftaV1fxAA/wZL1Y5D7C\noUASICSE3Jmzjzq+vz+quqdnpudKOt2dzPv5eNSjqqu+XfOZRjvv+da3vvVK/u7v/o6dO3fy29/+\nlquuuoqFCxfi+z6//vWvefTRR2d8zjPOOINvfvObnHnmmTz44INs2rSJY489lo0bN3L00Udz4YUX\nsmnTJtatW8dxxx3HvHnzuOCCC5gzZw5f+tKXDsBvefBRsBIRETkIrV69moGBAZYuXcqSJUt4zWte\nw5//+Z9z4oknsmbNGo477rgZn/Otb30rb3nLWzjxxBPxPI+vfe1rZLNZrrrqKr7xjW/g+z6LFy/m\nfe97H7feeivvec97cBwH3/f57Gc1lSXoIcwiIiIzoocwH/oO+EOYRURERGRquhQoIiIyC9xzzz28\n9pqtEcYAACAASURBVLWvHbUvm81y8803t6iiQ5OClYiIyCxw4oknctddd7W6jEOeLgWKiIjMUKvG\nJ8uBt7//bRWsREREZiCXy7Fr1y6Fq0OQtZZdu3aRy+X2+Ry6FCgiIjIDy5YtY/PmzezYsaPVpcgB\nkMvlWLZs2T6/X8FKRERkBnzf56ijjmp1GdKmdClQREREpEEUrEREREQaRMFKREREpEEUrEREREQa\nRMFKREREpEEUrEREREQaRMFKREREpEEUrEREREQaRMFKREREpEEUrEREREQaRMFKREREpEEUrERE\nREQaRMFKREREpEEUrEREREQaRMFKREREpEEUrEREREQaRMFKREREpEEUrEREREQaRMFKREREpEEU\nrEREREQaRMFKREREpEFaFqxKGzdio6hVP15ERESk4VoWrOxwgb6rr27VjxcRERFpuJYFK6cjz/ZP\nfJJocKhVJYiIiIg0VMuClbdkCdHOnez64hdbVYKIiIhIQ7Wuxyqfp+e889j91a8SbNnSqjJERERE\nGqaldwUuvPidYAzbP3ZFK8sQERERaYiWBiv/8MOZ9zdvoP+66xi+885WliIiIiKy31o+j9X8N74R\nb8ECtl12GTaOW12OiIiIyD5rebByOjtZcNFFFO9eR/+117W6HBEREZF91vJgBdD70peQPf7JbL/i\nCuJCodXliIiIiOyTtghWxnFYdOmlhFu3svtrX2t1OSIiIiL7pC2CFUDn2rV0P/8sdn7xSwTbtre6\nHBEREZEZa5tgBbDw3e/GBgE7PvnJVpciIiIiMmNtFawyy5cz74IL6Lv6agr33dfqckRERERmpK2C\nFcD8t7wZd84ctl/2Yay1rS5HREREZNraLli5PT3Mf8fbGb71Vgavv77V5YiIiIhMW9sFK4C5r3gF\nmZUr2PaRj2DL5VaXIyIiIjItbRmsjOex6JJLCB7dxO5vfqvV5YiIiIhMS1sGK4CuM86g84wz2Hnl\nlYR79rS6HBEREZEptW2wAlh0yXuJh4fZ+en/bHUpIiIiIlNq62CVXbmSua98BXu+8x1K69e3uhwR\nERGRSbUuWPU9Nq1m89/xDpyODrZdfvkBLkhERERk/7QuWA3thD9dO2Uzb+5c5r/lLQz97vcM/v6G\nJhQmIiIism9aF6z8PFz3XigNTtl07gWvwT/ySLZ9+DJsGDahOBEREZGZa12w6j0C+jfDbz40ZVMn\nk2Hhe95Nef0G9n73u00oTkRERGTmWhesMp3wtNfBTZ+FJ+6Zsnn3WWfRccop7PjUp4n6+5tQoIiI\niMjMtPauwLP+FfJz4SfvhDietKkxhoWXXkK0dy87P/f5ppQnIiIiMhOtDVYd8+Ds/4DNt8IdX5uy\neX71anpf8hJ2f+MblDdtOvD1iYiIiMxA6+exOumV8KQz4Ff/CoPbp2y+4KKLML7P9o989MDXJiIi\nIjIDrQ9WxsC5V0B5GH7x/imb+4sWctgb/5aBX/6SoVtuaUKBIiIiItPT+mAFsOAYeNY7Yd13YONv\npmx+2BvegLd4Mdsv+zB2irFZIiIiIs3SHsEK4IyLYe5R8JOLIShO2tTJ51n4rosp3n8/fdf8sEkF\nioiIiEyufYKVn4dzPwa7N8D/fXLK5j3nnkvupJPY8fGPEw8NNaFAERERkcm1T7ACWPk8OOEv4Pcf\ng10bJm1qHIdFl15KuGMHu7785SYVKCIiIjKx9gpWAGd/ELwsXHsxWDtp046nnUzPi85h11e+SrB1\na5MKFBEREamv/YJV92J43j8ng9jv/f6UzRe+610Qx2y/4uMHvjYRERGRSbRfsAJY8zdw+Mnws3+E\nwt5Jm/pLlzLv9a+n/8c/prBuXZMKFBERERmvPYOV48J5n4DhnXD9B6Zsftib3oQ7fz7bPnQZdorL\nhyIiIiIHSnsGK4DDnwpr/x5u+wpsvm3Spm5XJwv+4UIKd97JwE9/2qQCRUREREZr32AF8Nz3JWOu\nfnIRROGkTee87GVkjzuO7R/9GHGp1KQCRUREREa0d7DK9cA5H4Yn7oFbPj9pU+O6LLr0EoLHH2f3\n177epAJFRERERrR3sAJ48oth1Qvgf/8D+jZP2rTz1FPpOvNMdn3+84Q7dzapQBEREZFE+wcrY+BF\nHwEbw88unbL5wve8m7hcZscnP9WE4kRERERGtH+wApj7JHjOe+GPP4YHfjZp0+xRRzHvNX/F3u9/\nn+IDDzSnPhEREREOlmAF8My3w4Lj4Lr3QHnyZwPOf+tbcbu72XaZpl8QERGR5jl4gpWXgfM+Dn2b\n4LeXT9rU7e1l/tvfzvAfbmLw179pTn0iIiIy6x08wQpg+Wlw8gXwh/+EbfdP2nTuq15J5uij2X75\n5dhyuUkFioiIyGx2cAUrgLM+ANke+Mk7IY4nbGZ8n4XvfQ/lRx5hz7e/3cQCRUREZLY6+IJV52Hw\ngn+Hx26Cu/570qZdz3kOnaedxo7PXEm0d/JnDoqIiIjsr4MvWAE89TWw/HT45T/D0MTzVRljWHjp\nJcQDA+z4zJVNLFBERERmo4MzWBkD514BpQH4xT9N2jR3zDHM+cu/ZM///A+ljQ83qUARERGZjQ7O\nYAWw8Dg47UK4+1vw8O8nbbrgwnfg5HJsv3zyuwlFRERE9sfBG6wAnv0emLMcrr0Ywonv/PMOO4z5\nb/57Bn/zG4ZuvLGJBYqIiMhscnAHq0wHvOijsPNBuPGTkzad+9rX4i9bxrbLPoyNoiYVKCIiIrPJ\nwR2sAI55ARx/Pvzuo7B744TNnGyWhe9+F6UHH2Tv977fxAJFRERktjj4gxXACy8Dx08edzPJI2y6\nzz6b/NOfzo5PfYpocLCJBYqIiMhscGgEq57D4cz3w/pfwX1XT9jMGMOiSy8h2rWLXZ//fBMLFBER\nkdng0AhWAKe8EZY8BX72j1Dsm7BZ/sQT6T3/xez+2tcpb97cxAJFRETkUHfoBCvXSx7SPLgN/vc/\nJm264J3vBNdl+0c/1qTiREREZDaYVrAyxrzQGPOAMWa9MebSOsdfY4xZZ4y5xxhzozHmKY0vdRqW\nPh3W/h3c+kXYcseEzfzFiznsb/+WgZ/9jOHbb29igSIiInIomzJYGWNc4DPAOcDxwKuNMcePafYw\n8Bxr7YnAvwNfaHSh03bm+6FzQfqQ5omnVTjsb/8Gb+FCtn3oMuwkD3MWERERma7p9FitBdZbazda\na8vAt4HzaxtYa2+01u5JX94ELGtsmTOQ64UXfgi23gW3fmnCZk5HBwsufifFe++l/8c/bmKBIiIi\ncqiaTrBaCjxW83pzum8ifwv8dH+K2m+rXwYrngfX/zv0b52wWe+LX0zuhBPYfsXHiYeHm1igiIiI\nHIoaOnjdGPNckmB1yQTH32SMuc0Yc9uOHTsa+aPH/iA496MQleFn44aEjTRzHBZdegnhtm3s+spX\nD1w9IiIiMitMJ1htAY6oeb0s3TeKMeYk4EvA+dbaXfVOZK39grV2jbV2zYIFC/al3umbd3TyLMH7\nr4GHfjlhs441a+g++2x2ffnLBNu2HdiaRERE5JA2nWB1K7DKGHOUMSYDvAr4UW0DY8yRwA+A11pr\nH2x8mfvo9Ath/jFw7bsgKEzYbOG73wVhyI4rPt7E4kRERORQM2WwstaGwNuBnwN/BK6y1t5njHmz\nMebNabN/Bg4DrjTG3GWMue2AVTwTXhbOvQL2Pgq/+8iEzTJHHMG81/01fT/8IYV77m1igSIiInIo\nMXaSZ+sdSGvWrLG33dak/HX1m+Ge78Gbb4CFx9VtEg0OsuEFZ5M5+iiWf+MbGGOaU5uIiIi0PWPM\n7dbaNVO1O3RmXp/M8/8dMp1w7cUTPqTZ7epiwYUXUrjtdgZ+MfGYLBEREZGJzI5g1bUAnv8BePT/\n4K5vTdhszsv/guyqVWz/6EeJy+UmFigiIiKHgtkRrABOfi0c8Qz4xftheHfdJsbzWHjpJQSPPcae\nb3yjyQWKiIjIwW72BCvHSR7SXOqHX/7zhM26Tj+druc8h52f/RzhrrqzRoiIiIjUNXuCFcCi1fDM\nt8Gd34BHb5yw2cJL3ktcLLLj059uYnEiIiJysJtdwQrgOZdA7xHwk4shrD+OKnv00cx91avYe9V3\nKT7YPtNyiYiISHubfcEq0wkv+gjs+CPc9JkJm81/21txurvZ/uHLadWUFCIiInJwmX3BCuDYc+C4\n8+A3H4Y9j9Zt4s2dy4K3voWh//s/hn73uyYXKCIiIgej2RmsAM75MBgHrnvPhHNbzX31q8ksX862\nD1+ODYImFygiIiIHm9kbrHqXwXPfBw/9HP7447pNTCbDwkveS3njRvZ856omFygiIiIHm9kbrACe\n8WZYdCL89BIoDdRt0vXc59Jx6qns/PSnifr6mlygiIiIHExmd7ByPfjzT8DAVvj1B+s2Mcaw6NJL\niPr72XnlZ5tcoIiIiBxMZnewAli2Bta8AW7+HGy9u26T3HHHMeflf8Hub32L8iOPNLc+EREROWgo\nWAE871+g4zD48UUQR3WbLLjwQhzfZ9tHPtrk4kRERORgoWAFkJ8DZ38IHr8DbvtK3SbeggUc9qY3\nMXj99QzddHOTCxQREZGDgYJVxYkvh6OeA9d/AAa21W0y7/Wvwzt8Cdsuuwwb1e/ZEhERkdlLwarC\nGDj3CgiL8PP31W3i5HIsfNe7KP3pT/RdfXWTCxQREZF2p2BVa/5KOONdcO/3YP31dZv0vOhF5J/6\nVLZ/4pNEg0NNLlBERETamYLVWKdfBPNWwLXvgqAw7rAxhkX/eCnRzp3s+uIXW1CgiIiItCsFq7H8\nHJz7MdjzMNzw8bpN8k95Cj3nncfur36VYMuWJhcoIiIi7UrBqp4Vz4UTX5EEq50P1W2y8OJ3gjFs\n/9gVTS5ORERE2pWC1UTO/g/w8nDtxXUf0uwffjjz/uYN9F93HcN33tmCAkVERKTdKFhNpGshnPUv\n8PDvYF39BzDPf+Mb8RYsSKZfiOMmFygiIiLtRsFqMk9/Ayw7JZl+obBn3GGns5MFF11E8e519F97\nXQsKFBERkXaiYDUZx4HzPp6Eql/9a90mvS99Cdnjn8z2K64gLoy/i1BERERmDwWrqSw+EU59C9z+\nNdg0/lE2xnFYdOmlhFu3svvrX29+fSIiItI2FKym48/+EXqWwk/eCVEw7nDn2rV0P/8sdn7hiwTb\nt7egQBEREWkHClbTke2Ccy6H7ffBTZ+t22Thu9+NDQJ2fPKTTS5ORERE2oWC1XQddy4ccw785kOw\n97FxhzPLlzPvggvo+8HVFO+/vwUFioiISKspWE2XMfCiy5Ptn15St8n8t7wZd84ctl32YWydua9E\nRETk0KZgNRNzjoQ/uxQeuBb+dO24w25PD/Pf8XaGb7mFwevrP8RZREREDl0KVjN16lth4fFw3Xuh\nNDju8NxXvILMyhVs+8hHsOVyCwoUERGRVlGwminXh/M+Af2b4beXjTtsPI9Fl1xC8Ogmdn/zWy0o\nUERERFpFwWpfHPkMeNrr4A9XwhP3jjvcdcYZdJ5xBjuvvJJwz/gZ20VEROTQpGC1r876V8jPgZ9c\nBHWeE7jokvcSDw+z89P/2fTSREREpDUUrPZVxzx4wX/A5lvhjvEzrmdXrmTuK1/Bnu98h9L69S0o\nUERERJpNwWp/POVV8KQz4Ff/AoM7xh2e/4534HR0sO3yy1tQnIiIiDSbgtX+MAbOvQLKw/CL/zfu\nsDd3LvPf8haGfvd7Bn9/QwsKFBERkWZSsNpfC46BZ10E674DG3877vDcC16Df+SRbPvwZdgwbEGB\nIiIi0iwtC1axHT/g+6B1xrtg7lFw7cUQlkYdcjIZFr7n3ZTXb2Dvd7/bogJFRESkGVoWrB7Y8wDv\nv+H93PbEbQf/41/8PJz7Udi1Hm74xLjD3WedRccpp7DjU58m6u9vQYEiIiLSDC0LVr1hL/evu583\nXfcmzrv6PL6w7gs8MfREq8rZfyvPgtUvg99/DHZtGHXIGMPCSy8h2ruXnZ/7fIsKFBERkQOtZcHK\ntS4nbT+J8zefzwmbTuCHv/sh537nXP7+l3/Pzx7+GaWoNPVJ2s0LPwReFq59F4zphcuvXk3vS17C\n7m98g/KmTS0qUERERA4k06rLcGvWrLE//OEPue+++7jvvvvo6+sDA7u7drM+t56B3gHOXnk2L135\nUo4/7HiMMS2pc8Zu+SJc9274iy/DiS8fdSjYtp0N55xD1+mns+zTn2pRgSIiIjJTxpjbrbVrpmzX\nymB12223AWCtZcuWLdWQ1d/fj3Us2/Lb2NSxifySPOcfez7nrTiPebl5Lal32uIIvvQ86NsCb781\nmZ29xo4rr2Tnpz7Nkf/1dTrXrm1RkSIiIjITB1WwqhXHcTVk3XvfvQwODBKbmK35rWzt2sqKVSt4\nybEv4VnLnoXv+C2ofBoevwu++FxY8zdw7sdGHYoLBTac8yK8efN40ve+i3E044WIiEi7O2iDVa04\njtm8eTP33Xcf6+5dR2GoQGQitua30je3j2ec+AxeetxLWTFnRZOqnoGfXgo3fw7eeD0se/qoQ30/\n/jGPv+e9LPngB5nzspe2qEARERGZrkMiWNWK45jHHnuMe+69h3X3rqNcKBOakK0dW3EXu5z5tDM5\nd9W5dGe6D2DVM1Dsh8+shc758He/AderHrJxzCOvejXh1q2s+NlPcTo7W1eniIiITOmQC1a14jhm\n06ZN3H737dx3/33EpZjQhGzr3Mb8o+Zz7innctoRp+GYFl9mu+8a+O7r4OwPwTPfOurQ8B138uhf\n/RXz3/oWFlx4YYsKFBERkek4pINVrSiKePTRR7nh9hvY+OBGCCAwAXt79rLiuBW87NSX8aS5T9r/\ngveFtfCtV8CjN8LbboHepaMOb7n4Ygb+99es+Ol1+EuWtKZGERERmdKsCVa1oijioY0Pcf0t17Pt\n4W04oUNgAsqHlTnpxJM4f+359OR7Gvozp7TnEfjMM2DV8+GV/z3qULBlCxvOeRHdLzybpZdf3ty6\nREREZNpmZbCqFUURt99/O7++7dcMbB7AizwCJ8Bf7HPqyady1sln4Xne1CdqhN9/DK7/ALz6O3Ds\nC0cd2n7Fx9n1hS/wpKu+Q/6kk5pTj4iIiMxI2werp550kr3z7rubMvFnGIb84o5fcNOdNxFti/Bj\nn9AN6VnWw5mnnMnJTz4Z13UPYAFl+PwZUB6Gt90MmY7qoWhwiA0vfCGZI49k+Tf/++CZCFVERGQW\naftgdcS8OfZfX/uXrFp7GitPeSZLVh7TlDmd+gp9XHPzNay7dx25XTl86xN7MYuPWszz1j6PlUev\nPDAh65H/g6+9CE6/CJ7/b6MO7fnud3nin/6ZpR+/gp5zzmn8zxYREZH90vbBavUxx9gPvOHVbLrn\nbuIopGvuPFaufSar1p7GsiefgHMge5BSG3dv5Ac3/YCND2xkbv9cfOuDDyuOXcHpTzud5cuXNzZk\nXfM2WPdt+Pvfw6Ljq7ttFPHwX7yceGCAo6+7FiebbdzPFBERkf3W9sGqMsaqODTIw3fcykO3/IGH\n77qdsFwi193DiqevZdXa01h+4lPxMpkDWksUR9zw2A1cd+t17HpkF4uHFuNZDzfrcvzxx/O0k57G\n8uXLcfa3R21oF/znGlhwLLz+Oqg539BNN7Hp9W+g66zn0fWsM8iuXEF25UrcOXMmOaGIiIg0w0ET\nrGoFpSKP3H0HD93yBzbefgul4SH8XJ6jT17DqmecxlFPfTqZfMcEZ2yMvlIfP3noJ/zvHf8L22DJ\n8BI865HJZzjphJM4YfUJHHnkkfsesu78b/jh2+DFn4an/fWoQ0988IPs/d73scPD1X3u/PlkV64k\nu2JFNWxlVq7Emzt3f35NERERmYGDMljVisKAx+5dx0O3/IH1t93EcN9eXN9n+Ukns+qUZ7JizTPI\ndx/YqRMe2P0AV//pam6991bm7p3LksISXOuS78xz4uoTWb16NUccccTMQpa18NUXwY4/wttvS2Zm\nrz0cx4Rbt1LasIHSQ+uT9Yb1lNdvIB4aqrZz581LAtfKFWRWriS7YiXZVSvx5rX5Q6pFREQOQgd9\nsKoVxxGPP/BHHrrlDzx0y40M7NyBcRyOOP4EVq49jZWnnEr3vPlTn2gfBVHAbzf/lmseuIaH1z/M\n4YOHs6SwBMc6dHV3sfr41ZxwwgksXbp0eiFr+5/gc6fDSa+El1w5rRqstYRPPEFp/XpK69OwlQav\neHCw2s6dO5fsihVkVqVhKw1f7mGH6Y5DERGRfXRIBata1lq2P7yhGrJ2b3kMgCWrjk3uMFz7TOYu\nPrzR5VbtGN7BTzb+hB898COKTxRZPrycRYVFmNjQ09PD6tWrWb16NUuXLp08yPzq3+CGK+D118KT\nnrXP9VhrCbdto7R+A+UN60eC1/r1xAMD1XZub++4sJVZsQJvwQIFLhERkSkcssFqrF2bH2P9rUnI\n2rZxPQALjnwSK9eexqpnnMb8I5YfkOBgrWXdznVcs/4afrn+l/T097CyuJJ5Q/Mght7e3mrIOvzw\nw8fXUB6GK08FLwtv/j/wGjtA31pLuH3HuLBVWr+euL+/2s7p7U3Hb42ErezKVXgLFbhEREQqZk2w\nqtW/Y3u1J2vLA/eDtcxZvIRVa09j1drTWLxi1QGZK6sQFrh+0/Vc89A13LHlDg4fPpwTohPI9+Wx\nsWXOnDnVkLVkyZKRwPLgL+Bbfwln/hM8+90Nr6seay3hjh2UN2wYCVvpZcWor6/azunpqQ6Yr4St\n7MoVeIsWKXCJiMisMyuDVa2hvXvYcNvNPHTLjWy6927iKKJr3mGsPKUyV9bqAzJX1pbBLfxo/Y/4\n4YYfsr1vO0eXjuaE6ATMboONLXPnzq2GrMWLF2O++zp48Ofw1ptg3lENr2e6rLVEu3aNC1ul9euJ\n9u6ttnO6upIxXCtHwlZ25Uq8xYsVuERE5JA164NVreLQIBvvuJWHbr6RR+6+oylzZcU25tYnbuWa\n9dfwq0d/RVyOeYp9CseUj6G0vYS1lnnz5rF61XJW3/5+Fi0/BnPB96ANw0m4e3d6h+J6yjWXFaPd\nu6ttnM7OtGcrnRpiVbL2lixpyoz6IiIiB1LbB6unPfkp9oZv/RKTcTAZF+On64yLU912MG5j/1Fu\nxVxZA+UBfv7Iz7l6/dWs27GOfJzn2Zlns2xoGX1b+7DWchi7Wf3kJ7N8zQvo7Oyks7OTjo6OA/sM\nw/0U7t6dXlKs3KmYBq6dO6ttTEdHErTSsFUJX/7hhytwiYjIQaPtg9VJS46z173ui1M3dEw1fDk1\n4ctkHIyfrJ1RwWyi/SPbTnq+2ERs/uO99efKWnsaK56+tuFzZW3cu5FrNlzDjzf8mJ2FnSzyFvHc\nzJ8x76FtbC/PxTI6bHR0dFSDVmXp6uoat6+zs5NsmzwKJ9yzZ/wYrvUbCHfsqLYx+TzZo49OJzxd\nUZ2Hy1+6VIFLRETaTtsHqzUnP93+4Re/x5ZjbBAl63KEDWLicjRuf5wes7Xrckw86r0RxDMsxDXV\nIBYTUiwNMzzcR7k8TGhDst2ddC+cT+/hi8n0dFaDmzMmxFWCW3V/xk2OufUv7YVxyI2P38jVD13N\nbzb/hjAOOXHYcEK8CCezCOP0gukBOiD0iUuWuBwTFSPioP4v6XgOmXyGbD5LLp8j15Ej35mno6Mj\nCWhdnXR1dtHVmQSzrJfFNS6e4+E5Hq5xD+g4qWjvXkobN9ZcVkyCV7h9e7WNyeXIHn30qLCVPfro\nZFqIjg6N4xIRkZZo/2B1AMZYWWshskkQqwlfowJa7f40jNlyPDq4lSPKA8OU+ocICyVMZPAcH8/J\nYJjhP+xpcHPGXfJ0qvvLbsjDw4/yp913sSvqIzARITGBiYmNJSIiNpaYmMhExNZiAayDjQ3GOpjY\nwVgHJ3ZxrYsTe7ixhxd7dWu2WAInoOyWKDtlSm6JklsicAPKXkDglwn8kNAPiTMxrudWw1cliHkm\nDWSOO2rbd3w8k+6vCW2+448/h+ORHQ7p2rKXji276Ni8i9ymnWQf246/s290za6L7eqArg7o7sJ0\nd2K6uzDd3Tg93bjd3Tg9vXi9PbjdPfi9c9JlLpneuXjZnIKZiIjsk+kGK68ZxTSLMQY8g/EcGnkx\nqTpX1s03suPhh/Ecn4XLjuKok05h+fFPpXfuQgjtSM/ahL1wNeGtGBEPlAnSNkeW53JE8GxocM61\nWEqEFEyZImUKJlmKpkyBMgUTjDoWmKjueTzrkscnZzNk8cnhkTE+GTyyuPjGxSdZHGNqAmFMSERM\nTGhCIpuuiYkIiE2JyMQM4tNnFhLPmU80dxXxU2JMHNFRtHQUIrwgwgsj3DDCD9LXQUhmW0Rm8xCZ\nUj8miojjiNBG2DiCOAIbQpy8LjsRBT+ikIkp+JbhbEwxZxnOQjFnGM5DMe9Q7vQo5T2CzgxBR4ag\nM0uUz+K6NWGxJkhWQqNr3AmPV/ZVjo/tKZzO8Xr7cm6OnJcj62YVGkVE2sAh1WPVDH3bt7H+1psO\nyFxZ1trkUqa12NhCuiTb6fHiEOx6BLv7Ydj9KOx+BLvnMdj7eHIcB3CxufnQewS25wjoPgK6D8d2\nHQ65OWDBRrbm51D9OUEQUCgPM1QsMFwqMFQuMFxZgiLD5QKFoMhwWKQQlrB1kqCDIedm6XCy5N0s\neSdH3mToMFnyJkPOZOkgQ85kyMU+jjXV39fGaV2RTUJm9XMAYjBN+J+rtXEaypJwZitrYmKbRMLY\nJL2HoYkITUzoRAROEt7KTkjJiSi6IQU3pOQl+0MnfQ+V90Uj52Bku7quaVevbWBCSk5A0SlRMmVc\nzyPnJ0GrErhybo6slyXv5qsBLOflyHv5+ttunqyXJeeO3l97Ts85pP4eExGZlll5KbDZWjVXVl1R\nCHsegZ0Pws4HYMeD6faDUBqZaZ1MN8xfBfOPgQXHwPxjk+15R4Hrz+hHxnFMoVBgaGiIwcFBITi0\n7QAAHiZJREFUhoaGRi1j9wVBUPc82Wx20kH5lYH5mUwG3/PxXQ/f9ZJAFqVhLLLYKE7CWTTy2kZ2\n9L44To+NvLaRxYYxtlgiHi4SF4rYYpm4WErW5QBbDpMlCJPeyDCq+ZkkIdBxwbjguJh0jeOBcTGV\nbccF42BM4wfox8YSuhGBGxK4SegquQElU6bglCiYIgVTZMgUGKbAAEOUnDJFp0TRKVM06brmdeV4\nyQRUrijX9pSNXVdCXL1wNlFQm/Bc6oUTkTaiYNVkrZgra1qshcFtsOOBkaC14wHY+RAMPD7SzvFg\n3tFp4ErD1vxjkhCW7W5IKeVyedohbHh4eMrzua6L7/v4vp+ErjrryY5N1sb3/Rn9o26DgGhggLi/\nn6i/n6h/gHign6ivn2ign7h/gKi/P9lX3R4iGhwiGhzGxLYaykaCWGXbw+Q7cTq7kqWjC6ejE5Pv\nxPg5jJfFuFlwM0k4Nj7GeFhcsC5YB2KDjQ02JLlsHc7s//cWS+RZIjcm9JLwVnZDyk5Q7TUrmHRx\nCgxRYMgUGLRDDDLEkCkkgc2UKTplSmPCWzxBV2Q1bM2wF853fDJuhoyTIeNm8F2fjJMZ2Z8eq+yv\n7Kt9n+u071QnItJ8ClYt1Iq5svZJsR92PZT2bqVha8cDsHtjchmsomfpSNBaUAlcx0LXwgM2oWkU\nRQwPD1eDVrlcplwuEwTBuHW9ffWOzdSBCm2ZTGbU/GTWWmyxmISt/n6igQGivj7igYHJg1l/P3Gp\nhC0WiUslmOnv6GYwbgb8PE5HN06+E5Pvwsl2YrIdmEwHJpvHeDmMnwMvm7R3M2B8cDwMlQDnVG+m\nIDIzHisYu5bYS8Jb6MWEbkjgRpTdkJJTTnrfnJGet+G0122QYYbMMAN2iAGG6Lf9FE057WkrUzJl\nAhMy03tOAFzjknEzeI43YfiqDW21+zzHG/V6Ou+rhj+FPZG2pGDVJqIw4LF71zV1rqz9FpZhz8Oj\ne7l2PpgEr/LgSLtc70jImr9qpKdrznJw22scjrV2whA2WTCbbniLovqD/ifiOE5DQpvnebium6wB\nE0a4cYQbBJgwxA0CbKmMLaWXOEvF5BJnqUhcLCaXP4uFZF1KXlfbFAvpujgqwNlCIQlyYTjJL+gl\nwa2mN814WXCzGD+XhLhqgMsni5cDP2lv3Aw4frIYj+Q+GyftgZtZSrIAvsH6YD2IPYi9NMi5MaEb\nE7pR9TJq2UkupZZNUA101cuilCk4RYYpMGyKDFNk2A4zaIYpxSWCOKAclSnHZcpRmSAKCO0kn9MM\nVcJeJWyNXU8W9rJulg6/g7yXJ+/l6fCS7XH7/JFjuhwrMkLBqg3FccTjD/yx+qDogZ07MI7DEcef\nyKq1p7HylFPpmndYq8ucmLXQ//j4MVw7H0wuN1a4GThsZTqW69iRnq7DVkGmDXrqDoAoivY5mE2n\nzf5wHGckfM1wPWkbY3BiixtHOHGME0U4YYibrk0Q4gblZF0uYUplnHIJSqWRIFco1gl0ldA3OtBV\ng5xxkoDmZarrJLzV7HP9kR45L5sGuBzGz2P8bBry0uOujzFpgDMuSYjbpy6ukUmI/ZE57vCdJNhN\nFuq85OaHwAkpuwFlJ0hugqj0vpkSBadEyaahrRLcKkEuDXNBHBBEY8Jd2qYYFSkEhRkFPcc444JY\nbRibcJ8/RXjz8up9k4OOglWbs9ay/eENPHTLjTx0843sfnwzAD0LFpLv7qWjp4d8Ty/5nl460iU/\nat2Dn8u3z1+ThT1Jj1btGK6dDyQD6m3NhKa9R9ZcTjxmZExX5/yWld7urLWEYVg3dEVRRBiG+72e\nTptGMcbsU9BzHYdK7HEsuFicOMaN42qwc9Ng50cRfhjhhwFeuYxfKuOUilDpvavtiSuV6uwrJ3PE\npZc+6669Cfa7mWTsWyaf9Nh52ZG2Nb1wxuxjr65nMH4S4hzfxWS9UU+XqD5xwq+ZO893k/d4DpFj\nk+BmAkqUKZGMjStSYpgChbjIkB1mmGEG42GGo2EKYYHhMFkXwgKFYMzrdJmJrJsdH9r8ycPYlPv8\nDjJOpn2+F+WQomB1kKnMlbVry2MU+vsY7u+j0N9Pob+PMCjXfY/r+0nI6u4l39MzJnz11BxL9mU7\nO5v/hRMUkzFbtWO4KpcVa7+I8/PGj+FacEwSxPSIm5az1hLHccNCWiPOMdPvLmMM2Wx2WksulyOT\nyZAxJlmATBzjWYsfhiOXV4tFbKlUvaw6+lJrJahVjpVGX1ItFqtPjiCM02GNXtrrNt0wl60Jc5UQ\nN/pS6n7fgeoYTDo/oPGSHrjKdrIkx/AcYidOeuKcmNAJkylITEBgkrtTSwTJ3aeUKFCkYAsM22I1\nxA3FwwzaIQbiQfrjAQaiQfqjAQKm32vrGGfavWt5Lz/lWLgp96XbzgG401fai4LVIcJaS1Aqjgpb\nybomfA301ezrJyjW/8vRcV3y3T3VHq989/hesNrXua4unAPVXR/H0PfYSM/WzgdHBtEP7xpp5+WS\ny4qdC5K7E3M9kO2t2e6ZeL+fP2CD66X1Jgp6ld68YrFIqVSa0TLdy65ThbLphLdMJoMz5o8GGwQj\noat2LNy4fWPGydVeTq291FpMt4OYeLhEPFQgHi4lU4S4PsbxwfWSteMl+9wMpiO9+zTfiZPrGLmU\nmkmDm5cZeY9Jx7+R3Hma3HUaVxeiBvwb4yaTP+MarJve7OAml1MjJ5lHLnSSud3KJqgulRA3EuSK\nDNsCQ3a4ugQmJDAhYboOTFTdHllH1bWd4A7WUTc5VG5CqB33NvaOVMefsk3d9425uWGifZ7jqeeu\nwRSsZrGwXK4GrUJ/H8MD/TVBLAlfhf6+aiArDQ3VP5Ex5Lu6R/WCVUJX7eXKkX09uF4DBq0P7Ro/\naL6wO7mLsdQPpYHRg+gn4nhJ0Mp2J6Erl4awbE+d7Z76+zPdbTcQXw6cKIr2OZSVSqXq+6Yb0DKZ\nzIwDWb1lbECbjLUWWygk04L09RP391W3o/6+5M7Uvsq0IX3E1e1+4r4+7GS/mzE4XV24PT04vT24\nPb3Jds8cnO5e3K45OF3dOJ3dyR2oHZ042fTu00w+mQi5EsqCGKJkbceENRvGEKTbkU3b1ByrniN5\nH3Hj/p2zJgl1sQOxExM7lsiJRxaThLyRMBZWw1uZNPQRUCbpwatcji1RpkCRgKD63qAm2NUGv8AE\no84/UfCbqNdtOnegVl77rp+sa26UGLs9ZZua476bPPLsYAx9ClYybVEYUqgXvgb66vaUFQYHkoHs\ndWQ7O5Og1V2/F6yje/TYsX2e2yuOkoBV6k8DV+12/wT7B6DUN3p/PI2xQ37n6MA1qresp07PWZ39\n6j2bVeI43udQVruUy/WHAYxVG9AmWnzfHz1mLV3Gvp6qjRME2MHBZEqQyrxtlYA2QSCrvLZT/D4j\noSwJZGMDmtvbg9OTvu7tGWnb3Y2Z5I86G9sxgasmkEVx0stWmVA4TPbZdB+VCYSrx+zo91TaVY/V\ntAvTcDju/LZxvXljxI5NlyToxcYSOlHSs1d9kkM4qqcuCXshZVMeFfqSdXkkIBJVg1xY07tXe97a\nkDdRm4g4CVs188tVw1e6HhUA64SzUb1+aduxQa92/6g20wiL9YLfrHxWoOwb1/PomjuPrrnzptU+\njiOKg4N1e8FqL1P2bdvK1of+RGGgP5nxvA4/lx8dviYYL5bJ5fEyGbxMFtf3k3WuF5Ofs++/uLUQ\nFuuEsTR4jdruG72/f8vIdjBBj18tx5vksuXYXrRJetd0J9VBwXEc8vk8+Xx+v84TxzHlcnmfQtnA\nwMCo141UuQGhbhA7bB7eooX1jwMminGiECcMcYIQJyhjymVMqYQplnCKBUyhAMMFzN49mM2bYXAQ\nUyolNytE6V2o6VJ57eVyeN1d+N09eHVC2LhA1tOD29uLk89jstl9fhTZ/rDW1jwJoiaAhSNPiBgX\nzsK0bU04s2PC2qhzVELhNM9BFNdsH7iOl8jEowJgZGIiJyIytQGwJgQSEpiAsgkJSHr9AoL0Im/A\nYLV3cOqwN+410bh91jUY12BcJ3nihzP9J5MoWMmMOY5bvVNxOpND2DimODw0KniNGys20M/A7l1s\nf2Qjhf4+ounchWZMNWx5mQx+JoPrZ9J96X6/ZjvjV9sm+7M1bdPFn4uXWYTXm8FbMP6449bpwo5C\nKA9Mr4esdrtvM2yvCXR2GnNhZbpqes660gHKXjLA3/FGFlP7uvZxOu6Y/SOP3RnXdkbnqmmz3+dy\n1buXchyHXC5HLpfbr/PEcUwURdWlMibtQL2uvQmhVCpN+Z5RMplk6e3d98/N2pHwtWc3zs4dOFE8\nMjXImFDmRhGOtXgYXJMsnmNwHQfPcXFdJ5l+pDINie+nSwYv4+NnMnjZHH42g5fL4edyyTqfx8nm\nMNkMTi6HyWZxstkkyKXb+D6Olzzjtd0kz6+t85iw2t652oAW2/E9e1FtmKs5R93jdc4bjQ2Z489R\naU/9v9/3W/JsWMvR/HJa7RWs5IAzjpOM1erqZt7hy6Zsb62lXCiMCl1BqUhYLqdLafR2EIzfVy5T\n6O8nCuq8Z4K7LKf1uxhndHDLZJIv2FEhbWQ7CXpzkrCWyeDnM3i9Y4+nQc+xeJTxbRkvLuLGw8k6\nGBzTi9aXhrOB5FJmHCeTusbD6YOjw+RSaVzZDpMpLyrblfdUj0XTuyTaTJUANmFImyQkOj542eTG\nh+o6k65zdY5Nta6zz/EOqvDnOE51Utp2M/aO05mGt+m8DoOAsFgkLJUISyWicjnZF4SU44gojoni\nmNBaIqiuJ/1vbC2US8kyCSeKcNOlGuLSkFfdZ8HD4hiDVwl2lWlJHAfPdXFdD89z8bwk1PkZH9fz\nk++VbPq9k8uSSUOdl8uRyefx8nmcXC4JdLkcJpMZCXdTPMvWmOSGAdN+ma+u2ufGjuqli0f32I0O\niTXPma17vCbsTZOClbQdYwzZjg6yHR3MWbyk4ee31hIGZaJybSCbXlgLq0Gttu3I9nB//7h9YblM\ntB+TfBrHGR/c/F68zAJcP/lydT2vZnvktZP+de24yWsv3Tf6Pelr10kv3Ti4Driek8wd5YDrGFzX\n4BiL5zoY4slD2rhgV/t6hoFvWueKRtpHZQhLyeXbsJRc7h21LoyeW23f/qOMBC03u2/hbFrvydQ/\n5mYOqmA3mdrLiu2kNvDVW2p75YJSiaBYJCwWCUqlJMSVywTV74mAMAyS75X0vUEl/EURZWspVIJd\nGuoiIDSm/n/nKIRCCExv7jBnTLCrbscxrrW4kC4mufnScXBM0mNnjKkGc9d1R9aVMXeVtefhel51\n7Xp+9TvI9X3cTCZd+7h+BjebGdmXyyXb2SxeNoubTYZ8zOSmDADjmGR6EL+1U18oWMmsY4zBz2Tx\nM1mgqyk/08Zx8sVaL8SVy0TlMkG93rVyiWhMyAtq2sRhSLkwTBSERGGQLiFRMHp7ojFu+8pJv0g9\nr+aLsxrYfFzfGx360i/ZyvbIezKjj406z5iw6Nc/j1fz5e15/vTGykRhErSichq4xoavydZj95Xq\ntx3eNfE54v2bTR+YeZDz85DpTO50zXQml5LHva4snclyiIS3fVEb+LLZbEtqmCjcVacWKZeTIFco\nEBQKhJWAV9mffl9EQRLqgjAkCkPCam9eTBjHRDamGMejQl1cuxiDTZeqKEqWA/sB4FiLsRYntjik\nr0km+Bi1GIODIclWaRhMw6FTs7iOk3x/1QmGjuvW/PHpjf5u86d/o5WClUgTGMepCXPdTf/5cRwR\nhSFxNXTVrCuBLAiSgBYFddvEYUgYJOtRAa7eedLX5WKBaIr3xA3+cjaOMz7o1fTgjQ9uHo6XPHfR\nGRXc0uOuh+t34nq9o/8Cz489x9igWT8sGsdJetjGhq1qyJtuoJtonS6FvaPfGwxDaRCi6Q5kN2nA\n6poghM0gpFVeu+13KbKdtUO4qxXH8bgliqKR/0+nl1mjUpmoXCIqB8nrICAK0nU5IK75nojTsBdF\nEXH63NU47c2zUXKZNo6SQe1xHBOnYTOKbbJt033WJiHQQoCthkKLITYQY7COIXYcrEnWtdu2gTcv\nKFiJzAKO4+JkXMi0/st5LBvHaciqDWdhNaTFYUiYBrsoCEZt1w+II9vVL+1RbULi9LJM0tuXHotG\nn7PycyeaWmRf1Q1+fk0P3LheQG9MOOzA9XrG9+5lay7/ev64kOi4LsZxcYhx4hImKuOEBUxUwokK\nmLCYvA6HcaJhTFDACYZwwmFMMIQTDGKCYczAE5jdQ0lIKw+lc8pN8zNys5MHr1GvpwhpmS5NY9Jk\nlV6fg1FyB2aEDQJsuZysa5a4XCYulwlLpWSdXklIliQg/ts0f5aClYi0VDKGLLPvc5odQNbaNPgF\n4y63jurBq4az8b1943rrwtHBb9Q5opEgGJbLlIaGkr/u65zjQAW/yRmgB+PMwXEcTDrmxknH4hjH\n4BiDSYa64Bgw2HQd4xiLIcbYGIcIx0YYSjh2GMc+jrEhxobJfpNe+qmew46sa87reD7GTYKk42WS\n5zP6mWTbz+L4ORw/i/FzOH4Ok+nAyVTWHTh+BsfP4vpZnEwWx8viZrI4mTxOJovr53DSHsjkEpKH\n47lJWNX0JwcNYwx4XjLf2b5Og/Kav5pWMwUrEZEJGGOS8OC6+O3X2TcS/Mb22o0Ndmlos3FMHEfp\nOsZG0ah15ZhNL79U1nFlX+W9UYyNK/sj4igetT26bc3PS99Te25bc+5qLVGEjZPak3OG2Gj0ueI4\nTscg2TrZspwu03hCw/79F8A16WwjBlwHHMek2wbHNem4nnTtOjiuk/QguzWDwNMeRccdGduTBLhk\noHflhhPHz+B6mTQMJuHR9TM4mVwSDNPFzeZwvEx13FAlFI7/Oa7C4QGgYCUicpAaFfxaXUwLWWux\nNh4XGKtBLAyxpUFscYC4NERcGsCWhoiLg8RhmTgoEQVl4jAgDspJb2JlHYbEUZj2HEbEURpYo5g4\nSscGpaEvSgNmHCXTN8SxJYoscWSJwyQEltMn7MQxRNYQW5OsMcTWqe6LrSGmGZfdLK4xOE5NMHSS\nXkdjTHJXYNJFWN02TnK3omNGeiqTdfraOOk+J9lX7dGsbFf2u9XXjutCzbYxDjjp5Ws3aYeTBEGT\nXtY2rpeeo3bbGVNHTS3VGpLXlZ5Wantdx76nZv90KViJiMhBLfmHL5nLbOL+lwVNrGia4ji5QzQq\nQxSkSzndF2DDEnFQIi4XicrFZDsoEZXT/UGZqBIMw4A4CNKgGBBFQfUycuVydBwmvX+jA2GYhr/K\nQPFK76LFkswBlQTXpGfQ2mS+p9gC6b4Yk4yysyYdQJ68tpjkPelxa82Y7Zp1bdsJ3h/bZDydxVR/\nRnJ5ur1MK1gZY14IfJJkqosvWWsvG3PcpMdfBAwDr7fW3tHgWkVERA4djgNOOg9aHQaqc0y1bY+k\ntSPzye3XPHZ15qQbe64657dRgI0jbBRi4zBZV5Y4To+H2PQuQxtXlpE2cZScz9rKpexkTr240j7t\nDX33VdP7SKYMVsYYF/gM8HxgM3CrMeZH1tr7a5qdA6xKl2cAn03XIiIicqgyBlwvWVrx42lin9X/\nm95Pms5Fw7XAemvtRmttGfg2cP6YNucD/2UTNwFzjDGNnzJbREREpI1NJ1gtBR6reb053TfTNiIi\nIiKHtKbO9GWMeZMx5jZjzG07duxo5o8WEREROeCmE6y2AEfUvF6W7ptpG6y1X7DWrrHWrlmwoA3v\n0BARERHZD9MJVrcCq4wxRxljMsCrgB+NafMj4K9N4lSgz1q7tcG1ioiIiLS1KYfxW2tDY8zbgZ+T\n3PX5FWvtfcaYN6fHPwdcRzLVwnqS6RbecOBKFhEREWlP07o/0lp7HUl4qt33uZptC7ytsaWJiIiI\nHFwOzsdUi4iIiLQhBSsRERGRBlGwEhEREWkQBSsRERGRBlGwEhEREWkQBSsRERGRBlGwEhEREWkQ\nBSsRERGRBlGwEhEREWkQBSsRERGRBlGwEhEREWkQBSsRERGRBlGwEhEREWkQBSsRERGRBlGwEhER\nEWkQBSsRERGRBjHW2tb8YGMGgAda8sPb23xgZ6uLaEP6XMbTZ1KfPpf69LnUp89lPH0m9R1rre2e\nqpHXjEom8IC1dk0Lf35bMsbcps9lPH0u4+kzqU+fS336XOrT5zKePpP6jDG3TaedLgWKiIiINIiC\nlYiIiEiDtDJYfaGFP7ud6XOpT5/LePpM6tPnUp8+l/r0uYynz6S+aX0uLRu8LiIiInKo0aVAERER\nkQZpSbAyxrzQGPOAMWa9MebSVtTQbowxXzHGbDfG3NvqWtqFMeYIY8yvjTH3G2PuM8b8Q6tragfG\nmJwx5hZjzN3p5/Jvra6pXRhjXGPMncaYn7S6lnZijHnEGHOPMeau6d7ZdKgzxswxxnzPGPMnY8wf\njTHPbHVNrWaMOTb930hl6TfGXNTqutqBMead6fftvcaY/zHG5CZs2+xLgcYYF3gQeD6wGbgVeLW1\n9v6mFtJmjDHPBgaB/7LWntDqetqBMWYJsMRae4cxphu4HXiJ/rdiDNBprR00xvjADcA/WGtvanFp\nLWeMuRhYA/RYa89rdT3twhjzCLDGWqu5iVLGmK8Dv7fWfskYkwE6rLV7W11Xu0j/rd4CPMNa+2ir\n62klY8xSku/Z4621BWPMVcB11tqv1Wvfih6rtcB6a+1Ga20Z+DZwfgvqaCvW2t8Bu1tdRzux1m61\n1t6Rbg8AfwSWtraq1rOJwfSlny6zfrCkMWYZcC7wpVbXIu3NGNMLPBv4MoC1tqxQNc7zgA2zPVTV\n8IC8McYDOoDHJ2rYimC1FHis5vVm9I+lTMEY8yTgZODm1lbSHtJLXncB24FfWmv1ucAngPcCcasL\naUMW+JUx5nZjzJtaXUwbOArYAXw1vXT8JWNMZ6uLajOvAv6n1UW0A2vtFuCjwCZgK9Bnrf3FRO01\neF3anjGmC/g+cJG1tr/V9bQDa21krX0qsAxYa4yZ1ZePjTHnAduttbe3upY29az0fy/nAG9Lhx7M\nZh7wNOCz1tqTgSFA431T6aXRFwPfbXUt7cAYM5fkytpRwOFApzHmgonatyJYbQGOqHm9LN0nMk46\nhuj7wDettT9odT3tJr188Wvgha2upcVOB16cjiX6NnCmMea/W1tS+0j/4sZaux24mmRIxmy2Gdhc\n09P7PZKgJYlzgDustdtaXUibOAt42Fq7w1obAD8ATpuocSuC1a3AKmPMUWkqfhXwoxbUIW0uHaT9\nZeCP1torWl1PuzDGLDDGzEm38yQ3gvyptVW1lrX2H621y6y1TyL5Tvlfa+2Ef1HOJsaYzvTmD9LL\nXS8AZvXdx9baJ4DHjDHHprueB8zqm2LGeDW6DFhrE3CqMaYj/XfpeSRjfutq+kOYrbWhMebtwM8B\nF/iKtfa+ZtfRbowx/wP8GTDfGLMZ+Bdr7ZdbW1XLnQ68FrgnHU8E8D5r7XUtrKkdLAG+nt614wBX\nWWs1vYBMZBFwdfLvAR7wLWvtz1pbUlt4B/DN9A/8jcAbWlxPW0jD9/OBv291Le3CWnuzMeZ7wB1A\nCNzJJLOwa+Z1ERERkQbR4HURERGRBlGwEhEREWkQBSsRERGRBlGwEhEREWkQBSsRERGRBlGwEpFZ\nxxjzZ8YYTVEhIg2nYCUiIiLSIApWItK2jDEXGGNuMcbcZYz5fPrw6UFjzMeNMfcZY643xixI2z7V\nGHOTMWadMebq9PleGGNWGmN+ZYy52xhzhzFmRXr6LmPM94wxfzLGfDOdUVlEZL8oWIlIWzLGPBl4\nJXB6+gDhCHgN0AncZq1dDfwW+Jf0Lf8FXGKtPQm4p2b/N4HPWGufQvJ8r63p/pOBi4DjgaNJZvoX\nEdkvTX+kjYjIND0PeDpwa9qZlAe2AzHwnbTNfwM/MMb0AnOstb9N938d+G76jLyl1tqrAay1RYD0\nfLdYazenr+8CngTccOB/LRE5lClYiUi7MsDXrbX/OGqnMf80pt2+PperVLMdoe9DEWkAXQoUkXZ1\nPfByY8xCAGPMPGPMcpLvrZenbf4KuMFa2wfsMcacke5/LfBba+0AsNkY85L0HFljTEdTfwsRmVX0\nF5qItCVr7f3GmPcDvzDGOEAAvA0YAtamx7aTjMMCeB3wuTQ4bQTekO5/LfB5Y8wH0nP8ZRN/DRGZ\nZYy1+9qLLiLSfMaYQWttV6vrEBGpR5cCRURERBpEPVYiIiIiDaIeKxEREZEGUbASERERaRAFKxER\nEZEGUbASERERaRAFKxEREZEGUbASERERaZD/D2gbNCDBRGlRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1e8c365e978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "%matplotlib inline\n",
    "data = pd.DataFrame.from_csv('train_results/logBackup.csv')\n",
    "data.dropna(how=\"all\", inplace=True) \n",
    "data.plot(figsize=(10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_model.load_weights(\"train_results/weights-05.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import cv2\n",
    "# for i in range(0,100):    \n",
    "out = train_model.predict([x_train[0].reshape(-1,28,28,1),letter[0].reshape(-1,10)])\n",
    "#     print(str(i)+\"->\"+str(np.argmax(out[0])))\n",
    "    \n",
    "# data = np.zeros((10,28,28,1))\n",
    "# data[0] = x_train[2]\n",
    "# data[5] = x_train[0]\n",
    "# data[2] = x_train[6]\n",
    "# data[7] = x_train[11]\n",
    "# data[4] = x_train[16]\n",
    "# data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.argmax(out[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "cv2.imshow('img',out[1][0])\n",
    "cv2.waitKey(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "plt.imshow(out[1][0].reshape(28,28), interpolation='nearest', cmap='gray')\n",
    "plt.xticks([]), plt.yticks([]) \n",
    "plt.savefig('images/reconstruct_capsulenet/caps_.png')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "out = train_model.predict([x_train[0].reshape(-1,28,28,1),letter[8].reshape(-1,10)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.argmax(out[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "letter[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "letter = np.zeros((10,10))\n",
    "for i in range(10):\n",
    "        letter[i][i] = 1\n",
    "letter[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import os, sys\n",
    "import numpy as np\n",
    "os.chdir('D:/MNIST-GAN-BASIC/images/reconstruct_capsulenet/Selected/epoch_8')\n",
    "list_im =  ['0.png', '1.png', '6.png','7.png','9.png']\n",
    "imgs    = [ Image.open(i) for i in list_im ]\n",
    "# pick the image which is the smallest, and resize the others to match it (can be arbitrary image shape here)\n",
    "min_shape = sorted( [(np.sum(i.size), i.size ) for i in imgs])[0][1]\n",
    "imgs_comb = np.hstack( (np.asarray( i.resize(min_shape) ) for i in imgs ) )\n",
    "\n",
    "# save that beautiful picture\n",
    "imgs_comb = Image.fromarray( imgs_comb)\n",
    "imgs_comb.save( 'D:/MNIST-GAN-BASIC/images/reconstruct_capsulenet/Selected/epoch_8/epoch.jpg' )    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import keras\n",
    "# counter = 995\n",
    "# generator = keras.models.load_model(\"models/gan_cnn/gan_generator_epoch_50.h5\")\n",
    "# examples=1\n",
    "# noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "\n",
    "# generatedImages = generator.predict(noise)\n",
    "# generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "# # generatedImages.shape\n",
    "# pl.subplot(2, 2, 1)\n",
    "# pl.figure(figsize=(2,2))\n",
    "# pl.imshow(generatedImages[0], cmap = 'gray', interpolation = 'bicubic')\n",
    "# pl.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis    \n",
    "\n",
    "# counter=counter-1\n",
    "# pl.tight_layout()\n",
    "# pl.savefig('images/gan_cnn/digits/epoch_50/random_%d.png' % counter)\n",
    "# randomDim = 100\n",
    "# examples=1\n",
    "# noise = np.random.normal(0, 1, size=[examples, randomDim])\n",
    "\n",
    "# generatedImages = generator.predict(noise)\n",
    "# generatedImages = generatedImages.reshape(examples, 28, 28)\n",
    "# # generatedImages.shape\n",
    "# pl.subplot(2, 2, 1)\n",
    "# pl.figure(figsize=(2,2))\n",
    "# pl.imshow(generatedImages[0], cmap = 'gray', interpolation = 'bicubic')\n",
    "# pl.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis    \n",
    "\n",
    "# counter=counter-1\n",
    "# pl.tight_layout()\n",
    "# pl.savefig('images/gan_cnn/digits/epoch_50/random_%d.png' % counter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from PIL import Image\n",
    "# import os, sys\n",
    "# import numpy as np\n",
    "\n",
    "# os.chdir('/images/gan_cnn/epoch_50')\n",
    "# list_im =  ['0.png', '3.png', '6.png','7.png','8.png']\n",
    "# imgs    = [ Image.open(i) for i in list_im ]\n",
    "# # pick the image which is the smallest, and resize the others to match it (can be arbitrary image shape here)\n",
    "# min_shape = sorted( [(np.sum(i.size), i.size ) for i in imgs])[0][1]\n",
    "# imgs_comb = np.hstack( (np.asarray( i.resize(min_shape) ) for i in imgs ) )\n",
    "\n",
    "# # save that beautiful picture\n",
    "# imgs_comb = Image.fromarray( imgs_comb)\n",
    "# imgs_comb.save( '/images/gan_cnn/digits/epoch_50.jpg' )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3 (tf-gpu)",
   "language": "python",
   "name": "tf-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}