SerialGANWithMultipleLayers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 65
    },
    "colab_type": "code",
    "id": "_iaQa45ebnmi",
    "outputId": "cb1ce274-5607-416a-f5e8-84de3a2b01f2"
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import collections\n",
    "import gzip\n",
    "import os\n",
    "\n",
    "import numpy\n",
    "from six.moves import urllib\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "\n",
    "from tensorflow.python.framework import dtypes\n",
    "from tensorflow.python.framework import random_seed\n",
    "from tensorflow.python.platform import gfile\n",
    "from tensorflow.python.util.deprecation import deprecated\n",
    "\n",
    "_Datasets = collections.namedtuple('_Datasets', ['train', 'validation', 'test'])\n",
    "\n",
    "# CVDF mirror of http://yann.lecun.com/exdb/mnist/\n",
    "DEFAULT_SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/'\n",
    "\n",
    "\n",
    "def _read32(bytestream):\n",
    "  dt = numpy.dtype(numpy.uint32).newbyteorder('>')\n",
    "  return numpy.frombuffer(bytestream.read(4), dtype=dt)[0]\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use tf.data to implement this functionality.')\n",
    "def _extract_images(f):\n",
    "  \"\"\"Extract the images into a 4D uint8 numpy array [index, y, x, depth].\n",
    "  Args:\n",
    "    f: A file object that can be passed into a gzip reader.\n",
    "  Returns:\n",
    "    data: A 4D uint8 numpy array [index, y, x, depth].\n",
    "  Raises:\n",
    "    ValueError: If the bytestream does not start with 2051.\n",
    "  \"\"\"\n",
    "  print('Extracting', f.name)\n",
    "  with gzip.GzipFile(fileobj=f) as bytestream:\n",
    "    magic = _read32(bytestream)\n",
    "    if magic != 2051:\n",
    "      raise ValueError('Invalid magic number %d in MNIST image file: %s' %\n",
    "                       (magic, f.name))\n",
    "    num_images = _read32(bytestream)\n",
    "    rows = _read32(bytestream)\n",
    "    cols = _read32(bytestream)\n",
    "    buf = bytestream.read(rows * cols * num_images)\n",
    "    data = numpy.frombuffer(buf, dtype=numpy.uint8)\n",
    "    data = data.reshape(num_images, rows, cols, 1)\n",
    "    return data\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use tf.one_hot on tensors.')\n",
    "def _dense_to_one_hot(labels_dense, num_classes):\n",
    "  \"\"\"Convert class labels from scalars to one-hot vectors.\"\"\"\n",
    "  num_labels = labels_dense.shape[0]\n",
    "  index_offset = numpy.arange(num_labels) * num_classes\n",
    "  labels_one_hot = numpy.zeros((num_labels, num_classes))\n",
    "  labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
    "  return labels_one_hot\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use tf.data to implement this functionality.')\n",
    "def _extract_labels(f, one_hot=False, num_classes=10):\n",
    "  \"\"\"Extract the labels into a 1D uint8 numpy array [index].\n",
    "  Args:\n",
    "    f: A file object that can be passed into a gzip reader.\n",
    "    one_hot: Does one hot encoding for the result.\n",
    "    num_classes: Number of classes for the one hot encoding.\n",
    "  Returns:\n",
    "    labels: a 1D uint8 numpy array.\n",
    "  Raises:\n",
    "    ValueError: If the bystream doesn't start with 2049.\n",
    "  \"\"\"\n",
    "  print('Extracting', f.name)\n",
    "  with gzip.GzipFile(fileobj=f) as bytestream:\n",
    "    magic = _read32(bytestream)\n",
    "    if magic != 2049:\n",
    "      raise ValueError('Invalid magic number %d in MNIST label file: %s' %\n",
    "                       (magic, f.name))\n",
    "    num_items = _read32(bytestream)\n",
    "    buf = bytestream.read(num_items)\n",
    "    labels = numpy.frombuffer(buf, dtype=numpy.uint8)\n",
    "    if one_hot:\n",
    "      return _dense_to_one_hot(labels, num_classes)\n",
    "    return labels\n",
    "\n",
    "\n",
    "class _DataSet(object):\n",
    "  \"\"\"Container class for a _DataSet (deprecated).\n",
    "  THIS CLASS IS DEPRECATED.\n",
    "  \"\"\"\n",
    "\n",
    "  @deprecated(None, 'Please use alternatives such as official/mnist/_DataSet.py'\n",
    "              ' from tensorflow/models.')\n",
    "  def __init__(self,\n",
    "               images,\n",
    "               labels,\n",
    "               fake_data=False,\n",
    "               one_hot=False,\n",
    "               dtype=dtypes.float32,\n",
    "               reshape=True,\n",
    "               seed=None):\n",
    "    \"\"\"Construct a _DataSet.\n",
    "    one_hot arg is used only if fake_data is true.  `dtype` can be either\n",
    "    `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into\n",
    "    `[0, 1]`.  Seed arg provides for convenient deterministic testing.\n",
    "    Args:\n",
    "      images: The images\n",
    "      labels: The labels\n",
    "      fake_data: Ignore inages and labels, use fake data.\n",
    "      one_hot: Bool, return the labels as one hot vectors (if True) or ints (if\n",
    "        False).\n",
    "      dtype: Output image dtype. One of [uint8, float32]. `uint8` output has\n",
    "        range [0,255]. float32 output has range [0,1].\n",
    "      reshape: Bool. If True returned images are returned flattened to vectors.\n",
    "      seed: The random seed to use.\n",
    "    \"\"\"\n",
    "    seed1, seed2 = random_seed.get_seed(seed)\n",
    "    # If op level seed is not set, use whatever graph level seed is returned\n",
    "    numpy.random.seed(seed1 if seed is None else seed2)\n",
    "    dtype = dtypes.as_dtype(dtype).base_dtype\n",
    "    if dtype not in (dtypes.uint8, dtypes.float32):\n",
    "      raise TypeError('Invalid image dtype %r, expected uint8 or float32' %\n",
    "                      dtype)\n",
    "    if fake_data:\n",
    "      self._num_examples = 10000\n",
    "      self.one_hot = one_hot\n",
    "    else:\n",
    "      assert images.shape[0] == labels.shape[0], (\n",
    "          'images.shape: %s labels.shape: %s' % (images.shape, labels.shape))\n",
    "      self._num_examples = images.shape[0]\n",
    "\n",
    "      # Convert shape from [num examples, rows, columns, depth]\n",
    "      # to [num examples, rows*columns] (assuming depth == 1)\n",
    "      if reshape:\n",
    "        assert images.shape[3] == 1\n",
    "        images = images.reshape(images.shape[0],\n",
    "                                images.shape[1] * images.shape[2])\n",
    "      if dtype == dtypes.float32:\n",
    "        # Convert from [0, 255] -> [0.0, 1.0].\n",
    "        images = images.astype(numpy.float32)\n",
    "        images = numpy.multiply(images, 1.0 / 255.0)\n",
    "    self._images = images\n",
    "    self._labels = labels\n",
    "    self._epochs_completed = 0\n",
    "    self._index_in_epoch = 0\n",
    "\n",
    "  @property\n",
    "  def images(self):\n",
    "    return self._images\n",
    "\n",
    "  @property\n",
    "  def labels(self):\n",
    "    return self._labels\n",
    "\n",
    "  @property\n",
    "  def num_examples(self):\n",
    "    return self._num_examples\n",
    "\n",
    "  @property\n",
    "  def epochs_completed(self):\n",
    "    return self._epochs_completed\n",
    "\n",
    "  def next_batch(self, batch_size, fake_data=False, shuffle=True):\n",
    "    \"\"\"Return the next `batch_size` examples from this data set.\"\"\"\n",
    "    if fake_data:\n",
    "      fake_image = [1] * 784\n",
    "      if self.one_hot:\n",
    "        fake_label = [1] + [0] * 9\n",
    "      else:\n",
    "        fake_label = 0\n",
    "      return [fake_image for _ in xrange(batch_size)\n",
    "             ], [fake_label for _ in xrange(batch_size)]\n",
    "    start = self._index_in_epoch\n",
    "    # Shuffle for the first epoch\n",
    "    if self._epochs_completed == 0 and start == 0 and shuffle:\n",
    "      perm0 = numpy.arange(self._num_examples)\n",
    "      numpy.random.shuffle(perm0)\n",
    "      self._images = self.images[perm0]\n",
    "      self._labels = self.labels[perm0]\n",
    "    # Go to the next epoch\n",
    "    if start + batch_size > self._num_examples:\n",
    "      # Finished epoch\n",
    "      self._epochs_completed += 1\n",
    "      # Get the rest examples in this epoch\n",
    "      rest_num_examples = self._num_examples - start\n",
    "      images_rest_part = self._images[start:self._num_examples]\n",
    "      labels_rest_part = self._labels[start:self._num_examples]\n",
    "      # Shuffle the data\n",
    "      if shuffle:\n",
    "        perm = numpy.arange(self._num_examples)\n",
    "        numpy.random.shuffle(perm)\n",
    "        self._images = self.images[perm]\n",
    "        self._labels = self.labels[perm]\n",
    "      # Start next epoch\n",
    "      start = 0\n",
    "      self._index_in_epoch = batch_size - rest_num_examples\n",
    "      end = self._index_in_epoch\n",
    "      images_new_part = self._images[start:end]\n",
    "      labels_new_part = self._labels[start:end]\n",
    "      return numpy.concatenate((images_rest_part, images_new_part),\n",
    "                               axis=0), numpy.concatenate(\n",
    "                                   (labels_rest_part, labels_new_part), axis=0)\n",
    "    else:\n",
    "      self._index_in_epoch += batch_size\n",
    "      end = self._index_in_epoch\n",
    "      return self._images[start:end], self._labels[start:end]\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please write your own downloading logic.')\n",
    "def _maybe_download(filename, work_directory, source_url):\n",
    "  \"\"\"Download the data from source url, unless it's already here.\n",
    "  Args:\n",
    "      filename: string, name of the file in the directory.\n",
    "      work_directory: string, path to working directory.\n",
    "      source_url: url to download from if file doesn't exist.\n",
    "  Returns:\n",
    "      Path to resulting file.\n",
    "  \"\"\"\n",
    "  if not gfile.Exists(work_directory):\n",
    "    gfile.MakeDirs(work_directory)\n",
    "  filepath = os.path.join(work_directory, filename)\n",
    "  if not gfile.Exists(filepath):\n",
    "    urllib.request.urlretrieve(source_url, filepath)\n",
    "    with gfile.GFile(filepath) as f:\n",
    "      size = f.size()\n",
    "    print('Successfully downloaded', filename, size, 'bytes.')\n",
    "  return filepath\n",
    "\n",
    "\n",
    "@deprecated(None, 'Please use alternatives such as:'\n",
    "            ' tensorflow_datasets.load(\\'mnist\\')')\n",
    "def read_data_sets(train_dir,\n",
    "                   fake_data=False,\n",
    "                   one_hot=False,\n",
    "                   dtype=dtypes.float32,\n",
    "                   reshape=True,\n",
    "                   validation_size=5000,\n",
    "                   seed=None,\n",
    "                   source_url=DEFAULT_SOURCE_URL):\n",
    "  if fake_data:\n",
    "\n",
    "    def fake():\n",
    "      return _DataSet([], [],\n",
    "                      fake_data=True,\n",
    "                      one_hot=one_hot,\n",
    "                      dtype=dtype,\n",
    "                      seed=seed)\n",
    "\n",
    "    train = fake()\n",
    "    validation = fake()\n",
    "    test = fake()\n",
    "    return _Datasets(train=train, validation=validation, test=test)\n",
    "\n",
    "  if not source_url:  # empty string check\n",
    "    source_url = DEFAULT_SOURCE_URL\n",
    "\n",
    "  train_images_file = 'train-images-idx3-ubyte.gz'\n",
    "  train_labels_file = 'train-labels-idx1-ubyte.gz'\n",
    "  test_images_file = 't10k-images-idx3-ubyte.gz'\n",
    "  test_labels_file = 't10k-labels-idx1-ubyte.gz'\n",
    "\n",
    "  local_file = _maybe_download(train_images_file, train_dir,\n",
    "                               source_url + train_images_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    train_images = _extract_images(f)\n",
    "\n",
    "  local_file = _maybe_download(train_labels_file, train_dir,\n",
    "                               source_url + train_labels_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    train_labels = _extract_labels(f, one_hot=one_hot)\n",
    "\n",
    "  local_file = _maybe_download(test_images_file, train_dir,\n",
    "                               source_url + test_images_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    test_images = _extract_images(f)\n",
    "\n",
    "  local_file = _maybe_download(test_labels_file, train_dir,\n",
    "                               source_url + test_labels_file)\n",
    "  with gfile.Open(local_file, 'rb') as f:\n",
    "    test_labels = _extract_labels(f, one_hot=one_hot)\n",
    "\n",
    "  if not 0 <= validation_size <= len(train_images):\n",
    "    raise ValueError(\n",
    "        'Validation size should be between 0 and {}. Received: {}.'.format(\n",
    "            len(train_images), validation_size))\n",
    "\n",
    "  validation_images = train_images[:validation_size]\n",
    "  validation_labels = train_labels[:validation_size]\n",
    "  train_images = train_images[validation_size:]\n",
    "  train_labels = train_labels[validation_size:]\n",
    "\n",
    "  options = dict(dtype=dtype, reshape=reshape, seed=seed)\n",
    "\n",
    "  train = _DataSet(train_images, train_labels, **options)\n",
    "  validation = _DataSet(validation_images, validation_labels, **options)\n",
    "  test = _DataSet(test_images, test_labels, **options)\n",
    "\n",
    "  return _Datasets(train=train, validation=validation, test=test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3BshiewAbfPU",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np,sys,time\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets\n",
    "import os\n",
    "import numpy as np,sys\n",
    "from sklearn.utils import shuffle\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZffTWt8ZbfPo"
   },
   "outputs": [],
   "source": [
    "def ReLu(x):\n",
    "    mask = (x>0) * 1.0\n",
    "    return mask *x\n",
    "def d_ReLu(x):\n",
    "    mask = (x>0) * 1.0\n",
    "    return mask \n",
    "\n",
    "def Lrelu(x):\n",
    "#    alpha = 0.0001\n",
    "#    return np.where(x > 0, x, x * alpha) \n",
    "    y1 = ((x > 0) * x)                                                 \n",
    "    y2 = ((x <= 0) * x * 0.01)                                         \n",
    "    return y1 + y2\n",
    "def d_Lrelu(x,alpha = 0.01):\n",
    "  dx = np.ones_like(x)\n",
    "  dx[x < 0] = alpha\n",
    "  return dx\n",
    "def arctan(x):\n",
    "    return np.arctan(x)\n",
    "def d_arctan(x):\n",
    "    return 1 / (1 + x ** 2)\n",
    "\n",
    "def log(x):\n",
    "    return 1 / ( 1+ np.exp(-1*x))\n",
    "def d_log(x):\n",
    "    return log(x) * (1 - log(x))\n",
    "\n",
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "def d_tanh(x):\n",
    "    return 1 - np.tanh(x) ** 2\n",
    "\n",
    "def plot(samples, title):\n",
    "    fig = plt.figure(figsize=(4, 4))\n",
    "    gs = gridspec.GridSpec(4, 4)\n",
    "    gs.update(wspace=0.05, hspace=0.05)\n",
    "\n",
    "    for i, sample in enumerate(samples):\n",
    "        ax = plt.subplot(gs[i])\n",
    "        plt.axis('off')\n",
    "        ax.set_xticklabels([])\n",
    "        ax.set_yticklabels([])\n",
    "        ax.set_aspect('equal')\n",
    "        plt.title(title)\n",
    "        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aPAKJeiAbfP4"
   },
   "outputs": [],
   "source": [
    "random_numer = 1222 #int(input(\"Please Input a Random Number to Seed\"))\n",
    "np.random.seed(random_numer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 412
    },
    "colab_type": "code",
    "id": "Ulnl3-qYbfQH",
    "outputId": "77c1acc1-c68d-49bb-cdb1-867abf11a735"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0317 16:40:27.609751 12108 deprecation.py:323] From <ipython-input-60-083e66dce198>:3: read_data_sets (from __main__) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as: tensorflow_datasets.load('mnist')\n",
      "W0317 16:40:27.612679 12108 deprecation.py:323] From <ipython-input-56-f01ec5041f04>:266: _maybe_download (from __main__) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------- Load Data ----------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0317 16:40:27.925784 12108 deprecation.py:323] From <ipython-input-56-f01ec5041f04>:268: _extract_images (from __main__) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0317 16:40:29.666369 12108 deprecation.py:323] From <ipython-input-56-f01ec5041f04>:273: _extract_labels (from __main__) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0317 16:40:29.945544 12108 deprecation.py:323] From <ipython-input-56-f01ec5041f04>:297: _DataSet.__init__ (from __main__) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/_DataSet.py from tensorflow/models.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# 1. Load Data and declare hyper\n",
    "print('--------- Load Data ----------')\n",
    "mnist = read_data_sets('MNIST_data', one_hot=False)\n",
    "temp = mnist.test\n",
    "images, labels = temp.images, temp.labels\n",
    "images, labels = shuffle(np.asarray(images),np.asarray(labels))\n",
    "num_epoch = 50\n",
    "learing_rate = 0.0002\n",
    "G_input = 100\n",
    "hidden_input,hidden_input2,hidden_input3 = 128,256,346\n",
    "hidden_input4,hidden_input5,hidden_input6 = 480,560,686\n",
    "hidden_input7,hidden_input8,hidden_input9 = 800,1020,1400\n",
    "hidden_input10 = 1800"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "id": "nh1txYT4bfQT",
    "outputId": "d6e9e4af-be67-47bb-b53f-da91a9335f89"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------- Declare Hyper Parameters ----------\n"
     ]
    }
   ],
   "source": [
    "print('--------- Declare Hyper Parameters ----------')\n",
    "# 2. Declare Weights\n",
    "D_W1 = np.random.normal(size=(784,hidden_input),scale=(1. / np.sqrt(784 / 2.)))   *0.002\n",
    "# D_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))       *0.002\n",
    "D_b1 = np.zeros(hidden_input)\n",
    "\n",
    "D_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "D_b2 = np.zeros(hidden_input2)\n",
    "\n",
    "D_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "D_b3 = np.zeros(hidden_input3)\n",
    "\n",
    "D_W4 = np.random.normal(size=(hidden_input3,1),scale=(1. / np.sqrt(hidden_input / 2.)))     *0.002\n",
    "# D_b2 = np.random.normal(size=(1),scale=(1. / np.sqrt(1 / 2.)))           *0.002\n",
    "D_b4 = np.zeros(1)\n",
    "\n",
    "\n",
    "G_W1 = np.random.normal(size=(G_input,hidden_input),scale=(1. / np.sqrt(G_input / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b1 = np.zeros(hidden_input)\n",
    "\n",
    "G_W2 = np.random.normal(size=(hidden_input,hidden_input2),scale=(1. / np.sqrt(hidden_input / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b2 = np.zeros(hidden_input2)\n",
    "\n",
    "G_W3 = np.random.normal(size=(hidden_input2,hidden_input3),scale=(1. / np.sqrt(hidden_input2 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b3 = np.zeros(hidden_input3)\n",
    "\n",
    "G_W4 = np.random.normal(size=(hidden_input3,hidden_input4),scale=(1. / np.sqrt(hidden_input3 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b4 = np.zeros(hidden_input4)\n",
    "\n",
    "G_W5 = np.random.normal(size=(hidden_input4,hidden_input5),scale=(1. / np.sqrt(hidden_input4 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b5 = np.zeros(hidden_input5)\n",
    "\n",
    "G_W6 = np.random.normal(size=(hidden_input5,hidden_input6),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b6 = np.zeros(hidden_input6)\n",
    "\n",
    "G_W7 = np.random.normal(size=(hidden_input6,hidden_input7),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b7 = np.zeros(hidden_input7)\n",
    "\n",
    "G_W8 = np.random.normal(size=(hidden_input7,hidden_input8),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b8 = np.zeros(hidden_input8)\n",
    "\n",
    "G_W9 = np.random.normal(size=(hidden_input8,hidden_input9),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b9 = np.zeros(hidden_input9)\n",
    "\n",
    "G_W10 = np.random.normal(size=(hidden_input9,hidden_input10),scale=(1. / np.sqrt(hidden_input5 / 2.)))   *0.002\n",
    "# G_b1 = np.random.normal(size=(128),scale=(1. / np.sqrt(128 / 2.)))      *0.002\n",
    "G_b10 = np.zeros(hidden_input10)\n",
    "\n",
    "G_W11 = np.random.normal(size=(hidden_input10,784),scale=(1. / np.sqrt(hidden_input6 / 2.)))  *0.002\n",
    "# G_b2 = np.random.normal(size=(784),scale=(1. / np.sqrt(784 / 2.)))      *0.002\n",
    "G_b11 = np.zeros(784)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "efhLIYTtbfQn"
   },
   "outputs": [],
   "source": [
    "# 3. For Adam Optimzier\n",
    "m1w1,m1b1 = 0,0\n",
    "m2w2,m2b2 = 0,0\n",
    "m3w3,m3b3 = 0,0\n",
    "m4w4,m4b4 = 0,0\n",
    "\n",
    "v1w1,v1b1 = 0,0\n",
    "v2w2,v2b2 = 0,0\n",
    "v3w3,v3b3 = 0,0\n",
    "v4w4,v4b4 = 0,0\n",
    "\n",
    "v5,m5 = 0,0\n",
    "v6,m6 = 0,0\n",
    "v7,m7 = 0,0\n",
    "v8,m8 = 0,0\n",
    "v9,m9 = 0,0\n",
    "v10,m10 = 0,0\n",
    "v11,m11 = 0,0\n",
    "v12,m12 = 0,0\n",
    "v13,m13 = 0,0\n",
    "v14,m14 = 0,0\n",
    "v15,m15 = 0,0\n",
    "v16,m16 = 0,0\n",
    "v17,m17 = 0,0\n",
    "v18,m18 = 0,0\n",
    "v19,m19 = 0,0\n",
    "v20,m20 = 0,0\n",
    "v21,m21 = 0,0\n",
    "v22,m22 = 0,0\n",
    "v23,m23 = 0,0\n",
    "v24,m24 = 0,0\n",
    "v25,m25 = 0,0\n",
    "v26,m26 = 0,0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iYhaQBh8bfQy"
   },
   "outputs": [],
   "source": [
    "beta_1,beta_2,eps = 0.5,0.999,0.000001\n",
    "D_list_cost = []\n",
    "G_list_cost = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 954
    },
    "colab_type": "code",
    "id": "sIXkEq74bfQ7",
    "outputId": "b365edb5-0524-46db-fdec-33e4a4d529df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------- Started Training ----------\n",
      "Current Iter:  1  Current D cost: [[1.38629436]]  Current G cost:  [[0.69325054]]\n",
      "Current Iter:  2  Current D cost: [[1.38629437]]  Current G cost:  [[0.69336041]]\n",
      "Current Iter:  3  Current D cost: [[1.38629441]]  Current G cost:  [[0.69347981]]\n",
      "Current Iter:  4  Current D cost: [[1.38629447]]  Current G cost:  [[0.69361123]]\n",
      "Current Iter:  5  Current D cost: [[1.38629458]]  Current G cost:  [[0.69375687]]\n",
      "Current Iter:  6  Current D cost: [[1.38629473]]  Current G cost:  [[0.6939188]]\n",
      "Current Iter:  7  Current D cost: [[1.38629495]]  Current G cost:  [[0.69409924]]\n",
      "Current Iter:  8  Current D cost: [[1.38629526]]  Current G cost:  [[0.69430082]]\n",
      "Current Iter:  9  Current D cost: [[1.38629565]]  Current G cost:  [[0.69452714]]\n",
      "Current Iter:  10  Current D cost: [[1.38629601]]  Current G cost:  [[0.69478723]]\n",
      "Current Iter:  11  Current D cost: [[1.38629329]]  Current G cost:  [[0.69512633]]\n",
      "Current Iter:  12  Current D cost: [[1.38626207]]  Current G cost:  [[0.69572872]]\n",
      "Current Iter:  13  Current D cost: [[1.38608176]]  Current G cost:  [[0.69706995]]\n",
      "Current Iter:  14  Current D cost: [[1.38529701]]  Current G cost:  [[0.70004933]]\n",
      "Current Iter:  15  Current D cost: [[1.38321891]]  Current G cost:  [[0.70616884]]\n",
      "Current Iter:  16  Current D cost: [[1.3800473]]  Current G cost:  [[0.71775097]]\n",
      "Current Iter:  17  Current D cost: [[1.37470088]]  Current G cost:  [[0.7381106]]\n",
      "Current Iter:  18  Current D cost: [[1.3557063]]  Current G cost:  [[0.77129509]]\n",
      "Current Iter:  19  Current D cost: [[1.35221367]]  Current G cost:  [[0.82471508]]\n",
      "Current Iter:  20  Current D cost: [[1.30261628]]  Current G cost:  [[0.90492039]]\n",
      "Current Iter:  21  Current D cost: [[1.27441207]]  Current G cost:  [[1.02503139]]\n",
      "Current Iter:  22  Current D cost: [[1.23255515]]  Current G cost:  [[1.19959498]]\n",
      "Current Iter:  23  Current D cost: [[1.10858213]]  Current G cost:  [[1.4329475]]\n",
      "Current Iter:  24  Current D cost: [[1.03611259]]  Current G cost:  [[1.73380883]]\n",
      "Current Iter:  25  Current D cost: [[0.98188238]]  Current G cost:  [[2.10109556]]\n",
      "Current Iter:  26  Current D cost: [[1.26197402]]  Current G cost:  [[2.57420139]]\n",
      "Current Iter:  27  Current D cost: [[1.25379967]]  Current G cost:  [[3.10848326]]\n",
      "Current Iter:  28  Current D cost: [[1.94175733]]  Current G cost:  [[3.66949242]]\n",
      "Current Iter:  29  Current D cost: [[1.01367324]]  Current G cost:  [[4.17680102]]\n"
     ]
    }
   ],
   "source": [
    "print('--------- Started Training ----------')\n",
    "for iter in range(1,30):\n",
    "    random_int = np.random.randint(len(images) - 5)\n",
    "    current_image = np.expand_dims(images[random_int],axis=0)\n",
    "    #fig = plot(current_image,\"REAL\")\n",
    "    # Func: Generate The first Fake Data\n",
    "    Z = np.random.normal(0., 1., size=[1, G_input])\n",
    "    Gl1 = Z.dot(G_W1) + G_b1\n",
    "    Gl1A = Lrelu(Gl1)\n",
    "    Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "    Gl2A = Lrelu(Gl2)\n",
    "    Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "    Gl3A = Lrelu(Gl3)\n",
    "    Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "    Gl4A = Lrelu(Gl4)\n",
    "    Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "    Gl5A = Lrelu(Gl5)\n",
    "    Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "    Gl6A = Lrelu(Gl6)\n",
    "    Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "    Gl7A = Lrelu(Gl7)\n",
    "    Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "    Gl8A = Lrelu(Gl8)\n",
    "    Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "    Gl9A = Lrelu(Gl9)\n",
    "    Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "    Gl10A = Lrelu(Gl10)\n",
    "    Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "\n",
    "    current_fake_data = log(Gl11)\n",
    "\n",
    "    # Func: Forward Feed for Real data\n",
    "    Dl1_r = current_image.dot(D_W1) + D_b1\n",
    "    Dl1_rA = Lrelu(Dl1_r)\n",
    "    Dl2_r = Dl1_rA.dot(D_W2) + D_b2\n",
    "    Dl2_rA = Lrelu(Dl2_r)\n",
    "    Dl3_r = Dl2_rA.dot(D_W3) + D_b3\n",
    "    Dl3_rA = Lrelu(Dl3_r)\n",
    "    Dl4_r = Dl3_rA.dot(D_W4) + D_b4\n",
    "    Dl4_rA = log(Dl4_r)\n",
    "\n",
    "    # Func: Forward Feed for Fake Data\n",
    "    Dl1_f = current_fake_data.dot(D_W1) + D_b1\n",
    "    Dl1_fA = Lrelu(Dl1_f)\n",
    "    Dl2_f = Dl1_fA.dot(D_W2) + D_b2\n",
    "    Dl2_fA = Lrelu(Dl2_f)\n",
    "    Dl3_f = Dl2_fA.dot(D_W3) + D_b3\n",
    "    Dl3_fA = Lrelu(Dl3_f)\n",
    "    Dl4_f = Dl3_fA.dot(D_W4) + D_b4\n",
    "    Dl4_fA = log(Dl4_f)\n",
    "\n",
    "\n",
    "    # Func: Cost D\n",
    "    D_cost = -( np.log(Dl4_rA) +  np.log(1.0- Dl4_fA))\n",
    "    D_list_cost.append(D_cost[0][0])\n",
    "\n",
    "    # Func: Gradient\n",
    "    grad_f_w4_part_1 =  1/(1.0- Dl4_fA)\n",
    "    grad_f_w4_part_2 =  d_log(Dl4_f)\n",
    "    grad_f_w4_part_3 =   Dl3_fA\n",
    "    grad_f_w4 =     grad_f_w4_part_3.T.dot(grad_f_w4_part_1 * grad_f_w4_part_2) \n",
    "    grad_f_b4 = grad_f_w4_part_1 * grad_f_w4_part_2\n",
    "\n",
    "    grad_f_w3_part_1 =  (grad_f_w4_part_1 * grad_f_w4_part_2).dot(D_W4.T)\n",
    "    grad_f_w3_part_2 =  d_Lrelu(Dl3_f)\n",
    "    grad_f_w3_part_3 =   Dl2_fA\n",
    "    grad_f_w3 =       grad_f_w3_part_3.T.dot(grad_f_w3_part_1 * grad_f_w3_part_2) \n",
    "    grad_f_b3 =      grad_f_w3_part_1 * grad_f_w3_part_2\n",
    "    \n",
    "    grad_f_w2_part_1 = (grad_f_w3_part_1 * grad_f_w3_part_2).dot(D_W3.T)\n",
    "    grad_f_w2_part_2 = d_Lrelu(Dl2_f)\n",
    "    grad_f_w2_part_3 = Dl1_fA\n",
    "    grad_f_w2 = grad_f_w2_part_3.T.dot(grad_f_w2_part_1 * grad_f_w2_part_2)\n",
    "    grad_f_b2 = (grad_f_w2_part_1 * grad_f_w2_part_2)\n",
    "\n",
    "    grad_f_w1_part_1 = (grad_f_w2_part_1 * grad_f_w2_part_2).dot(D_W2.T)\n",
    "    grad_f_w1_part_2 = d_Lrelu(Dl1_f)\n",
    "    grad_f_w1_part_3 = current_fake_data\n",
    "    grad_f_w1 = grad_f_w1_part_3.T.dot(grad_f_w1_part_1 * grad_f_w1_part_2)\n",
    "    grad_f_b1 = grad_f_w1_part_1 * grad_f_w1_part_2\n",
    "    \n",
    "    #LATER\n",
    "    grad_r_w4_part_1 =  1/(1.0- Dl4_rA)\n",
    "    grad_r_w4_part_2 =  d_log(Dl4_r)\n",
    "    grad_r_w4_part_3 =   Dl3_rA\n",
    "    grad_r_w4 =     grad_r_w4_part_3.T.dot(grad_r_w4_part_1 * grad_r_w4_part_2) \n",
    "    grad_r_b4 = grad_r_w4_part_1 * grad_r_w4_part_2\n",
    "\n",
    "    grad_r_w3_part_1 =  (grad_r_w4_part_1 * grad_r_w4_part_2).dot(D_W4.T)\n",
    "    grad_r_w3_part_2 =  d_Lrelu(Dl3_r)\n",
    "    grad_r_w3_part_3 =   Dl2_rA\n",
    "    grad_r_w3 =       grad_r_w3_part_3.T.dot(grad_r_w3_part_1 * grad_r_w3_part_2) \n",
    "    grad_r_b3 =      grad_r_w3_part_1 * grad_r_w3_part_2\n",
    "    \n",
    "    grad_r_w2_part_1 = (grad_r_w3_part_1 * grad_r_w3_part_2).dot(D_W3.T)\n",
    "    grad_r_w2_part_2 = d_Lrelu(Dl2_r)\n",
    "    grad_r_w2_part_3 = Dl1_rA\n",
    "    grad_r_w2 = grad_r_w2_part_3.T.dot(grad_r_w2_part_1 * grad_r_w2_part_2)\n",
    "    grad_r_b2 = (grad_r_w2_part_1 * grad_r_w2_part_2)\n",
    "\n",
    "    grad_r_w1_part_1 = (grad_r_w2_part_1 * grad_r_w2_part_2).dot(D_W2.T)\n",
    "    grad_r_w1_part_2 = d_Lrelu(Dl1_r)\n",
    "    grad_r_w1_part_3 = current_image\n",
    "    grad_r_w1 = grad_r_w1_part_3.T.dot(grad_r_w1_part_1 * grad_r_w1_part_2)\n",
    "    grad_r_b1 = grad_r_w1_part_1 * grad_r_w1_part_2\n",
    "\n",
    "    grad_w1 =grad_f_w1 + grad_r_w1\n",
    "    grad_w2 =grad_f_w2 + grad_r_w2\n",
    "    grad_w3 =grad_f_w3 + grad_r_w3\n",
    "    grad_w4 =grad_f_w4 + grad_r_w4\n",
    "\n",
    "    grad_b1 =grad_f_b1 + grad_r_b1\n",
    "    grad_b2 =grad_f_b2 + grad_r_b2\n",
    "    grad_b3 =grad_f_b3 + grad_r_b3\n",
    "    grad_b4 =grad_f_b4 + grad_r_b4\n",
    "\n",
    "    # ---- Update Gradient ----\n",
    "    m1w1 = beta_1 * m1w1 + (1 - beta_1) * grad_w1\n",
    "    v1w1 = beta_2 * v1w1 + (1 - beta_2) * grad_w1 ** 2\n",
    "    \n",
    "    m2w2 = beta_1 * m2w2 + (1 - beta_1) * grad_w2\n",
    "    v2w2 = beta_2 * v2w2 + (1 - beta_2) * grad_w2 ** 2\n",
    "    \n",
    "    m3w3 = beta_1 * m3w3 + (1 - beta_1) * grad_w3\n",
    "    v3w3 = beta_2 * v3w3 + (1 - beta_2) * grad_w3 ** 2\n",
    "    \n",
    "    m4w4 = beta_1 * m4w4 + (1 - beta_1) * grad_w4\n",
    "    v4w4 = beta_2 * v4w4 + (1 - beta_2) * grad_w4 ** 2\n",
    "    \n",
    "    m1b1 = beta_1 * m1b1 + (1 - beta_1) * grad_b1\n",
    "    v1b1 = beta_2 * v1b1 + (1 - beta_2) * grad_b1 ** 2\n",
    "    \n",
    "    m2b2 = beta_1 * m2b2 + (1 - beta_1) * grad_b2\n",
    "    v2b2 = beta_2 * v2b2 + (1 - beta_2) * grad_b2 ** 2\n",
    "    \n",
    "    m3b3 = beta_1 * m3b3 + (1 - beta_1) * grad_b3\n",
    "    v3b3 = beta_2 * v3b3 + (1 - beta_2) * grad_b3 ** 2\n",
    "    \n",
    "    m4b4 = beta_1 * m4b4 + (1 - beta_1) * grad_b4\n",
    "    v4b4 = beta_2 * v4b4 + (1 - beta_2) * grad_b4 ** 2\n",
    "\n",
    "    D_W1 = D_W1 - (learing_rate / (np.sqrt(v1w1 /(1-beta_2**iter )) + eps)) * (m1w1/(1-beta_1**iter))\n",
    "    D_b1 = D_b1 - (learing_rate / (np.sqrt(v1b1 /(1-beta_2**iter )) + eps)) * (m1b1/(1-beta_1**iter))\n",
    "        \n",
    "    D_W2 = D_W2 - (learing_rate / (np.sqrt(v2w2 /(1-beta_2**iter )) + eps)) * (m2w2/(1-beta_1**iter))\n",
    "    D_b2 = D_b2 - (learing_rate / (np.sqrt(v2b2 /(1-beta_2**iter )) + eps)) * (m2b2/(1-beta_1**iter))\n",
    "    \n",
    "    D_W3 = D_W3 - (learing_rate / (np.sqrt(v3w3 /(1-beta_2**iter )) + eps)) * (m3w3/(1-beta_1**iter))\n",
    "    D_b3 = D_b3 - (learing_rate / (np.sqrt(v3b3 /(1-beta_2**iter )) + eps)) * (m3b3/(1-beta_1**iter))\n",
    "    \n",
    "    D_W4 = D_W4 - (learing_rate / (np.sqrt(v4w4 /(1-beta_2**iter )) + eps)) * (m4w4/(1-beta_1**iter))\n",
    "    D_b4 = D_b4 - (learing_rate / (np.sqrt(v4b4 /(1-beta_2**iter )) + eps)) * (m4b4/(1-beta_1**iter))\n",
    "    \n",
    "    # Func: Forward Feed for G\n",
    "    Z = np.random.normal(0., 1., size=[1, G_input]) \n",
    "    Gl1 = Z.dot(G_W1) + G_b1\n",
    "    Gl1A = Lrelu(Gl1)\n",
    "    Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "    Gl2A = Lrelu(Gl2)\n",
    "    Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "    Gl3A = Lrelu(Gl3)\n",
    "    Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "    Gl4A = Lrelu(Gl4)\n",
    "    Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "    Gl5A = Lrelu(Gl5)\n",
    "    Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "    Gl6A = Lrelu(Gl6)\n",
    "    Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "    Gl7A = Lrelu(Gl7)\n",
    "    Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "    Gl8A = Lrelu(Gl8)\n",
    "    Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "    Gl9A = Lrelu(Gl9)\n",
    "    Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "    Gl10A = Lrelu(Gl10)\n",
    "    Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "    \n",
    "    current_fake_data = log(Gl11)\n",
    "    #fig = plot(current_fake_data,\"FAKE\")\n",
    "    Dl1 = current_fake_data.dot(D_W1) + D_b1\n",
    "    Dl1_A = Lrelu(Dl1)\n",
    "    Dl2 = Dl1_A.dot(D_W2) + D_b2\n",
    "    Dl2_A = Lrelu(Dl2)\n",
    "    Dl3 = Dl2_A.dot(D_W3) + D_b3\n",
    "    Dl3_A = Lrelu(Dl3)\n",
    "    Dl4 = Dl3_A.dot(D_W4) + D_b4\n",
    "    Dl4_A = log(Dl4)\n",
    "\n",
    "    # Func: Cost G\n",
    "    G_cost = -np.log(Dl4_A)\n",
    "    G_list_cost.append(G_cost[0][0])\n",
    "    # Func: Gradient\n",
    "    grad_G_w11_part_1 = (( (-1/Dl4_A) * d_log(Dl4).dot(D_W4.T) * (d_Lrelu(Dl3)) ).dot(D_W3.T).dot(D_W2.T).dot(D_W1.T))\n",
    "    grad_G_w11_part_2 = d_log(Gl11)\n",
    "    grad_G_w11_part_3 = Gl10A\n",
    "    grad_G_w11 = grad_G_w11_part_3.T.dot(grad_G_w11_part_1 * grad_G_w11_part_1)\n",
    "    grad_G_b11 = grad_G_w11_part_1 * grad_G_w11_part_2\n",
    "\n",
    "    grad_G_w10_part_1 = (grad_G_w11_part_1 * grad_G_w11_part_2).dot(G_W11.T)\n",
    "    grad_G_w10_part_2 = d_Lrelu(Gl10)\n",
    "    grad_G_w10_part_3 = Gl9A\n",
    "    grad_G_w10 = grad_G_w10_part_3.T.dot(grad_G_w10_part_1 * grad_G_w10_part_2)\n",
    "    grad_G_b10 = (grad_G_w10_part_1 * grad_G_w10_part_2)\n",
    "    \n",
    "    grad_G_w9_part_1 = (grad_G_w10_part_1 * grad_G_w10_part_2).dot(G_W10.T)\n",
    "    grad_G_w9_part_2 = d_Lrelu(Gl9)\n",
    "    grad_G_w9_part_3 = Gl8A\n",
    "    grad_G_w9 = grad_G_w9_part_3.T.dot(grad_G_w9_part_1 * grad_G_w9_part_2)\n",
    "    grad_G_b9 = (grad_G_w9_part_1 * grad_G_w9_part_2)\n",
    "    \n",
    "    grad_G_w8_part_1 = (grad_G_w9_part_1 * grad_G_w9_part_2).dot(G_W9.T)\n",
    "    grad_G_w8_part_2 = d_Lrelu(Gl8)\n",
    "    grad_G_w8_part_3 = Gl7A\n",
    "    grad_G_w8 = grad_G_w8_part_3.T.dot(grad_G_w8_part_1 * grad_G_w8_part_2)\n",
    "    grad_G_b8 = (grad_G_w8_part_1 * grad_G_w8_part_2)\n",
    "   \n",
    "    grad_G_w7_part_1 = (grad_G_w8_part_1 * grad_G_w8_part_2).dot(G_W8.T)\n",
    "    grad_G_w7_part_2 = d_Lrelu(Gl7)\n",
    "    grad_G_w7_part_3 = Gl6A\n",
    "    grad_G_w7 = grad_G_w7_part_3.T.dot(grad_G_w7_part_1 * grad_G_w7_part_2)\n",
    "    grad_G_b7 = (grad_G_w7_part_1 * grad_G_w7_part_2)\n",
    "\n",
    "    grad_G_w6_part_1 = (grad_G_w7_part_1 * grad_G_w7_part_2).dot(G_W7.T)\n",
    "    grad_G_w6_part_2 = d_Lrelu(Gl6)\n",
    "    grad_G_w6_part_3 = Gl5A\n",
    "    grad_G_w6 = grad_G_w6_part_3.T.dot(grad_G_w6_part_1 * grad_G_w6_part_2)\n",
    "    grad_G_b6 = (grad_G_w6_part_1 * grad_G_w6_part_2)\n",
    "\n",
    "    grad_G_w5_part_1 = (grad_G_w6_part_1 * grad_G_w6_part_2).dot(G_W6.T)\n",
    "    grad_G_w5_part_2 = d_Lrelu(Gl5)\n",
    "    grad_G_w5_part_3 = Gl4A\n",
    "    grad_G_w5 = grad_G_w5_part_3.T.dot(grad_G_w5_part_1 * grad_G_w5_part_2)\n",
    "    grad_G_b5 = (grad_G_w5_part_1 * grad_G_w5_part_2)\n",
    "\n",
    "    grad_G_w4_part_1 = (grad_G_w5_part_1 * grad_G_w5_part_2).dot(G_W5.T)\n",
    "    grad_G_w4_part_2 = d_Lrelu(Gl4)\n",
    "    grad_G_w4_part_3 = Gl3A\n",
    "    grad_G_w4 = grad_G_w4_part_3.T.dot(grad_G_w4_part_1 * grad_G_w4_part_2)\n",
    "    grad_G_b4 = (grad_G_w4_part_1 * grad_G_w4_part_2)\n",
    "\n",
    "    grad_G_w3_part_1 = (grad_G_w4_part_1 * grad_G_w4_part_2).dot(G_W4.T)\n",
    "    grad_G_w3_part_2 = d_Lrelu(Gl3)\n",
    "    grad_G_w3_part_3 = Gl2A\n",
    "    grad_G_w3 = grad_G_w3_part_3.T.dot(grad_G_w3_part_1 * grad_G_w3_part_2)\n",
    "    grad_G_b3 = (grad_G_w3_part_1 * grad_G_w3_part_2)\n",
    "\n",
    "    grad_G_w2_part_1 = (grad_G_w3_part_1 * grad_G_w3_part_2).dot(G_W3.T)\n",
    "    grad_G_w2_part_2 = d_Lrelu(Gl2)\n",
    "    grad_G_w2_part_3 = Gl1A\n",
    "    grad_G_w2 = grad_G_w2_part_3.T.dot(grad_G_w2_part_1 * grad_G_w2_part_2)\n",
    "    grad_G_b2 = (grad_G_w2_part_1 * grad_G_w2_part_2)\n",
    "\n",
    "    grad_G_w1_part_1 = (grad_G_w2_part_1 * grad_G_w2_part_2).dot(G_W2.T)\n",
    "    grad_G_w1_part_2 = d_Lrelu(Gl1)\n",
    "    grad_G_w1_part_3 = Z\n",
    "    grad_G_w1 = grad_G_w1_part_3.T.dot(grad_G_w1_part_1 * grad_G_w1_part_2)\n",
    "    grad_G_b1 = grad_G_w1_part_1 * grad_G_w1_part_2\n",
    "\n",
    "    # ---- Update Gradient ----\n",
    "    m5 = beta_1 * m5 + (1 - beta_1) * grad_G_w1\n",
    "    v5 = beta_2 * v5 + (1 - beta_2) * grad_G_w1 ** 2\n",
    "\n",
    "    m6 = beta_1 * m6 + (1 - beta_1) * grad_G_b1\n",
    "    v6 = beta_2 * v6 + (1 - beta_2) * grad_G_b1 ** 2\n",
    "\n",
    "    m7 = beta_1 * m7 + (1 - beta_1) * grad_G_w2\n",
    "    v7 = beta_2 * v7 + (1 - beta_2) * grad_G_w2 ** 2\n",
    "\n",
    "    m8 = beta_1 * m8 + (1 - beta_1) * grad_G_b2\n",
    "    v8 = beta_2 * v8 + (1 - beta_2) * grad_G_b2 ** 2\n",
    "\n",
    "    m9 = beta_1 * m9 + (1 - beta_1) * grad_G_w3\n",
    "    v9 = beta_2 * v9 + (1 - beta_2) * grad_G_w3 ** 2\n",
    "\n",
    "    m10 = beta_1 * m10 + (1 - beta_1) * grad_G_b3\n",
    "    v10 = beta_2 * v10 + (1 - beta_2) * grad_G_b3 ** 2\n",
    "\n",
    "    m11 = beta_1 * m11 + (1 - beta_1) * grad_G_w4\n",
    "    v11 = beta_2 * v11 + (1 - beta_2) * grad_G_w4 ** 2\n",
    "\n",
    "    m12 = beta_1 * m12 + (1 - beta_1) * grad_G_b4\n",
    "    v12 = beta_2 * v12 + (1 - beta_2) * grad_G_b4 ** 2\n",
    "\n",
    "    m13 = beta_1 * m13 + (1 - beta_1) * grad_G_w5\n",
    "    v13 = beta_2 * v13 + (1 - beta_2) * grad_G_w5 ** 2\n",
    "\n",
    "    m14 = beta_1 * m14 + (1 - beta_1) * grad_G_b5\n",
    "    v14 = beta_2 * v14 + (1 - beta_2) * grad_G_b5 ** 2\n",
    "\n",
    "    m15 = beta_1 * m15 + (1 - beta_1) * grad_G_w6\n",
    "    v15 = beta_2 * v15 + (1 - beta_2) * grad_G_w6 ** 2\n",
    "\n",
    "    m16 = beta_1 * m16 + (1 - beta_1) * grad_G_b6\n",
    "    v16 = beta_2 * v16 + (1 - beta_2) * grad_G_b6 ** 2\n",
    "\n",
    "    m17 = beta_1 * m17 + (1 - beta_1) * grad_G_w7\n",
    "    v17 = beta_2 * v17 + (1 - beta_2) * grad_G_w7 ** 2\n",
    "\n",
    "    m18 = beta_1 * m18 + (1 - beta_1) * grad_G_b7\n",
    "    v18 = beta_2 * v18 + (1 - beta_2) * grad_G_b7 ** 2\n",
    "    \n",
    "    m19 = beta_1 * m19 + (1 - beta_1) * grad_G_w8\n",
    "    v19 = beta_2 * v19 + (1 - beta_2) * grad_G_w8 ** 2\n",
    "    \n",
    "    m20 = beta_1 * m20 + (1 - beta_1) * grad_G_b8\n",
    "    v20 = beta_2 * v20 + (1 - beta_2) * grad_G_b8 ** 2\n",
    "    \n",
    "    m21 = beta_1 * m21 + (1 - beta_1) * grad_G_w9\n",
    "    v21 = beta_2 * v21 + (1 - beta_2) * grad_G_w9 ** 2\n",
    "    \n",
    "    m22 = beta_1 * m22 + (1 - beta_1) * grad_G_b9\n",
    "    v22 = beta_2 * v22 + (1 - beta_2) * grad_G_b9 ** 2\n",
    "    \n",
    "    m23 = beta_1 * m23 + (1 - beta_1) * grad_G_w10\n",
    "    v23 = beta_2 * v23 + (1 - beta_2) * grad_G_w10 ** 2\n",
    "\n",
    "    m24 = beta_1 * m24 + (1 - beta_1) * grad_G_b10\n",
    "    v24 = beta_2 * v24 + (1 - beta_2) * grad_G_b10 ** 2\n",
    "    \n",
    "    m25 = beta_1 * m25 + (1 - beta_1) * grad_G_w11\n",
    "    v25 = beta_2 * v25 + (1 - beta_2) * grad_G_w11 ** 2\n",
    "    \n",
    "    m26 = beta_1 * m26 + (1 - beta_1) * grad_G_b11\n",
    "    v26 = beta_2 * v26 + (1 - beta_2) * grad_G_b11 ** 2\n",
    "    \n",
    "\n",
    "    G_W1 = G_W1 - (learing_rate / (np.sqrt(v5 /(1-beta_2**iter )) + eps)) * (m5/(1-beta_1**iter))\n",
    "    G_b1 = G_b1 - (learing_rate / (np.sqrt(v6 /(1-beta_2**iter )) + eps)) * (m6/(1-beta_1**iter))\n",
    "    \n",
    "    G_W2 = G_W2 - (learing_rate / (np.sqrt(v7 /(1-beta_2**iter )) + eps)) * (m7/(1-beta_1**iter))\n",
    "    G_b2 = G_b2 - (learing_rate / (np.sqrt(v8 /(1-beta_2**iter )) + eps)) * (m8/(1-beta_1**iter))\n",
    "\n",
    "    G_W3 = G_W3 - (learing_rate / (np.sqrt(v9 /(1-beta_2**iter )) + eps)) * (m9/(1-beta_1**iter))\n",
    "    G_b3 = G_b3 - (learing_rate / (np.sqrt(v10 /(1-beta_2**iter )) + eps)) * (m10/(1-beta_1**iter))\n",
    "\n",
    "    G_W4 = G_W4 - (learing_rate / (np.sqrt(v11 /(1-beta_2**iter )) + eps)) * (m11/(1-beta_1**iter))\n",
    "    G_b4 = G_b4 - (learing_rate / (np.sqrt(v12 /(1-beta_2**iter )) + eps)) * (m12/(1-beta_1**iter))\n",
    "\n",
    "    G_W5 = G_W5 - (learing_rate / (np.sqrt(v13 /(1-beta_2**iter )) + eps)) * (m13/(1-beta_1**iter))\n",
    "    G_b5 = G_b5 - (learing_rate / (np.sqrt(v14 /(1-beta_2**iter )) + eps)) * (m14/(1-beta_1**iter))\n",
    "\n",
    "    G_W6 = G_W6 - (learing_rate / (np.sqrt(v15 /(1-beta_2**iter )) + eps)) * (m15/(1-beta_1**iter))\n",
    "    G_b6 = G_b6 - (learing_rate / (np.sqrt(v16 /(1-beta_2**iter )) + eps)) * (m16/(1-beta_1**iter))\n",
    "\n",
    "    G_W7 = G_W7 - (learing_rate / (np.sqrt(v17 /(1-beta_2**iter ))+ eps)) * (m17/(1-beta_1**iter))\n",
    "    G_b7 = G_b7 - (learing_rate / (np.sqrt(v18 /(1-beta_2**iter )) + eps)) * (m18/(1-beta_1**iter))\n",
    "    \n",
    "    G_W8 = G_W8 - (learing_rate / (np.sqrt(v19 /(1-beta_2**iter )) + eps)) * (m19/(1-beta_1**iter))\n",
    "    G_b8 = G_b8 - (learing_rate / (np.sqrt(v20 /(1-beta_2**iter )) + eps)) * (m20/(1-beta_1**iter))\n",
    "    \n",
    "    G_W9 = G_W9 - (learing_rate / (np.sqrt(v21 /(1-beta_2**iter )) + eps)) * (m21/(1-beta_1**iter))\n",
    "    G_b9 = G_b9 - (learing_rate / (np.sqrt(v22 /(1-beta_2**iter )) + eps)) * (m22/(1-beta_1**iter))\n",
    "    \n",
    "    G_W10 = G_W10 - (learing_rate / (np.sqrt(v23 /(1-beta_2**iter )) + eps)) * (m23/(1-beta_1**iter))\n",
    "    G_b10 = G_b10 - (learing_rate / (np.sqrt(v24 /(1-beta_2**iter )) + eps)) * (m24/(1-beta_1**iter))\n",
    "    \n",
    "    G_W11 = G_W11 - (learing_rate / (np.sqrt(v25 /(1-beta_2**iter )) + eps)) * (m25/(1-beta_1**iter))\n",
    "    G_b11 = G_b11 - (learing_rate / (np.sqrt(v26 /(1-beta_2**iter )) + eps)) * (m26/(1-beta_1**iter))\n",
    "\n",
    "    print(\"Current Iter: \",iter, \" Current D cost:\",D_cost, \" Current G cost: \", G_cost)\n",
    "    \n",
    "    if iter == 0:\n",
    "        learing_rate = learing_rate #* 0.01\n",
    "    if iter == 40:\n",
    "        learing_rate = learing_rate #* 0.01\n",
    "\n",
    "    # ---- Print to Out put ----\n",
    "# -- end code --"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 284
    },
    "colab_type": "code",
    "id": "zYry6Fe6eL4m",
    "outputId": "ff02d8fc-befe-4618-90e0-a200bd5506e9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x27cc85fd8d0>]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xu8XPO5x/HPkxsiJCpRkUuDOjjivjdBDzkEkYpb3eJSFKm2TtGr9uXecrRaLdWKtCQ0SFRc4roblxI0l500JKRIK4ggm9yEyG0/549n5mQbs/fM3nv2rJnZ3/frtV5zWzPrWUa+e81v/dbvZ+6OiIhUlg5JFyAiIoWncBcRqUAKdxGRCqRwFxGpQAp3EZEKpHAXEalACncRkQqkcBcRqUAKdxGRCtQpqQ337NnTBwwYkNTmRUTK0syZMz9w91651kss3AcMGEBtbW1SmxcRKUtm9mY+66lZRkSkAincRUQqkMJdRKQCKdxFRCqQwl1EpAIp3EVEKpDCXUSkAincRUSK6Mor4Zln2n47CncRkSJ54w244gp49tm235bCXUSkSG6/HczgjDPaflsKdxGRIqivj3A/+GDo37/tt6dwFxEpgmeegQUL4KyzirM9hbuISBGMGQObbw7HHluc7SncRUTa2IoVcO+9cNJJ0LVrcbapcBcRaWN/+QusWlW8JhloRribWUcz+4eZPZzltY3MbIKZzTezaWY2oJBFioiUs7FjYccdYdCg4m2zOUfuFwDzGnntbGCpu38Z+A3wi9YWJiJSCV5/HZ57Lo7azYq33bzC3cz6Al8F/tTIKkcDt6fu3wscYlbM3RARKU1jx0KHDnD66cXdbr5H7r8FfgTUN/J6H+BtAHdfBywHtsxcycxGmlmtmdXW1dW1oFwRkfKxfj3ccQccfjhss01xt50z3M3sSGCxu89sarUsz/nnnnAf7e5V7l7Vq1fO+V1FRMrak0/CwoXFPZGals+R+wHAUWa2ABgPHGxm4zLWWQj0AzCzTkB3YEkB6xQRKTtjxsAWW8Dw4cXfds5wd/efuHtfdx8AnAw85e6nZaw2CUiPlnB8ap3PHbmLiLQXy5bB/ffDKafAxhsXf/udWvpGM7sKqHX3ScCtwJ/NbD5xxH5ygeoTESlL48fD6tXJNMkAWFIH2FVVVV5bW5vItkVE2tq++8Inn8BLLxW2C6SZzXT3qlzr6QpVEZECe+UVmD69+H3bG1K4i4gU2Nix0KkTnJZ5drKIFO4iIgW0bh38+c8wbBhstVVydSjcRUQKqKYG3nsvuROpaQp3EZECGjMGevWCr3412ToU7iIiBfLBBzBpEpx6KnTunGwtCncRkQK5+25Yuzb5JhlQuIuIFMyYMbDXXrDbbklXonAXESmIF1+Ef/wDzjwz6UqCwl1EpADGjoUuXWIsmVKgcBcRaaU1a2DcODjqKNjyczNZJEPhLiLSSo8+Gj1lSuFEaprCXUSklcaMgd694bDDkq5kA4W7iEgrLF0Kjz0Wbe2dWjyIeuEp3EVEWuH++6Nv+8klNouFwl1EpBXGj4ftt4e99066ks9SuIuItNDixfDUU3DSScmN294YhbuISAtNnAjr15dekwwo3EVEWmzCBNh5Zxg4MOlKPi9nuJvZxmY23cxeNLOXzezKLOucaWZ1ZjY7tZzTNuWKiJSGRYvg2WdLs0kGIJ+OO6uBg919pZl1Bp4zs8fcfWrGehPc/fzClygiUnr+8hdwj3AvRTnD3d0dWJl62Dm1eFsWJSJS6iZMgN13h512SrqS7PJqczezjmY2G1gMTHb3aVlW+5qZvWRm95pZv4JWKSJSQt58E/7+99I8kZqWV7i7+3p33wPoC+xjZpmnDx4CBrj7bsATwO3ZPsfMRppZrZnV1tXVtaZuEZHE3HNP3J54YrJ1NMWi1aUZbzC7HPjY3X/VyOsdgSXu3r2pz6mqqvLa2tpmbVtEpBTsvTd07AjTpxd/22Y2092rcq2XT2+ZXmbWI3V/E2AI8M+MdXo3eHgUMK955YqIlIfXX4dZs0q7SQby6y3TG7g9dUTeAbjH3R82s6uAWnefBHzXzI4C1gFLgDPbqmARkSRNmBC3J5yQbB25NLtZplDULCMi5WjXXaFHD5gyJZntF6xZRkREwssvw9y5pdu3vSGFu4hIniZMgA4d4Pjjk64kN4W7iEge3CPcBw+GrbdOuprcFO4iInmYPRtee608mmRA4S4ikpcJE2IaveOOS7qS/CjcRURySDfJDBkCPXsmXU1+FO4iIjlMnw4LFpT+hUsNKdxFRHIYPx66dIFjjkm6kvwp3EVEmlBfH2O3Dx0K3ZscMau0KNxFRJrw/PPwzjvl1SQDCncRkSaNHw+bbALDhyddSfMo3EVEGrFuHdx7Lxx5JHTrlnQ1zaNwFxFpxN/+BosXl8+FSw0p3EVEGjFhQhyxDxuWdCXNp3AXEclizRqYOBGOPjra3MuNwl1EJIsnnoClS8uzSQYU7iIiWY0fH5NyHHZY0pW0jMJdRCTDJ5/A/ffHIGEbbZR0NS2jcBcRyTBpEqxcCaefnnQlLZcz3M1sYzObbmYvmtnLZnZllnU2MrMJZjbfzKaZ2YC2KFZEpBjGjYO+feHAA5OupOXyOXJfDRzs7rsDewBDzWxQxjpnA0vd/cvAb4BfFLZMEZHiqKuDxx+HU06JKfXKVc7SPaxMPeycWjxjtaOB21P37wUOMTMrWJUiIkUyYQKsXw+nnZZ0Ja2T198lM+toZrOBxcBkd5+WsUof4G0Ad18HLAe2LGShIiLFMG4c7LYb7Lpr0pW0Tl7h7u7r3X0PoC+wj5kNzFgl21F65tE9ZjbSzGrNrLaurq751YqItKH582HatPI/aodm9pZx92XA34ChGS8tBPoBmFknoDuwJMv7R7t7lbtX9erVq0UFi4i0lTvvBDMYMSLpSlovn94yvcysR+r+JsAQ4J8Zq00CzkjdPx54yt0/d+QuIlKq3KNJZvDg6ClT7vI5cu8NPG1mLwEziDb3h83sKjM7KrXOrcCWZjYf+B5wcduUKyLSNqZPj2aZSmiSAeiUawV3fwnYM8vzlzW4/ylwQmFLExEpnnHj4mrUr30t6UoKo4x7cYqIFMbatdEF8qijymue1KYo3EWk3Zs8OS5eqpQmGVC4i4gwbhx84QswNLMfYBlTuItIu/bRR/DAA3DiidClS9LVFI7CXUTatfvvh1WrKqtJBhTuItLO3XknDBgA+++fdCWFpXAXkXbr3XdjOr3TTosrUyuJwl1E2q3x46G+Hk49NelKCk/hLiLt1rhxsPfesNNOSVdSeAp3EWmX5s2DWbMq70RqmsJdRNqlO++MmZZOPjnpStqGwl1E2p36+gj3Qw+FrbdOupq2oXAXkXbnhRdgwYLKPJGapnAXkXZn3Djo2hWOPTbpStqOwl1E2pU1a+Cee+CYY6Bbt6SraTsKdxFpVx57DJYurdxeMmkKdxFpV8aNg1694mRqJVO4i0i7sWwZPPRQdH/slHMeuvKmcBeRdmPiRFi9uvKbZEDhLiLtyG23wY47QnV10pW0vZzhbmb9zOxpM5tnZi+b2QVZ1hlsZsvNbHZquSzbZ4mIJOWll6J/+8iRlTcCZDb5tDqtA77v7rPMbDNgpplNdvdXMtab4u5HFr5EEZHWu+UW2GgjOOOMpCspjpxH7u7+rrvPSt3/CJgH9GnrwkRECmXlSvjzn2MqvS23TLqa4mhWm7uZDQD2BKZleXk/M3vRzB4zs10aef9IM6s1s9q6urpmFysi0hLjx8dcqd/8ZtKVFI+5e34rmnUDngGudvf7Ml7bHKh395VmNgy4wd13aOrzqqqqvLa2toVli4jkr7oaPv002t3Lvb3dzGa6e1Wu9fI6cjezzsBE4M7MYAdw9xXuvjJ1/1Ggs5n1bGbNIiIFV1sbyze/Wf7B3hz59JYx4FZgnrtf38g6W6fWw8z2SX3uh4UsVESkJW65JQYJO/30pCsprnx6yxwAnA7MMbPZqed+CvQHcPdRwPHAt8xsHbAKONnzbe8REWkjy5fD3XfDiBHQvXvS1RRXznB39+eAJn/MuPtNwE2FKkpEpBDuvBM+/rh9nUhN0xWqIlKR3GHUKNhrL6jKefqx8lT40Dki0l5NnQpz5sDo0e3rRGqajtxFpCKNGgWbbRbt7e2Rwl1EKs6SJTHb0mmnVfZsS01RuItIxbnjjrhoqT2eSE1TuItIRUmfSB00CHbfPelqkqMTqiJSUZ59Fl59FcaOTbqSZOnIXUQqyqhR0KNHjADZnincRaRiLF4cU+mdcQZssknS1SRL4S4iFWPsWFi7tn2fSE1TuItIRaivj0HCDjoIdt456WqSp3AXkYrw5JPw73/rqD1N4S4iFWHUKOjZE447LulKSoPCXUTK3qJF8OCD8I1vxCTYonAXkQpw222wfj2ce27SlZQOhbuIlLX162Pkx0MPhS9/OelqSofCXUTK2qOPwttvw3nnJV1JaVG4i0jZcoerroIvfQmGD0+6mtKisWVEpGw9+CDU1kabe+fOSVdTWnIeuZtZPzN72szmmdnLZnZBlnXMzG40s/lm9pKZ7dU25YqIhPXr4ZJLYMcd4fTTk66m9ORz5L4O+L67zzKzzYCZZjbZ3V9psM4RwA6pZV/g5tStiEibGD8eXn4ZJkyATmqD+JycR+7u/q67z0rd/wiYB/TJWO1o4A4PU4EeZta74NWKiBDjx1x2GeyxBxx/fNLVlKZm/b0zswHAnsC0jJf6AG83eLww9dy7rahNRCSrMWNiqIGHH4YO6haSVd7/WcysGzARuNDdV2S+nOUtnuUzRppZrZnV1tXVNa9SERFi+ryrroL994dhw5KupnTlFe5m1pkI9jvd/b4sqywE+jV43BdYlLmSu4929yp3r+rVq1dL6hWRdu7mm+Gdd+Dqq8GyHVYKkF9vGQNuBea5+/WNrDYJ+Hqq18wgYLm7q0lGRArqo4/gmmviatTBg5OupoHRo+NqqhKST5v7AcDpwBwzm5167qdAfwB3HwU8CgwD5gOfAGcVvlQRae9uuAE++AB+/vOkK2ng00/hwgthr71Kqp0oZ7i7+3Nkb1NvuI4D3ylUUSIimZYsgeuug2OOgX32SbqaBqZMgVWrYNYsWLeuZPpl6jyziJSF666LZpmf/SzpSjLU1MTtqlUwb16ytTSgcBeRkvfee9Ekc8opMHBg0tVkqKmBbbeN+zNmJFtLAwp3ESl5V18Na9bAFVckXUmGhQth7twYkrJ7d4W7iEi+3nwzJr4+++wSHK/9r3+N26FDoapK4S4ikq8rr4yrUC+9NOlKsqipgd69YdddoboaXnoJVq9OuipA4S4iJezVV+H22+Hb34a+fZOuJsP69TB5Mhx+eFxNVVUVg968+GLSlQEKdxEpYZddBptsAhdfnHQlWcyYAUuXRrhDHLlDDDBfAhTuIlKSZs+Ge+6Biy6CrbZKuposamriiP3QQ+Nxv35RaIm0uyvcRaQkXXIJ9OgB3/9+0pU0oqYmjta33DIem8VjhbuISHZTpsAjj8CPfxwBX3KWLoVp0zY0yaRVV8eFTCtXJlNXAwp3ESkpq1fDyJEx6fX//E/S1TTiiSegvj57uNfXx1AECVO4i0hJ+d//hX/+M4b23XTTpKtpRE1NXLS0b8ZsolVVcVsCTTMKdxEpGa+8EkP6jhgBRxyRdDWNcI9wHzLk84OEbbUV9O+vcBcRSauvh3PPhc02g9/+NulqmvDKKzHsQGaTTFp1dUl0h1S4i0hJuOUWeOEF+PWvS7TrY1p6FMimwv1f/4oxihOkcBeRxL3zTvSMOeQQOOOMpKvJoaYGdt45ml+yKZGLmRTuIpK488+PK/dHjSrxeVFXrYJnn238qB1g773jNuF299KYMkRE2q377oMHHoBrry3BUR8zPfNMTKvXVLh37w7/8R+Jh7uO3EUkMcuWxVH77rvD976XdDV5qKmBjTaCAw9ser0SuFI1Z7ib2W1mttjM5jby+mAzW25ms1PLZYUvU0Qq0cUXw/vvw5/+BJ07J11NHmpq4KCDoGvXpterroZFi2JJSD5H7mOBoTnWmeLue6SWq1pflohUuilToofMBRdsuPanpL31Vgwt0FSTTFoJnFTNGe7u/iyQbJ8eEakoDYcYuKpcDgdzdYFsaI89oGPHRJtmCtXmvp+ZvWhmj5nZLgX6TBGpUNdcE0MMjBoF3bolXU2eampixpD//M/c63btGjN5l3m4zwK+5O67A78DHmhsRTMbaWa1ZlZbV1dXgE2LSLl5+eUYP+aUU2Lq0bKwbl0MFpaedSkf6TlV3du2tka0OtzdfYW7r0zdfxTobGY9G1l3tLtXuXtVr169WrtpESkz9fXRHLPZZvCb3yRdTTNMmwbLl+fXJJNWXR1Xqb7xRtvV1YRWh7uZbW0Wf8rMbJ/UZ37Y2s8VkcozalQMMXD99SU+xECmmpqYpXvIkPzfkz6pmlDTTD5dIe8G/g7saGYLzexsMzvPzM5LrXI8MNfMXgRuBE52T+h3iIiUrDfeiK6PQ4bA17+edDXNVFMTw/tusUX+79l11+gTn1CPmZxXqLr7iByv3wTcVLCKRKTifPQRHHVUjJB7yy0lPsRApg8/jKPvyy9v3vs6d45eMwkduZff8AMrVsQoQ9J+ZCZBw8dm+S8dOkT3tMyl4fMdOpRZ8pS++no4/fToIv7447DddklX1EyTJ8dJ0ea0t6dVV8PYsbB+ffz/VUTlF+41NXDiiUlXIZWsQ4c4xOzUKY6+0vcbe9ylS/z83njjuM1c0s9vuil88YuwzTbQu3fc9upV9H/0xXb55fDgg3DDDc1rsi4ZNTXRHJNuQ2+Oqiq46SZ49dX8ulAWUPmF+6BBMH580lVIsWSevmn42L15S319HEE1XLI9l17Wro0ucOkl2+O1a2HNmrgqZ/nyuG1sWbPm8/vXsWOcWWwY+L17Q8+e8Ycj/cek4W3mc9tsA9tvX5K/OCZMgJ//HM4+u4TnQ21KetalQw9t2R/hhidVFe459OsHJ52UdBUizbdmTQyksmgRvPvuhtv0/YULYfp0qKtrft/ovn3h4INjQPSDD47HCZs1C846Cw44AH7/+5L825PbnDnx/bSkSQZgxx3jKq0ZM4o+UH35hbtIuerSJQ5O+vVrer21a+NXQPqXQfoXQ7b7a9fC/Pnw5JPwyCNwxx3xGTvssCHsBw+O5p8iev99OPro+AEycWK0SpWl9JADhx3Wsvd37BjjuydwUlXhLlJqOneOVMzXkCFw3nnRxDRnDjz1VCx33RVdUwB22y2Cfp99YqzxHXaIK4nawOrVcNxxcf3O88/HaYayVVMTwwi05pdQdTX87nfxy61Ll8LVloPCXaRSdOgQA6PvvjtcdFEc4dfWbgj7m2/+7GWhW28dQZ8O+/T97bdv8aG2O3zrW3Gh0j33RE/AsvXxxzF0ZWtPFlRXx1+8uXNhr70KU1seFO4ilapTp+iAMGgQ/PSnMYPQa6/B669/9vbBB6OdP80shmvcY48N76+qit4+OdxwA4wZA5ddBiec0Ib71lLusa8bbxz709RJ0r/9LY62W9renpYez3jGjKKGuyV1MWlVVZXXJjyBrIikLFsWYZ8O/FdfhZkz4zFECO6664awHzQojvY7bLjI/a9/hSOOiLb2e+/9zEul45xz4NZbNzxOd1FtuHTtGrdvvRWX1S5ZEn8MWso9znkcc0zMStJKZjbT3XOOgK8jdxGBHj2i+SCzL/cHH0QPnqlTY7nrrhggBqLv9777wqBBvDZ4JCed1JtddolzuiUZ7PfeG8F+5pnRjv7xx40vH3wQoXz++a0LdohfQukRIotIR+4ikr/6+hiIPR3206axfM5b7GvT+LBrP2a8uBEDtivBZH/vvQj0bbeNEwLFntPv0ktjnOMVK3JP0ZdDvkfuJfgtiEjJ6tAhLsb5xjdg9GjWzHiRkw98h3/5dkxceTgDzhwczTqlxD3GGV65Mn5WJDFZa3V1XBg3e3bRNqlwF5EW+egjOPJIePyZrvxhVEcOHPON6Iq5225w7bXRB78UjB0LDz0UR84775xMDQ1PqhaJwl1Emq2uLq6Reuqp6B1z7kiLtux58yLxf/KTaI//xz+SLfTNN2MG7oMOitukbLNNLAp3ESlVCxbAV74S3bbvvz8y/f9tvXWcuJw4MS7br66OoF+1qviF1tfH+Afu8Rco6bO81dUKdxEpTXPmxFgxixfHlKLDhzey4nHHwSuvxHgq114bfeanTClqrdx0Ezz9dFy4te22xd12NtXVcT5i2bKibE7hLiJ5ee45OPDAuD9lSoR8k7bYIroeTp4c7e8HHhiXr779dpvXyquvwo9/DF/9agxJWQrS3UxnzizK5hTuIpLTQw/FqLdbbRU9CQcObMabhwyJQ/6LLoLRo+Pq18MPj6G7P/208MWuWxfz+HXtCn/8Y+kMR1nkk6oKdxFp0tixcOyxcYHqc89FNjfbppvGrNivvw6XXBJ95UeMiLHrv/WtuFCqUNfc/OIX8Xl/+EN8fqn4whdi3J4iXd+jcBeRRl13XZyTTPeMafXIwdttB1ddFZf1P/FENJuMHRs9awYOhF/9Ki44aqnZs+HKK2POh1Kc96GIV6rmDHczu83MFpvZ3EZeNzO70czmm9lLZla8kXFEpE3U18MPfgA/+lFk5MMPx5wTBdOhQwxBPG5chPktt0D37vDDH8bwusOHR4+blSvz/8zVq2Oy1i23jNlBSlF1dYxZs3hxm28qnyP3scDQJl4/AtghtYwEbm59WSKSlKVL4dRT4de/jqFV7rqrjYch7949riB94YXoJ/+DH8RJx+OPj6aMQw6JI/q5c5tuurn88ljn1lsj4EtRw2n32ljOcHf3Z4ElTaxyNHCHh6lADzMroYYuEcmHe5zj3HnnGIv96qvhxhuL3D18p52i6+Rbb8XsUhdeGEe5P/xhNPr37w/nngv33RezVaU9/3y0IZ1zDgwbVsSCm2mvveI/aBHCvRCjQvYBGvZtWph67t0CfLaIFMEbb8R5zZqaaBZ+7DHYc88EC+rUKRr6Dz4YfvnLmF/28cdjueeeGDq3Y0fYf/8YZ/jWWyP4r78+waLz0K1bzBi+335tvqlChHu2fkZZfzuZ2Uii6Yb+/fsXYNMi0hpr10YeXnllZOUNN8B3vtP0HBaJ6Ns3jsrPOSeKnjo1/gI9/nhMRGIWFyy10dSBBfWTnxRlM4UI94VAwxl/+wKLsq3o7qOB0RBD/hZg2yLSQlOnRlP3nDnR1fHGG1s3VWjRdO4M//VfsVxzTZyQXbw4BiyT/1eI1rRJwNdTvWYGAcvdXU0yIiVq+fI4Ot9//zh5+sAD0YRdFsGezdZbK9izyHnkbmZ3A4OBnma2ELgc6Azg7qOAR4FhwHzgE+CstipWRFrOPXoXfve78P77cfuzn5VHS4Y0X85wd/cROV534DsFq0hECmr9enjkkWhPf+qpOFE6adKGq+GlMmkOVZEK9cEH0Ynk5ptjWPM+fWKAxPPPj84oUtn0FYtUmOnT4wLNCRPios3Bg+OCpKOOSmaGOUmGwl2kAqxaFWH++9/HuFTdusVIt9/+NuyyS9LVSRIU7iJlyj2u1r/99mh++fDDuLr0pptiiJXNN0+6QkmSwl2kjCxbFoMp1tTE9TsLF8YFR0cfHd0b//u/S2f4ckmWwl2khK1fH2NoPf54BPrUqTFi4+abxxwYl14ao+b26ZN0pVJqFO4iJWTdOvjXv+Dvf49AnzwZliyJo/GqqrjSfujQGP5cPV6kKfrfQyQB7tGkMnduXP6fvp03L3q4QFx4OXx4zEh36KHQs2eyNUt5UbiLtJH6+uhr/s47sSxYECGeXhqOWNunT4xoO2RITEi0557xWO3n0lIKd5FmcI/JgZYvj5ObS5duCO9sy9q1n33/FltEaJ96aoT4rrtGV8Uttkhmf6RyKdylpLnHUl8fS8P79fVxwnHduvyW1avh009jWbVqw/3M51atghUrIrzTId7wtr4+e61du8YReJ8+8JWvbLifXvr3j6YWHY1LMZRduNfUwEUXff75xmbfyvZ8MddtzfubetzS11ryOY2tn/l6Y7fp++kl1+OGgd7UrGptYaONYOONY+a37t2hR48YLXHgwA2PM2+32SbW6d5dwS2lo+zCffPN4x9aNo39w8r2fDHXbc37m3rc0tda8jmNrZ/5emO36fvpJdfjDh0+v2Q+bxY9RvJZOnaM0M62bLJJ3HbpUuQp5UTaUNmF+377FWWGKhGRsqbjFBGRCqRwFxGpQAp3EZEKpHAXEalACncRkQqkcBcRqUAKdxGRCqRwFxGpQObFvr47vWGzOuDNFr69J/BBAcspJZW6b9qv8lOp+1bu+/Uld++Va6XEwr01zKzW3auSrqMtVOq+ab/KT6XuW6XuVyY1y4iIVCCFu4hIBSrXcB+ddAFtqFL3TftVfip13yp1vz6jLNvcRUSkaeV65C4iIk0ou3A3s6Fm9qqZzTezi5Oup1DMbIGZzTGz2WZWm3Q9rWFmt5nZYjOb2+C5L5jZZDN7PXVbdrOGNrJfV5jZO6nvbbaZDUuyxpYws35m9rSZzTOzl83sgtTzlfCdNbZvZf+95VJWzTJm1hF4DTgUWAjMAEa4+yuJFlYAZrYAqHL3cu5/C4CZHQisBO5w94Gp534JLHH3a1N/lLdw9x8nWWdzNbJfVwAr3f1XSdbWGmbWG+jt7rPMbDNgJnAMcCbl/501tm8nUubfWy7lduS+DzDf3f/t7muA8cDRCdckGdz9WWBJxtNHA7en7t9O/AMrK43sV9lz93fdfVbq/kfAPKAPlfGdNbZvFa/cwr0P8HaDxwupnC/Kgb+a2UwzG5l0MW3gi+7+LsQ/OGCrhOsppPPN7KVUs03ZNV00ZGYDgD2BaVTYd5axb1BB31s25Rbu2aaaLp92paYd4O57AUcA30k1AUjpuxnYHtgDeBf4dbLltJyZdQMmAhe6+4qk6ymkLPtWMd9bY8ot3BcC/Ro87gssSqiWgnL3RanbxcD9RBNUJXk/1f6ZbgddnHA9BeHu77v7enevB/5ImX5vZtaZCL873f2+1NMV8Z1l27dK+d4hPznFAAAA70lEQVSaUm7hPgPYwcy2NbMuwMnApIRrajUz2zR1sgcz2xQ4DJjb9LvKziTgjNT9M4AHE6ylYNLhl3IsZfi9mZkBtwLz3P36Bi+V/XfW2L5VwveWS1n1lgFIdVn6LdARuM3dr064pFYzs+2Io3WATsBd5bxfZnY3MJgYfe994HLgAeAeoD/wFnCCu5fVyclG9msw8dPegQXAN9Pt1OXCzL4CTAHmAPWpp39KtE2X+3fW2L6NoMy/t1zKLtxFRCS3cmuWERGRPCjcRUQqkMJdRKQCKdxFRCqQwl1EpAIp3EVEKpDCXUSkAincRUQq0P8BEyPBK9Ho7JcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(D_list_cost,color = 'r')\n",
    "plt.plot(G_list_cost,color = 'b')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 209
    },
    "colab_type": "code",
    "id": "Sub4goywbfRI",
    "outputId": "c94ef91e-84fa-48dc-f021-151b415f562b"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPMAAAD7CAYAAAC/paJSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnV9sVdfZ5p99fGqMj019Csb8M6aGYlywCf8UBWaKESg0UTojMYiP5LvhItzlOlyMonw3EdHczF3ETaVJGBT0ERKUi0ykyBBSE1FGEJBxDYbiui0Fp7VNTYzBNpy5oL+17WNnT7eTfJi13keKTmz22Wc/+3g/73r/rqhQKMhgMDz9yDzpCzAYDN8P7GE2GDyBPcwGgyewh9lg8AT2MBsMnsAeZoPBE9jDbDB4gh/sYY6i6Jtx/z2Komh43M//GkXRv0VRNFp03J1x7/+vURRdjKJoMIqiv0VR1BpF0bIoig6NO36k6Bz/54fiExq/EDh6x69QKPzg/0n6g6QdRb/7N0n/+1uOXyHp75K2S4okVUr6b5KW/rPn+I/8z3d+IXD0gV82/eP/H4JnJHUXCoXWf/x8V9LxJ3g93zd85yf5z3HG8ZupPvMFSauiKPqfURRti6Ko4klf0PcM3/lJ/nOccfye9MO8J4qiO+P+OyVJhULhhqQWSYsl/bukv0VR9L9mwg1LCd/5Sf5zfGr4PemH+d8LhULVuP+28Q+FQuFsoVDYUygUqiX9Z0m/kPTfn9iVTg++85P85/jU8HvSD/M/hUKh8H8lfShpzZO+lh8CvvOT/Oc4E/jNyIc5iqL/FEXR/iiK5v/j51WS/ouks0/2yr4f+M5P8p/jTOT3pB/mfynK4X3zj5tzR49vTHsURd9I+lTSR5L+x5O82GnAd36S/xyfGn7RP3JhBoPhKceTtswGg+F7gj3MBoMnsIfZYPAE9jAbDJ7AHmaDwROkarRob28vSNKPfvQjSVJFxePKtW+++UaSNDg4qJqaGknS119/LUkaGxuTJD18+FCSlM/nJUm3b9/W+HPNnj1bkpTL5dz5fvzjH0uSSkpKJEn9/f2SpMWLF0uSRkdHJUlRFEmS/va3v024rkwmo7/+9a+SpLlz5054T2VlpSSprq4uCoVfCBx955eEVA8zBObNmydJ+vOf/zzh52w2q5s3b0qSqqqqJEn37993Fy1JfX19kqT58+dLim/grFmz3Dk4FsJ/+MMfJE2+Idzcv/zlL5Kkn//855KkP/7xj5Ief0nc5AcPHkiSysrKJvwcEr8QOPrOLwm2zDYYPEEqy4zZZymBgrAsyGQybqkwODgoKVY1lLG8vFxSvCxhqXP37l133IoVKyTJKSjHLlq0SFKsnCyHli5dKkm6evWqJOknP/mJJKm3t9cpJQqJck9VLOM7vxA4+s4vCWaZDQZPkMoyoyCozaNHjyTFa/tcLufUa3h4WJI0MjIiKVZGfIslS5ZIitWwtLTUHXfw4EFJ0sqVKyVJCxculCT97Gc/m/Ae1BAlw6dBacvLy12wY2hoSFKsfnxeSPxC4Og7vySYZTYYPEGqRosrV64UJLlQOqqHsmSzWXV3d0uS5syZIymOFPKeBQsWSIr9EPyXO3fuuPdxXlSWY6urqyVJ27dvlxQrGeqHovb29kp67L/8/ve/lyTV1dVJiv0hrquhocGF/X3nFwJH3/klIdUym7wbF8xSgiBAJpPRqlWrJMUBAEL4pAE4tjh10NDQ4D7j73//uyTpT3/604RzLV++XFKc9/vpT38qSTpz5oykOPyfzWbdubgevgjOTWAkJH4hcPSdXxJsmW0weIJUlrnYeWfJwO9LSkrccgNVQ5EIFIxP3ktxiP7ChQuSpMuXL7uKHJSKxPzZs4+HOBBkyOVykuLgBudiaXP+/Hm3lEJ1OTdBj5D4hcDRd35JMMtsMHiCVJYZBWFNj8oQ4r9z545LmlOzum3b42GGnZ2dE85x7NgxSXFggCBAc3OzLl68KCkuiaPOFXVFWa9cuSJJunfv3oTPIKQ/MDDgfJeBgQFJcv4JHELiFwJH3/klwSyzweAJptVoQXId4DfMnz/fJetRFRQJpfr1r38tSfrNb34jSdqyZYukiYpWW1srKVYqfBgK30nQkx7o6emRFCsoPkdTU5ML8zc2NkqKI5Mh8guBo+/8kmCW2WDwBNOKZhPlI9qG0g0NDbkCd1Tut7/9raS4aBxlWr16taS4JQxVGhwc1O9+9ztJ0ubNmyXFSXwUks/r6OiQJO3fv19S7ONwfGNjo/OZKAggp4gPFRK/EDj6zi8JZpkNBk+QyjKTK0P1inN6uVzO+Sr4CEQGif4RzaNEjtwavkdXV5drY6N6hldK52hdI+pHzm7Xrl2SpPb2dne95P/wmVA7fJqQ+IXA0Xd+STDLbDB4glSWGT+guJ2LetjZs2e736Fm1KKSj2tubpYU+yfLli2TFPs2VVVVLlLIOfAhiD7yXpSTnB1RQH4fRZFTOc6Br4IKhsQvBI6+80uCWWaDwROkssxUqqAydIyMH4hWXLOK6pw+fVpS3N514MABSXFejuqanTt3uq4VamE5P5U7fAbnOnnypKS4ood62Orqatd5QsRwfFN4aPxC4Og7vySYZTYYPEEqy4yCEZljbQ/6+/udIvLa2toqKfYhUK5Tp05Jin0a+kAPHz7slAnF4nOLh7MdOXJEklRfXy8pVj06Ze7fv+8qc7hmoo18bkj8QuDoO78kpHqYWRrgqBMwIB1QWVnpliRHjx6dcCxLGsgwN5gUwrlz59wrSxRSBhs3bpQU32R+z8QIzl1Mvr6+3n0OhfB8gVNNWPGdXwgcfeeXBFtmGwyeIJVlpiCd6YEoGCH7XC7nggSE3vft2ycpLnMjNM+ShWbu48ePS3qsoOvWrZMkbdiwQVLc6E1innQAwQfSDoxu4fpmzZrlggoEQlh2TbVbgO/8QuDoO78kmGU2GDzBtOZmM3cY/wRcvXrVrfNRREaxXLt2TZK0Y8cOSfGoFnwPfI3u7m6nonv37pUUq1pXV5ekuACeV3wgrovfP3r0yPk7hPmLC/BD4hcCR9/5JcEss8HgCVJZZhSN6B8Jetb4nZ2d7hjGkhLC37Rpk6TYT0Ex2Xunra1N0sREOeF7XgEqyDlI4KOooLe31/lQ69evlxS3mU21W4Dv/ELg6Du/JJhlNhg8wbQsM03d7MWDCq5evdq1fhXn4YjY4VNcunRJUuwXjN8cm/PduHFDUpyboyEchSRySKsY59yzZ4+7XvKMRCjHRxFD4xcCR9/5JcEss8HgCaZlmQEqhO8xNjbmCs5RFfaxRX2uX78uKa6uIZfW0tIiSTpx4oQbpIZS0qKGL8OQNMa4kJ9DQcdX1DDW5datWxOOTVJ1X/mFwNF3fkkwy2wweIJUlhnghxAFJPpWWVnp1vtE6FAshqhRVI7akXdDLXt6elyDefEAcY7ZvXu3pDjKiLKyWRhF77W1ta7ah5EvXF9SdY3v/ELg6Du/qWCW2WDwBNNqgSRSyJqeqptMJuNUZenSpZLipmzGmQL2okU56WCpra11o04ZzVK8BQh+ECNQUTuqahgZc/fuXXdt5Pt4hUNI/ELg6Du/JExrmf3PYPwy58GDByopKXEX/fzzz2twcFBnzpyZNOGf9927d09ffvmlW76Ul5crn8/r2rVrLgFfKBRUKBRccn3Tpk1uN4IfGr7zC4Gjb/ymFc1G2ag3RUEymYyL0hHt6+rq0muvvabnnntOS5Ys0fXr1zUyMqLh4WEtX75cL730kqTYPxgbG1NfX58+/vhjlZaWqqKiQgsWLNDQ0JBqamq0YsUKbd26VZL06aef6ubNm3rzzTfd9RQKBTe6NIoi1/CNP8QXQ24xJH4hcPSdXxJ+MMv8XdDf36/KykoX3i8pKdGcOXPcrOKnHb7zk/znOBP5pXqYWSoQsaMihp/nzp3ramBRnrq6OmWzWS1evFgrV650lTAnT57UyMiIG6dCdLC7u1uNjY1qa2vTli1b1NTUpHXr1qmsrGzSBtaff/65SktLnYLhL6Fso6OjbnwL41w4Fj8oJH4hcPSdXxJSPcyE6GkdI2CAkz8yMjJpbnBpaamiKFJzc7OeffZZF/Zva2tTR0eHXn/9dUmPlxv19fV66623tHDhQh06dEjvvvuu3nvvPR06dEhbt27Vyy+/rLKyMhd0KCkpUSaTcTeD5QpfUj6fd0sXvkCaw3t7eyVJa9euDYZfCBx955eEJ7rMXrt2rV599VVJk7fiaGpq0htvvCHp8eZbb7/9tk6cOOH6R58G+M5P8p/j08Qv1cOMP8CShTA8Yfnq6mrXwI2q0C5WU1Ojuro6FxFcuXKlysrK9Mwzz0iKI4XV1dVuKcIyY8OGDWppadHNmzfV0NDgjh0bG9PDhw/dsoRIJCpYKBTc+BbOhY/DtYfELwSOvvNLwowsGvnqq6/04YcfuomGPT09amtr05o1a57wlX0/8J2f5D/HmcgvlWVG3QgEgJqaGkmP824oEq1f2WzWOftSvCuAJH3yySf67LPPJpzr/fffV1VVlb744gu98847Gh4eVj6f1wsvvKBXXnlFo6OjTklRPXwMVI+0xMDAgAsq8G+kF6Zq/PadXwgcfeeXhCjNbN7W1taCFN8olgV0ioy/URzDPCMumFpVKmJQNrpJHj58OOn8vJdjuVHFQ8OLb9To6OikG8U5WI41NTVFofALgaPv/JKQyjJDjnA/aka1Sz6fn1TYTosYY1VYhqBURBYpRO/r63MVNfgZxbv2cW5SBvyMr8H7Kyoq3O+4eXwJHBMSvxA4+s4vCTPSZzYYDOmRapltMBhmLswyGwyewB5mg8ETpAqAtbe3F6SJUwqluDdzcHDQpQDoUiFySNAB5565wpwL5z+Xy7nzUZJHEIOyN5LpJN6JApKW4LoymYwLPBDN5D1ECuvq6lyk0Hd+IXD0nV8SUj3MEKB2ldGg/JzNZl3UkKje+B3rpThCSGsaN5BoYDabdcdCmChj8Q3h5jL2hfpbWtzGxsbcTSZ3RxpgqpEsvvMLgaPv/JJgy2yDwRNMqzabpUTxLvGZTMYtFRg0jqqhjCTTUSOWOnS7zJs3z+XzUFCOpaYW5WQ5xPgXkv3kDXt7eycl6VHuqaL4vvMLgaPv/JJgltlg8ATT2tIVtaFyhbV9Lpdz6kVXCSVwKCO+BXOUUEPqUEdHR3Xw4EFJ8VabtJ7R+M17UEOUDJ8GpS0vL3fBDsrqUL+p6l595xcCR9/5JcEss8HgCVJVgF25cqUgTa4zRVmy2ay6u7slxTWrRAp5D9MU8EPwXygunzNnjjsvKsux1N1u375dUqxkqB+KSp/qokWLXAcMY1Pxh7iuhoYGF/b3nV8IHH3nl4RUy2zyblwwSwmCAJlMRqtWrZIUBwAI4ZMG4Nji1AF7Ad2+fdsVtrPnDudavny5pDjvRwsbO9wT/qcL5vbt2+56+CI4N4GRkPiFwNF3fkmwZbbB4AlSWeZi550lA78vKSlxyw1UDUUiUDA+eS/FIfoLFy5Iki5fvuwqclAqEvNnz56VpEkTEAlucC6WNufPn3dLKVSXcxP0CIlfCBx955cEs8wGgydIZZlRENb0qAwh/jt37rikOTWr27ZtkyR1dnZOOMexY8ckxYEBggDNzc26ePGipMn796CuKGvxDnx8BiH9gYEB57uwwx7+CRxC4hcCR9/5JcEss8HgCabVaEFyHeA3zJ8/3yXrURUUCaVi0yz2+dmyZYukiYrGAHGUCh+GwncS9KQHenp6JMUKis/R1NTkwvzsYE9kMkR+IXD0nV8SzDIbDJ5gWtFsonxE21C6oaEhV+COyrE9CMUpKBP71NIShioNDg66vW03b94sKU7io5B8XkdHhyRp//79kmIfh+MbGxudz0RBADlFfKiQ+IXA0Xd+STDLbDB4glSWmVwZqlec08vlcs5XwUcgMkj0j2geJXLk1vA9urq6XBsb1TO8UjpH6xpRP3J2u3btkiS1t7e76yX/h8+E2hXvGxQCvxA4+s4vCWaZDQZPkMoy4wcUt3NRDzt79uxJw8CpRSUf19zcLCn2T5YtWyYp9m2qqqpcpJBz4EMUb8WJcpKzIwrI76MocirHOfBVUMGQ+IXA0Xd+STDLbDB4glSWmUoVVIaOkfED0YprVlGd06dPS4rbuw4cOCApzstRXbNz507XtUItLOencofP4FwnT56UFFf0UA9bXV3tOk+IGI5vCg+NXwgcfeeXBLPMBoMnSGWZUTAic6ztQX9/v1NEXltbWyXFPgTKderUKUmxT0Mf6OHDhyftrMfnFg9nO3LkiKR4cy5Uj06Z+/fvu8qc4t34+NyQ+IXA0Xd+SUj1MLM0wFEnYEA6oLKy0i1Jjh49OuFYljSQYW4wKYRz5865V5YopAw2btwoKb7J/J6JEZy7mHx9fb37HArh+QKnmrDiO78QOPrOLwm2zDYYPEEqy0xBOtMDUTBC9rlczgUJCL3v27dPUlzmRmieJQvN3MePH5f0WEHXrVsnSdqwYYOkuNGbxDzpAIIPpB0Y3cL1zZo1ywUVCISw7JpqtwDf+YXA0Xd+STDLbDB4gmnNzWbuMP4JuHr1qlvno4iMYrl27ZokaceOHZLiUS34Hvga3d3dTkX37t0rKVa1rq4uSXEBPK/4QFwXv3/06JHzdwjzFxfgh8QvBI6+80uCWWaDwROksswoGtE/EvSs8Ts7O90xjCUlhL9p0yZJsZ+CYrL3Tltbm6SJiXLC97wCVJBzkMBHUUFvb6/zodavXy8pbjObarcA3/mFwNF3fkkwy2wweIJpWWaautmLBxVcvXq1a/0qzsMRscOnuHTpkqTYLxi/OTbnu3HjhqQ4N0dDOApJ5JBWMc65Z88ed73kGYlQjo8ihsYvBI6+80uCWWaDwRNMyzIDVAjfY2xszBWcoyrsY4v6XL9+XVJcXUMuraWlRZJ04sQJN0gNpaRFDV+GIWmMcSE/h4KOr6hhrMutW7cmHJuk6r7yC4Gj7/ySYJbZYPAEqSwzwA8hCkj0rbKy0q33idChWAxRo6gctSPvhlr29PS4BvPiAeIcs3v3bklxlBFlZbMwit5ra2tdtQ8jX7i+pOoa3/mFwNF3flPBLLPB4Amm1QJJpJA1PVU3mUzGqcrSpUslxU3ZjDMF7EWLctLBUltb60adMpqleAsQ/CBGoKJ2VNUwMubu3bvu2sj38QqHkPiFwNF3fkmY1jL7n8H4Zc6DBw9UUlLiLvr555/X4OCgzpw5M2nCP++7d++evvzyS7d8KS8vVz6f17Vr11wCvlAoqFAouOT6pk2b3G4EPzR85xcCR9/4TSuajbJRb4qCZDIZF6Uj2tfV1aXXXntNzz33nJYsWaLr169rZGREw8PDWr58uV566SVJsX8wNjamvr4+ffzxxyotLVVFRYUWLFigoaEh1dTUaMWKFdq6dask6dNPP9XNmzf15ptvuuspFApudGkURa7hG3+IL4bcYkj8QuDoO78k/GCW+bugv79flZWVLrxfUlKiOXPmuFnFTzt85yf5z3Em8kv1MLNUIGJHRQw/z50719XAojx1dXXKZrNavHixVq5c6SphTp48qZGRETdOhehgd3e3Ghsb1dbWpi1btqipqUnr1q1TWVnZpA2sP//8c5WWljoFw19C2UZHR934Fsa5cCx+UEj8QuDoO78kpHqYCdHTOkbAACd/ZGRk0tzg0tJSRVGk5uZmPfvssy7s39bWpo6ODr3++uuSHi836uvr9dZbb2nhwoU6dOiQ3n33Xb333ns6dOiQtm7dqpdfflllZWUu6FBSUqJMJuNuBssVvqR8Pu+WLnyBNIf39vZKktauXRsMvxA4+s4vCU90mb127Vq9+uqrkiZvxdHU1KQ33nhD0uPNt95++22dOHHC9Y8+DfCdn+Q/x6eJX6qHGX+AJQtheMLy1dXVroEbVaFdrKamRnV1dS4iuHLlSpWVlemZZ56RFEcKq6ur3VKEZcaGDRvU0tKimzdvqqGhwR07Njamhw8fumUJkUhUsFAouPEtnAsfh2sPiV8IHH3nl4QZWTTy1Vdf6cMPP3QTDXt6etTW1qY1a9Y84Sv7fuA7P8l/jjORXyrLjLoRCAA1NTWSHufdUCRav7LZrHP2pXhXAEn65JNP9Nlnn0041/vvv6+qqip98cUXeueddzQ8PKx8Pq8XXnhBr7zyikZHR52Sonr4GKgeaYmBgQEXVODfSC9M1fjtO78QOPrOLwlRmtm8ra2tBSm+USwL6BQZf6M4hnlGXDC1qlTEoGx0kzx8+HDS+Xkvx3KjioeGF9+o0dHRSTeKc7Aca2pqikLhFwJH3/klIZVlhhzhftSMapd8Pj+psJ0WMcaqsAxBqYgsUoje19fnKmrwM4p37ePcpAz4GV+D91dUVLjfcfP4EjgmJH4hcPSdXxJmpM9sMBjSI9Uy22AwzFyYZTYYPIE9zAaDJ0gVAGtvby9IE6cUSnFv5uDgoEsB0KVC5JCgA849c4U5F85/Lpdz56MkjyAGZW8k00m8EwUkLcF1ZTIZF3ggmsl7iBTW1dW5SKHv/ELg6Du/JKR6mCFA7SqjQfk5m826qCFRvfE71ktxhJDWNG4g0cBsNuuOhTBRxuIbws1l7Av1t7S4jY2NuZtM7o40wFQjWXznFwJH3/klwZbZBoMnmFZtNkuJ4l3iM5mMWyowaBxVQxlJpqNGLHXodpk3b57L56GgHEtNLcrJcojxLyT7yRv29vZOStKj3FNF8X3nFwJH3/klwSyzweAJprWlK2pD5Qpr+1wu59SLrhJK4FBGfAvmKKGG1KGOjo7q4MGDkuKtNmk9o/Gb96CGKBk+DUpbXl7ugh2U1aF+U9W9+s4vBI6+80uCWWaDwROkqgC7cuVKQZpcZ4qyZLNZdXd3S4prVokU8h6mKeCH4L9QXD5nzhx3XlSWY6m73b59u6RYyVA/FJU+1UWLFrkOGMam4g9xXQ0NDS7s7zu/EDj6zi8JqZbZ5N24YJYSBAEymYxWrVolKQ4AEMInDcCxxakD9gK6ffu2K2xnzx3OtXz5cklx3o8WNna4J/xPF8zt27fd9fBFcG4CIyHxC4Gj7/ySYMtsg8ETpLLMxc47SwZ+X1JS4pYbqBqKRKBgfPJeikP0Fy5ckCRdvnzZVeSgVCTmz549K0mTJiAS3OBcLG3Onz/vllKoLucm6BESvxA4+s4vCWaZDQZPkMoyoyCs6VEZQvx37txxSXNqVrdt2yZJ6uzsnHCOY8eOSYoDAwQBmpubdfHiRUmT9+9BXVHW4h34+AxC+gMDA853YYc9/BM4hMQvBI6+80uCWWaDwRNMq9GC5DrAb5g/f75L1qMqKBJKxaZZ7POzZcsWSRMVjQHiKBU+DIXvJOhJD/T09EiKFRSfo6mpyYX52cGeyGSI/ELg6Du/JJhlNhg8wbSi2UT5iLahdENDQ67AHZVjexCKU1Am9qmlJQxVGhwcdHvbbt68WVKcxEch+byOjg5J0v79+yXFPg7HNzY2Op+JggByivhQIfELgaPv/JJgltlg8ASpLDO5MlSvOKeXy+Wcr4KPQGSQ6B/RPErkyK3he3R1dbk2NqpneKV0jtY1on7k7Hbt2iVJam9vd9dL/g+fCbUr3jcoBH4hcPSdXxLMMhsMniCVZcYPKG7noh529uzZk4aBU4tKPq65uVlS7J8sW7ZMUuzbVFVVuUgh58CHKN6KE+UkZ0cUkN9HUeRUjnPgq6CCIfELgaPv/JJgltlg8ASpLDOVKqgMHSPjB6IV16yiOqdPn5YUt3cdOHBAUpyXo7pm586drmuFWljOT+UOn8G5Tp48KSmu6KEetrq62nWeEDEc3xQeGr8QOPrOLwlmmQ0GT5DKMqNgROZY24P+/n6niLy2trZKin0IlOvUqVOSYp+GPtDDhw9P2lmPzy0eznbkyBFJ8eZcqB6dMvfv33eVOcW78fG5IfELgaPv/JKQ6mFmaYCjTsCAdEBlZaVbkhw9enTCsSxpIMPcYFII586dc68sUUgZbNy4UVJ8k/k9EyM4dzH5+vp69zkUwvMFTjVhxXd+IXD0nV8SbJltMHiCVJaZgnSmB6JghOxzuZwLEhB637dvn6S4zI3QPEsWmrmPHz8u6bGCrlu3TpK0YcMGSXGjN4l50gEEH0g7MLqF65s1a5YLKhAIYdk11W4BvvMLgaPv/JJgltlg8ATTmpvN3GH8E3D16lW3zkcRGcVy7do1SdKOHTskxaNa8D3wNbq7u52K7t27V1Ksal1dXZLiAnhe8YG4Ln7/6NEj5+8Q5i8uwA+JXwgcfeeXBLPMBoMnSGWZUTSifyToWeN3dna6YxhLSgh/06ZNkmI/BcVk7522tjZJExPlhO95Bagg5yCBj6KC3t5e50OtX79eUtxmNtVuAb7zC4Gj7/ySYJbZYPAE07LMNHWzFw8quHr1atf6VZyHI2KHT3Hp0iVJsV8wfnNsznfjxg1JcW6OhnAUksghrWKcc8+ePe56yTMSoRwfRQyNXwgcfeeXBLPMBoMnmJZlBqgQvsfY2JgrOEdV2McW9bl+/bqkuLqGXFpLS4sk6cSJE26QGkpJixq+DEPSGONCfg4FHV9Rw1iXW7duTTg2SdV95RcCR9/5JcEss8HgCVJZZoAfQhSQ6FtlZaVb7xOhQ7EYokZROWpH3g217OnpcQ3mxQPEOWb37t2S4igjyspmYRS919bWumofRr5wfUnVNb7zC4Gj7/ymgllmg8ETTKsFkkgha3qqbjKZjFOVpUuXSoqbshlnCtiLFuWkg6W2ttaNOmU0S/EWIPhBjEBF7aiqYWTM3bt33bWR7+MVDiHxC4Gj7/ySMK1l9j+D8cucBw8eqKSkxF30888/r8HBQZ05c2bShH/ed+/ePX355Zdu+VJeXq58Pq9r1665BHyhUFChUHDJ9U2bNrndCH5o+M4vFI4+YVrRbJSNelMUJJPJuCgd0b6uri699tpreu6557RkyRJdv35dIyMjGh4e1vLly/XSSy9Jiv2DsbEx9fX16eOPP1ZpaakqKiq0YMECDQ0NqaamRitWrNDWrVslSZ9++qlu3rypN998011PoVBwo0ujKHIN3/hD/OGRWwzUosLvAAAMTklEQVSJXwgc0/DD4jJIAN+YaDadUFj3F1980fHjPR999JGkuG6bV/iRX+aaub7v8h1+G34wy/xd0N/fr8rKShfeLykp0Zw5c9ys4qcdvvOTwuA405DqYWYpRMSOihh+njt3rquBRXnq6uqUzWa1ePFirVy50inVyZMnNTIy4sapEB3s7u5WY2Oj2tratGXLFjU1NWndunUqKyubtIH1559/rtLSUqdg+Eso2+joqFNQxrlwLKocEr8QOE6XnxR3TxUPn4fXeH70OlNPTX752zZZL95y5rt8h9+GVA8zyxBaxwgY4OSPjIxMmhtcWlqqKIrU3NysZ5991oX929ra1NHRoddff13S4+VGfX293nrrLS1cuFCHDh3Su+++q/fee0+HDh3S1q1b9fLLL6usrMwFHUpKSpTJZNzNYLnCl5TP593ShS+Q5vDe3l5J0tq1a4PhFwLH6fKTpF/84heS4tTUBx984K5Rih/Q5uZm98AT2CJeQCko/HjIi2eTfZfv8NvwRJfZa9eu1auvvippsho2NTXpjTfekPR48623335bJ06ccP2jTwN85yeFwfFpQaqHGX+HJQtheMLy1dXVLgCAqtAuVlNTo7q6OqdgK1euVFlZmZ555hlJsbJVV1c7FWOZsWHDBrW0tOjmzZtqaGhwx46Njenhw4duWUKkFZUvFApufAvnwofj2kPiFwLH78KPpTPX9qtf/WrCe5P4YWmLLTXX8X1+h9+GGVk08tVXX+nDDz90Ew17enrU1tamNWvWPOEr+37gOz8pDI4zDaksMwpFIADU1NRIepxXRJFo/cpms87nkOJdASTpk08+0WeffTbhXO+//76qqqr0xRdf6J133tHw8LDy+bxeeOEFvfLKKxodHXVKiqrjY6B6pCUGBgZcUIF/IyAxVeO37/xC4DhdfuMBP6ws/47F/Prrr935OS/XiHjBDx+ZBovv4zv8NkRpZvO2trYWpPhGsSzAyR9/oziGG8EFU6vKjYI8ZB8+fDjp/P+/G8VNKL5Ro6Ojk24U52A51tTUFIXCLwSOvvNLQirLDDnC/ag11Tz5fH5SYTstYoxVYZmFEhNZpBC9r6/PVQzhZxTv2se5SYnwM8rJ+ysqKtzvuHl8CRwTEr8QOPrOLwkz0mc2GAzpkWqZbTAYZi7MMhsMnsAeZoPBE6QKgLW3txekiVMKpbg3c3Bw0IXq6VIhckjQAeeeucKcC+c/l8u581GSRxCDsjeS6STeiQKSluC6MpmMCzwQzeQ9RArr6upcpNB3fqFwDBWpHma+IGpXGQ3Kz9ls1kUNieqN37FeiiOEtKbxB0I0MJvNumP5QokyFn/h/PEw9oX6W1rcxsbG3B8RuTvSAFONZPGdXygcQ4Utsw0GTzCt2myWSsW7xGcyGbcUYtA4qo3yk0xHbVnK0e0yb948l8/DQnAsNbVYBpZ7jH8h2U/esLe3d1KSHss0VRTfd36hcAwVZpkNBk8wrS1dUVMqV/BdcrmcU2e6SiiBQ/nxnZgThdpThzo6OqqDBw9KmtwsTuM370HtUWp8NixJeXm5C+ZQVoe6T1X36ju/UDiGCrPMBoMnSFUBduXKlYI0uc4U5cxms+ru7pYU16wSCeU9TFPAz8I/o7h8zpw57rxYEY6l7nb79u2SYqVG3bEY9KkuWrTIdcAwGgZ/j+tqaGhwaQ3f+YXCMVSkWmaTV+QLYalEkCOTyWjVqlWS4gAHKQrSHBxbnBphL6Dbt2+7wnb23OFcNI+T16SFjR3uSW/QBXP79m13PfyhcW4CPyHxC4VjqLBltsHgCVJZ5uLgBEsifl9SUuKWU6g2iksgZHxxghSnIC5cuCBJunz5sqs4QokpPDh79qykb5+AyLlYup0/f94tFbEqnJugTkj8QuEYKswyGwyeIJVlRiHxWVBRUhh37txxRQHU5G7btk2S1NnZOeEcx44dkxQHPghyNDc36+LFi5Im79+D9cByFO/Ax2eQshgYGHC+GTvs4X/BISR+oXAMFWaZDQZPMK1GC4oHAH7R/PnzXTECqoniosRsCsY+Rlu2bJE0UbEZII4S46NR2E8BAumPnp4eSbGFwKdqampyaQx2sCfyGiK/UDiGCrPMBoMnmFY0mygm0USUfGhoyBXwo+JsD0JxCsrLPrW0vKG6g4ODbm/bzZs3S4qLFLAAfF5HR4ckaf/+/ZJiH47jGxsbnU9IwUPxnj8h8QuFY6gwy2wweIJUlplcIKpenLPM5XLOF8MHIvJJdJNoJSWA5A7xrbq6ulybHtVBvFIaSGseUU1ykrt27ZIktbe3u+slv4lPiJoX74sUAr9QOIYKs8wGgydIZZnxc4rb1aj3nT179qRh4NTakm9sbm6WFPtfy5YtkxT7blVVVS4SyjnwkYq34sQykJMkysnvoyhyKs458MVQ+ZD4hcIxVJhlNhg8QSrLTCUOKkpHzPiBb8U1uajq6dOnJcXtawcOHJAU5x2pHtq5c6fryqHWl/NTmcRncK6TJ09KiiuWqPetrq52nTVERMc3vYfGLxSOocIss8HgCVJZZhSayCO+C+jv73eKz2tra6uk2EdCmU+dOiUp9tnocz18+PCknfX43OLhc0eOHJEUb86FqtMJdP/+fVd5VLwbH58bEr9QOIaKVA8zSx8CEQRESHdUVla6JdfRo0cnHMuSjS+LucikSM6dO+deWYKREtm4caOk+I+I3zMRg3MXf7n19fXucyj05w90qgkrvvMLhWOosGW2weAJUllmCu6ZjohCk5LI5XIuCEJqYd++fZLiMj5SDyzJaFY/fvy4pMcWYt26dZKkDRs2SIob2Sk8IN1BcIW0CqNpuL5Zs2a5oAmBHpaVU+2G4Du/UDiGCrPMBoMnmNbcbOYq43+Bq1evOj8GxWfUzLVr1yRJO3bskBSPosG3wpfq7u52VmLv3r2SYtXu6uqSFBf484qPx3Xx+0ePHjl/jjRGcYNBSPxC4RgqzDIbDJ4glWVGsYluUoCAD9PZ2emOYewqKYpNmzZJiv0wLAJ7C7W1tUmaWAhAeoJXgMpzDgoUsBigt7fX+Yjr16+XFLfRTbUbgu/8QuEYKswyGwyeYFqWmaZ19hpC5VevXu1a24rzjEQk8ZkuXbokKfZ7xm/+zflu3LghKc490vCOBSAySisc59yzZ4+7XvKoRGDHR0lD4xcKx1Bhltlg8ATTsswAlcW3GhsbcwX1qCb79KKu169flxRXD5ErbGlpkSSdOHHCDYrDEtCCh6/GEDjG1JB/xEKMrxhibM2tW7cmHJtkmX3lFwrHUGGW2WDwBKksM8DPIspJdLGystL5M0QgUWSGxFE0j5qTV8Qa9PT0uAb64gHpHLN7925JcRQVy8FmaBT119bWumomRtpwfUnVQ77zC4VjaDDLbDB4gmm1QBIJxWehqiiTyTjVXLp0qaS46ZxxrYC9drEMdOjU1ta6Ua6Mnine4gQ/jxGvqDlVQ4zEuXv3rrs28pm8wiEkfqFwDBVmmQ0GTzCtaDbKTT0tCpnJZFwUErWmCR2/ikgoXTRYhhdffFHS42gq7/noo48kxTW/vG7dulVSnJvECnB9jGaNosg1tOPvcSy505D4hcIxVJhlNhg8QSrLTC0sEUkqfvh57ty5rsYXZcWvovOmeHA5UU9eu7u7XZ8stbjkJr9tg+7i7UpQ7tHRUWchGFeDmmN1QuIXCsdQEaUZvdLW1laQ4tY40goEMUZGRlxzOTecLwWQ1vjggw8kxcs9/kAymYz7f2YsU1bIMougCn8gxXOt+CPM5/PuPQRYuL7e3l5J0i9/+csoFH6hcAwVtsw2GDxBKst86dKlghSX3KG2LINQTilWTdrhWHbxHor5SVnw++rq6knpE5ZkWACOZYnIvkYEaMZfT3HLHyrPcrOpqcmpuu/8QuEYKswyGwyeIFUADAUm0AFqamokPS7ZQ3FpbWPEC6A0DwvAv1N4//XXX7vzc14CH4xjxVLgX2FlKC5A/QcGBtx7+TcCLlM1tvvOLxSOocIss8HgCVJZZpQYP4coJyNf8vn8pMJ9WuAYG7NmzRpJcUSSaCeF9n19fa4on2hq8a6EnJtdEPgZy8D7Kyoq3O9QeiwGx4TELxSOocIss8HgCVJFsw0Gw8yFWWaDwRPYw2wweAJ7mA0GT2APs8HgCexhNhg8gT3MBoMnsIfZYPAE9jAbDJ7AHmaDwRPYw2wweIL/B6It6n4Q9ZCmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 15 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "l=[]\n",
    "for i in range(15):\n",
    "    Z = np.random.normal(0., 1., size=[1, G_input])\n",
    "    Gl1 = Z.dot(G_W1) + G_b1\n",
    "    Gl1A = Lrelu(Gl1)\n",
    "    Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "    Gl2A = Lrelu(Gl2)\n",
    "    Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "    Gl3A = Lrelu(Gl3)\n",
    "    Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "    Gl4A = Lrelu(Gl4)\n",
    "    Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "    Gl5A = Lrelu(Gl5)\n",
    "    Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "    Gl6A = Lrelu(Gl6)\n",
    "    Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "    Gl7A = Lrelu(Gl7)\n",
    "    Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "    Gl8A = Lrelu(Gl8)\n",
    "    Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "    Gl9A = Lrelu(Gl9)\n",
    "    Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "    Gl10A = Lrelu(Gl10)\n",
    "    Gl11 = Gl10A.dot(G_W11) + G_b11\n",
    "\n",
    "    current_fake_data = log(Gl11)\n",
    "    l.append(current_fake_data)\n",
    "fig = plot(l,\"TEST\")\n",
    "fig.savefig('Click_Me_{}.png'.format(str(iter).zfill(3)+\"_Ginput_\"+str(G_input)+ \"_hiddenone\"+str(hidden_input) + \"_hiddentwo\"+str(hidden_input2) + \"_LR_\" + str(learing_rate)), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "id": "FnpFQdTpbfRS",
    "outputId": "30ca1f5c-f62d-435c-e4b8-9dfa2ea92a54"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RfjgFlc9bfRf",
    "outputId": "0905a244-2393-4be7-a7e2-455bbb82ea80"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(len(grad_r_w1_part_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sXhuAW8zbfRs",
    "outputId": "46292ee1-8885-480b-e0df-961713409251"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "print(len(current_fake_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZA9tB9_2bfR2"
   },
   "outputs": [],
   "source": [
    "Z = np.random.normal(-1., 1., size=[1, G_input]) \n",
    "Gl1 = Z.dot(G_W1) + G_b1\n",
    "Gl1A = Lrelu(Gl1)\n",
    "Gl2 = Gl1A.dot(G_W2) + G_b2\n",
    "Gl2A = Lrelu(Gl2)\n",
    "Gl3 = Gl2A.dot(G_W3) + G_b3\n",
    "Gl3A = Lrelu(Gl3)\n",
    "Gl4 = Gl3A.dot(G_W4) + G_b4\n",
    "Gl4A = Lrelu(Gl4)\n",
    "Gl5 = Gl4A.dot(G_W5) + G_b5\n",
    "Gl5A = Lrelu(Gl5)\n",
    "Gl6 = Gl5A.dot(G_W6) + G_b6\n",
    "Gl6A = Lrelu(Gl6)\n",
    "Gl7 = Gl6A.dot(G_W7) + G_b7\n",
    "Gl7A = Lrelu(Gl7)\n",
    "Gl8 = Gl7A.dot(G_W8) + G_b8\n",
    "Gl8A = Lrelu(Gl8)\n",
    "Gl9 = Gl8A.dot(G_W9) + G_b9\n",
    "Gl9A = Lrelu(Gl9)\n",
    "Gl10 = Gl9A.dot(G_W10) + G_b10\n",
    "Gl10A = Lrelu(Gl10)\n",
    "Gl11 = Gl10A.dot(G_W11) + G_b11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "56R3Qb35bfSC"
   },
   "outputs": [],
   "source": [
    "Gl10 = Gl9A.dot(G_W10) + G_b10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4twoZKgvbfSk",
    "outputId": "4befe795-f4fa-4f14-b52c-1d140ff1f06c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Gl9A.dot(G_W10)+ G_b10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0b_-c-6TbfTC"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lGK_P4habfTp"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "GANTRIQALSend.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}