Eager_Execution_Very_basics_notes.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Eager Execution Very basics notes.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/michelucci/TensorFlow20-Notes/blob/master/Eager_Execution_Very_basics_notes.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3MTPtVfSphMH",
        "colab_type": "text"
      },
      "source": [
        "# Eager execution basics - few various notes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nsm_LhqQdjkT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "2fac6951-761b-41ae-998e-8fde2c5968e9"
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x  #gpu\n",
        "except Exception:\n",
        "  pass\n",
        "import tensorflow as tf\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "`%tensorflow_version` only switches the major version: `1.x` or `2.x`.\n",
            "You set: `2.x  #gpu`. This will be interpreted as: `2.x`.\n",
            "\n",
            "\n",
            "TensorFlow 2.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZBlgbSRdplqT",
        "colab_type": "text"
      },
      "source": [
        "Consider any operation that in TensorFlow 1.X would have needed a `session`, in 2.X this is not necessary and the simple direct evaluation happens."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wMXuqKmOdet-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "cd01f1d9-033e-4733-f271-95022ff78d74"
      },
      "source": [
        "x = tf.constant(3)\n",
        "y = x + 3\n",
        "print(y)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(6, shape=(), dtype=int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wXGT9KkMpx5H",
        "colab_type": "text"
      },
      "source": [
        "Consider the following function, in which we calcualte the factorial of a number with TensorFlow tensors."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vEvSLPovdkGL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def factorial(n):\n",
        "  fact = tf.constant(1)\n",
        "  n = tf.convert_to_tensor(n)\n",
        "  for num in range(1, n.numpy()+1):\n",
        "    fact = fact * num\n",
        "\n",
        "  print (fact)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tVDnZOuIp3PH",
        "colab_type": "text"
      },
      "source": [
        "You can simply run it and get the results you want, without the need of creating a session."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8X30EdsBex1P",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4c9485d4-003e-40a2-9b42-cfef049fe81e"
      },
      "source": [
        "factorial(3)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(6, shape=(), dtype=int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_UCyaXuBe0IS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0d47590d-c4a6-474f-fecd-2396580e2ec7"
      },
      "source": [
        "factorial(4)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(24, shape=(), dtype=int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NHZziBH2gAN3",
        "colab_type": "text"
      },
      "source": [
        "# MNIST Example"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "enAwZouabVZX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "cfc19a07-4ddd-4ba5-8cfd-b516e52b8b6f"
      },
      "source": [
        "# Fetch and format the mnist data\n",
        "(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data()\n",
        "\n",
        "dataset = tf.data.Dataset.from_tensor_slices(\n",
        "  (tf.cast(mnist_images[...,tf.newaxis]/255, tf.float32),\n",
        "   tf.cast(mnist_labels,tf.int64)))\n",
        "dataset = dataset.shuffle(1000).batch(32)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "43W-Qn2RqC9z",
        "colab_type": "text"
      },
      "source": [
        "We define a dumb sequential model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GAzbrFo8qCcY",
        "colab": {}
      },
      "source": [
        "mnist_model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Conv2D(16,[3,3], activation='relu',\n",
        "                         input_shape=(None, None, 1)),\n",
        "  tf.keras.layers.Conv2D(16,[3,3], activation='relu'),\n",
        "  tf.keras.layers.GlobalAveragePooling2D(),\n",
        "  tf.keras.layers.Dense(10)\n",
        "])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ymeGkgl9bn1t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "242a091c-7ac6-4e47-8c14-e2d375b53b75"
      },
      "source": [
        "for images,labels in dataset.take(1):\n",
        "  print(\"Logits: \", mnist_model(images[0:1]).numpy())"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Logits:  [[-0.07004383 -0.04109984 -0.05877591 -0.01022927 -0.04516312 -0.01131148\n",
            "   0.03299733 -0.01285523  0.05138505 -0.03488974]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UZ9e84sCqGLI",
        "colab_type": "text"
      },
      "source": [
        "Then optimizer and loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BaMcTCi3b5qN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "loss_history = []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "O9XdBxhhqIiT",
        "colab_type": "text"
      },
      "source": [
        "Finally the `train_step()` that will perform the weight update each time is called. This step will be called each epoch during the training loop."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TZwLk7OecD1F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_step(images, labels):\n",
        "  with tf.GradientTape() as tape:\n",
        "    logits = mnist_model(images, training=True)\n",
        "    \n",
        "    # Add asserts to check the shape of the output.\n",
        "    tf.debugging.assert_equal(logits.shape, (32, 10))\n",
        "    \n",
        "    loss_value = loss_object(labels, logits)\n",
        "\n",
        "  loss_history.append(loss_value.numpy().mean())\n",
        "  grads = tape.gradient(loss_value, mnist_model.trainable_variables)\n",
        "  optimizer.apply_gradients(zip(grads, mnist_model.trainable_variables))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oeu-zlOQcLqq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train():\n",
        "  for epoch in range(3):\n",
        "    for (batch, (images, labels)) in enumerate(dataset):\n",
        "      train_step(images, labels)\n",
        "    print ('Epoch {} finished'.format(epoch))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rlRs9up2ca2s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "fed4c546-b81f-44e3-cc38-79130605c068"
      },
      "source": [
        "train()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0 finished\n",
            "Epoch 1 finished\n",
            "Epoch 2 finished\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Y7TI7mUccPQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "b90689d8-fb1f-4573-b283-d23cb528b9cc"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(loss_history)\n",
        "plt.xlabel('Batch #')\n",
        "plt.ylabel('Loss [entropy]')"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'Loss [entropy]')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXd8FOXWx38nISHUhBJqgFCl14g0\nBQQRREW91y6g4sXervde4NqwXK/X3uVFsF17wYKowKWDFEPvEHoJJHQkQEhy3j92drPZnZmd2Z3Z\nmc2e7+ezsPvMM8+c2cw+53nOc55ziJkhCIIgCACQ4LQAgiAIgnsQpSAIgiD4EKUgCIIg+BClIAiC\nIPgQpSAIgiD4EKUgCIIg+BClIAiCIPgQpSAIgiD4EKUgCIIg+KjgtABmqV27NmdmZjothiAIQkyx\nfPnyQ8ycHqpezCmFzMxMZGdnOy2GIAhCTEFEu4zUE/ORIAiC4EOUgiAIguBDlIIgCILgQ5SCIAiC\n4EOUgiAIguBDlIIgCILgQ5SCIAiC4CNulMKZc8XYfOAkCotKgo7N3HAQB0+cATNjz5ECB6QTBEFw\nB3GjFH5em4tLX5uPVo/9gtV7juHO/2bjnbk5YGb85eNsXDthMT76bScufGEOZm446LS4giAIjhBz\nO5rD5Vxx6Qxh2NuLAADT1x/EvM35AIDdRwowfuoGAMA/vlmNhWMuxu4jBWhTv3r0hRUEQXCIuJkp\nXN0lQ7V86Y4jQWUJRLjrk+UY8voCVXOTIAhCeSVulEJyhQRsemYwZjx8Uci6CQmEZYqyKGG2WzRB\nEATXEDdKAQBSkhLRqm41fHx7d916CVT6XnSCIAjxRFwpBS8XtUrHtucu0zxOIBBpHhYEQSi3xKVS\nAIDEBMJnf7lA81ggeSfPIO/kGbvFEgRBcJS4VQoA0Kt5bWzXmDGcOedZYGZ47Efd/zUL3f81K2qy\nCYIgOEFcKwXAs6g8sE2dMmX7jp32vZc1BUEQ4om4VwoA8Nw1HTSPiU4QBCGeiJvNa3rUqZaieaz9\nk9PRKSM1itIIgiA4h20zBSJqRERziGgDEa0nogdV6txMRGuIaC0R/UZEneySJxJW7z3utAiCIAhR\nwc6ZQhGAR5h5BRFVA7CciGYy8wa/OjsA9GXmo0Q0BMBEAOouQYIgCILt2DZTYOZcZl6hvD8JYCOA\nhgF1fmPmo8rHJQDUY1FEgU6N0gzVO3qq0GZJBEEQnCMqC81ElAmgC4ClOtVGAfglGvKo8cO9vQ3V\n6/LMTGTvDI6XJAiCUB6wXSkQUVUA3wJ4iJlPaNTpD49SGKNxfDQRZRNRdn5+vn3CGuTPExZjUc4h\n32dmlsB5giCUC2xVCkSUBI9C+JSZp2jU6QhgEoBhzHxYrQ4zT2TmLGbOSk9Pt03ehmmVDNe9edJS\nZI6dhsN/nMWr/9uKVo/9gj/OFtkmmyAIQjSw0/uIAEwGsJGZX9Go0xjAFADDmXmLXbIYZdYjfU2f\nsyH3BL76fQ8A4MTpc1aLJAiCEFXs9D7qDWA4gLVEtEop+yeAxgDAzBMAPAGgFoB3PDoERcycZaNM\nuqQkJUZ0vgTREwQh1rFNKTDzQgC63SQz3wHgDrtkCIe7+zXHu3O3OS2GIAiCI0iYiwDaNTCXfpNA\nvqB5oWBm5OT9EY5YgiAIUUGUQgAD29Q1VX/v0QLfewqYGM3edBAb9pc6XH2waCcGvjIPK3YfhSAI\nghsRpRBASlIi1j11qeH6Y6esxcETZ1WP3f5hNi57Y4Hv8+q9xwAAuw8XqNYXBEFwGlEKKsh6sSAI\n8YooBRXC9UIy6n1kdA1CEAQh2ohSUEEtHacRJi/cYbEkgiAI0UWUggZ39W1u+pyJ87cbqhe4IC0I\n4XDwxBm0feLXMs4MghApohQ0GDuktW1tmzUf5R4/jbdmbwVLblDTFBQWldu4VLM35aGgsBgfL97p\ntChCOUKUQgxw9ycr8NKMLdgqexxM0/aJ6bj2/xY7LYYgxAySjtMGdhw6hZdnbNY8btZ8dLqwGABQ\nIjOFsFi955jTIghCzCAzBR1a16sW1nnvzs3BT2tyLZZGcJJHv1uL8x5zLN2HIEQNmSnoQGFEuMsc\nOy1kHXFJjT0+XbrbaREEISrITCGG+e+SXVi37zgOnjjjtCiCIJQTZKagg12Oo1a5pD7+/Trf+6n3\n9UGHjFRL2hUEIX6RmYID2GE+2pYvnkmCIESOKAUdrEqakzl2Gu78b3ZQ+Ykz53DijDXZ2mSdQiiP\nnCsuwa7DpzSPl5QwflqzHyUl8vxbhZ3pOBsR0Rwi2kBE64noQZU6RERvEFEOEa0hoq52yRMOVmZS\nm77+IPYdPe1pVzEfdRw/Ax3Hz9A9r6SEfR3+O3O2Yc8RibAqxA/jf1yPvi/OxeE/1CMRf7p0F+77\nbCW+zN4TZcnKL3bOFIoAPMLMbQH0AHAvEbUNqDMEQEvlNRrAuzbKY5rxV7SztL3sXZ48CkZH9UdP\nFaLZP3/GloMe09CPq/fjklfnqdaV0BlCeWRRziEAwIkzRarH8056lEX+SXWlIZjHznScuQBylfcn\niWgjgIYANvhVGwbgY/bEb1hCRGlEVF8513GyMmva0u7DX6429BDnHg/2KjpzrgRbD55Ey7pl91AY\nVTQ7D53C6XPFaFPfXIY5QRDig6isKRBRJoAuAJYGHGoIwH/et1cpK/d8sSz86e7eY6fDPrffS3Mx\n5PUFoSsKghCX2K4UiKgqgG8BPMTMYYVzJKLRRJRNRNn5+fnWCugQWtPhaFNUXIKi4tgNGHeuuASZ\nY6fh7Tk5Tosi2IgEg4wetioFIkqCRyF8ysxTVKrsA9DI73OGUlYGZp7IzFnMnJWenm6PsFHmkMbC\nmRGsXD047/Ff0ffFuRa2GF1On/PEhZowd5ujcnywaAeuj+PAe3uOFNjScYcTVUCIDDu9jwjAZAAb\nmfkVjWo/AhiheCH1AHDcLesJsca4KWtxRukgzVBcwtgXgTlK8PDU1A1YuuOI02I4wsbcE7jwhTmO\nJpmSiYR12DlT6A1gOICLiWiV8rqMiO4ioruUOj8D2A4gB8B7AO6xUZ6Y45vlew3XPXOuBF8r9Y+c\nKkSf/8zG5gMn7RKt3LHz0Cm8OnOLmCnCYNdhj5v0MhuUovw9oo+d3kcLEcLSoXgd3WuXDLFMUXEJ\n3l+kPvIKNaWeuzkPe4+exoR52/Dq9Z3tEK/ccesHy7DzcAGuP78RGqRVclocIYBQz7xYmaxDdjS7\nkILCIpwqNG8K8v4uwh1c/ZZzCGv3Hg/vZIewaiB5tpxmZ4smdo7pZcYQPUQphGDhmP62X+O+z1bg\nj7Ol3khtn5iOTk9p73QmwND6gdnB002TluKKtxaaPEtwinjoJ61aaC4uYWQ9+z98vzLIj0UIQJRC\nCDJqVMa7N9sbfeOnNbn4y0fZePjLVYbyMYx4fxnu+mS55nHJ0CY4gZ0WnB2HtOMfaTFr40Fkjp2G\n3YcLcKqwCIf+OFsmsrATLN1+GMUuj9MkSsEAQzrUR7PaVWy9xuLth/GdiVHM3M3B+zUKCj2zjX//\nsgkAMGXlPtz+4e/WCOhSYnmPRaS4zY5uZ1c36qPggJKhmKL8nlbvdUc61sXbDuP6iUvwjsv31IhS\nKEc897NHGRw5Vegrm70pz9C54bizuoHbw+gs9DDbsf227ZCh2V15xgrlNGP9AWSOnRbWjAAwbkpz\ncozuTYaV4/Iw96IUTHJn32ZOi2ALkxZsd1oEVXKPn9adDazeo4wCbRg1MzO+/H13mfWeQL76XaJz\nWmGtnKrkNF9j06jeZZMqVyNKwSTXZTUKXSkGeWnGFqdFCOLoqUL0/PdsPP3ThtCVbSB711GM+XYt\nnnDYDi2Exm2mtFhG0nEKrsWbgEht/cQu/PuWU8oMIV8lJMnuwwVYv/+4pDaCNR2yuJy6B5kphMGt\nvTKdFsEU2/L/wE9r9lvS1u7DBTgdxh6KSLAiq9z9n6/UPX7itLkMeJe+Nh93f7oibHnW7z+OJ39Y\nF3OdITNj9Z5jtsltaayj2PpqXYMohTDomJHqtAimGPDyPNz3mX6naJSLXpyD0SqpRe1kz5HTuPWD\nZThbFL4ymrpaWyke+uOs6c2C3kB84faNN09aio8W78KxAmvSsUaLaWtzMeztRfh+VamnnJ16TaxC\n0UeUQhgkxKkB86ji1bRg66GoX3vu5nys2xdW5PWQ5J0oNQ+Z7d8i7Q9jbTC7Pf9Umf+twpbvQeVn\n6oaZmQtE0EWUQhjUqprstAiadH5aP+dzJDz380bb2lYjVlOMrtx9FMcKCnXr2HFnf5wtwjmb922o\ndWhWjpHCbSpURyshuI0jSiEMLmzp3pwOVpgjMsdOw7MqHj+FNnQ4O8P0S48meh2K2sjz6nd+ww0T\nl+i2eVT5O1k5cm3/5HTcrbPT3Uqi1cVaPah2wyDd7fpJlIJBejSvBQConpLksCTRYdLCHcg7WTZH\ntFb/tTH3RMjObf+x09h0oKz5Z9bGg+j30lxMW1P+Umhs0glbvmG/PWYwAPjfRmObFWMZZsaUFXvL\nbLh0e0cbS4hSMMj4K9ph9iN9kV6tIoDY80AKByNeRjM3HMSQ1xeEDNHR6/nZGPxa2dzQG3M9neOG\nXGsiszIzFmzNNz361upQjLQSzsjzdIzuHncCtT/NnM15+OtXq/HyjM2WtCeURZSCQZIrJKBZelXf\n5/FXtnNQmsh5/pdNpjtP/9oj31+Gez9dga15nhHx5oMncbaoGHuOFFgoZVn0RoME4JOluzF88jLf\n7tiwrhH2mUJEmHgUT5z27B/JOxkipa1Km24wH7kd2zavEdH7AC4HkMfM7VWOpwL4BEBjRY6XmPkD\nu+QRgB7PzfK9nzBvGwpN5hDwj746b4tnQ9m0tZ4OmED429drMHX1fmx6ZjBSkhItkNgcXoWUG0F6\nUdOdRhi9jL9yK0+dlBXLI3aYgYiMK/vVe46hRZ2qqFIxfvf12jlT+BDAYJ3j9wLYwMydAPQD8DIR\nudetpxxw4ETZNQKtzG5e/L1/9hwp0O1siYC5SvC9SBekV+w+qprs55p3fsP6/cZNTXZ74gDWbKyL\nJdx0v4aD4Bmsd+LMOQx7exEeCLHRMVLi1iWVmecD0EvaygCqkce1o6pSVzvymAuZck8vp0Wwnflb\n8vHQFytx4QtzsGK3sWBlHcfPQObYadh6MHix9cEvQv/grnnnN81kPy9NN25HHj55qeG64WLXD7yw\nqATjf1zv2xviOlSG9GZG+WfOFZcxX2oqG5U21a6j+ncwWk/h7DnPIMKuUNuxshju5JrCWwDaANgP\nYC2AB5k5poLju13jR8pNk5ZgxPvL8P2q0CEyyPdPKV7Tkj8/GGjLCpgZS7abTyR/ujC64xKtZ+iX\ndbn48LedeHZadPeGhELvmTf6e9hzpACtH/8VX6hEmM0/ebbsDM+i31isdMhuQFMpENEaA69ZWucb\n4FIAqwA0ANAZwFtEVF1DltFElE1E2fn50QuOFu/sPWrcNr/pwEmcPFO2Q4220pw4vzT892fLdofV\nxsBX5uO3nOjv2A7Em50rFrLome1vvfkEfl13IOjYU1M3YMw3a8zLEEIIN3yNbpDBCHqrKYkALtM5\nTgB+jODatwF4nj1zyBwi2gGgNYBlgRWZeSKAiQCQlZXloq/WRaI4jNFkPmrY8WPR2gvw2Pdr0TEj\nrUwI9MDZy7KdR9CrRW3fn1evvwmUPZoJd5zoZAI731V7juG4yWCCoZi54aA1Dbn05+n2WYueUriT\nmXfpnUxE90Rw7d0ABgBYQER1AZwHwJ2ZXoSwOVZQiM06G7n0sOPH88mS3QB2l1EKE+Zts/5CAJqN\nm4Z2DVIx9f4+ZcrN3JYbYvX4EyjOVW8vCqMRa2QxCpH7O2I3oWk+YuaFAEBEVxCRaj1vHTWI6HMA\niwGcR0R7iWgUEd1FRHcpVZ4B0IuI1gKYBWAMMzs/bxcsZcT7y3B9iJAPWj/YcBf8jHQAejGizMRc\n0vPGKWFg7T59bymt843cg5MdnRWXLuOaq/U16lwo3I2DoUKruEwPRx0jC83XA9hKRC8QUWujDTPz\njcxcn5mTmDmDmScz8wRmnqAc38/Mg5i5AzO3Z+ZPwr0JwZ2cLSrRNOPk5P2Bfcc8LrJaP0K1cN/+\ncYiOa8R5KiphfLpUf01BL0aUGbfLiDsQBjYfOInhk5fGRJ5st/aXZhbA+70011ZZYp2QSoGZbwHQ\nBcA2AB8S0WJl4bea7dK5nHgfUYRiwrxtKCpR/5IGvjIPn4exGLwx94QvZ3JOvrpZasoK9ZAbaqaY\nwypZ1d6bv13VnVa1TUO19Hn8h3VYsPUQVu2xxxXSTZw5Vxx68dzENES3qkMzqVDK3e39hiGXVGY+\nAeAbAF8AqA/gagAriOh+G2WLGbo1qeG0CHFD7vEzuOOj33XrFJxVdytVi1xaoBLf6VRhcXi28nDx\n6yTOFhXj1ZlbfD7zbiOSfvbMuWK0fvxX1RDs4XSUzMbndEbat8Ic9+Pq/Wj9+K+q62ixsq4RUikQ\n0ZVE9B2AuQCSAHRn5iEAOgF4xF7x3I3LFX7MYPbHsmT7Ed1MalqhrpfuML5vwT9o3cGAneCRhvvW\nC8Xd+/k5eH3WVkxc4E6fC/9nPtQi+OE/zuKVmVtQoswWvQEWtykJeghA3skzmibGgsIiw8pC9SvV\nOVfNY8qKEfysjR7PKW+wx1jEyEzhTwBeVWz/LzJzHgAwcwGAUbZK53K8DxEB6NmslqOyxDL+P8Yl\n2w8bOueb5XttkqYU7zg0MAy2f77rSDuSwNMPKeYs70zBrQMPdb1WVtpRH2XjjVlbNf+mDKD/i3Nx\n2RsLVNtr+8R07NBRwEYT56hV6/bMTEPnWonbzUZejKwpjASwRZkxXEFE9fyORbJ5rdxABCRVkICz\nkVBcwpi2Jtew++ribWU7mhN+G+dyj4cfEM9JYsS6YBjvGkmxTm/ozY0drvkoXNTWutSUxw+r9mHg\nK/MsdQ12uxkpZChAIhoF4EkAs+F5bt8koqeZ+X27hRPih48X78RTU4OzvWlRWFyC9RpmB421bVPo\ntVG2f4iR4Z9V6HaO5nq7cPvGaKbWfPCLVVG7llswMrz9B4AuzHyrMmvoBmCMvWLFBh0zUtGjWc2Y\nz63gBgIjuBrhiR/WR3zd6/5vcUTnWzGAdFPk0cjQv4/Ao3M25/sdi+w7CCcm0zITa0x2kHv8tK35\nR8LFiFI4DMB/Tn9SKYt7UpIS8cXonmjXINVpUYQwaDZuGnKPayujXYdD/2DD6cr8x7n+HZYklzdH\nyO8+RN6KZ6cZn5laifdv3vPfs3HhC3MckUEPI0ohB8BSIhpPRE8CWALPGsNfieiv9ooXezw2tA2e\nvSoop5AQgnmbox/oMJSZyYw5KxTHCgrxxA/rcLZI24ddbdYy18D38sXve5AfKguZxRBIZQSurtT8\nHTKMtW1+vSCU91H7J6ebai+oKZcmELIDI0phG4DvUfoV/wBgB4Bqykvwo0WdqrilRxOnxYg59BLd\nu41w+of//LoZHy/ehR9WlnWlffCLlUHRZQFgn5LQ6Pjpcz6XTj38TSFOeNZESmCna6WnTiRNxUpH\nbiUhF5qZ+SkAIKKqyuc/7BYqlgl8ACtWSMBZk2kvyzv7AzK4fR0F91K7MDqinb7eEya6mLlMR2Nk\n78TSHUfQs7lxl+fDNibmMbdq4MF7v2Y65wVRCF8eKy6i0cbI5rX2RLQSwHoA64loORHJymoAWgOK\nhHgcaoTgoQCPjmibPqzEaL9yROmomc13RtHOqfD2nBxkjp2GUxo7w+0g8Geil+zI7C/KqOJWm5HF\no+IwYj6aCOCvzNyEmZvAs4v5PXvFij20np30ahWjKkcssGyns14fVmIkHIV/joUSZgyLUgiNdfuO\nY7ffYvmWgycNKeBPlngi5ofKk6Di6a9a75/frQ15zXA632lrgjP7RUL35zzbrtREsUI3xIqCMaIU\nqjCzb4mcmecCqGKbROWMtMpJaNdANaGcEKP4/7i3GAyc5zvXQjmydx7VPX75mwtx0Yul3i2DXp2P\nC1+Ybcm1iYDX/rcloLT07vyD++05Yn4zodkONByXVKfCk7vdeGBEKWwnoseJKFN5PQZJhhOE1t+5\nYVolJMtu53KLadNOGMPFmyct9UVz/er3Pb7QD9+uKF2LOXFGe1S/cGupff6MhYH2AqPR+i+YRxpQ\nUO9bWr5LXxkG8tCX6hvQzP4pThcWo+szMzEnjCyDx0+f88nh9hmDkd7qdgDpAKYA+BZAbaVM8KOO\nYiaqnJQIALioVToA4NqsjHIXviDeiWSkF25/4M2X/Y9v1+DyNxZg/payrqrjpmibaG6ZvLTM57yT\nZ/CXj7NxUkORhDPqBswFHAxq12C9XYdP4cPfdgaVR2P0vevIKRw5VYjnf9lk+lw3blLTQtf7iIgS\nATzKzA+YbZiI3gdwOYA8ZlZ13CeifgBegyf66iFm7mv2Om5h/JXt0L1pTXRvWhMAkKA8pASSTUnl\niH3HTuOVmaVmE7OdvBWjxFOFxRjxflAqc8O8OSsHMzccxHcr92FEz8yw29l3zLoYU/55mc8Vs2YS\nJKvzQRvFbWlR7URXKTBzMRH10aujw4cA3gLwsdpBIkoD8A6Awcy8m4jqhHkdV1ClYgVc65f31wuD\nZaZQjrg+YIOZXgY3NcLtXF6ZuQVNa1uzlBcqpITaGOal6ZvRuFZlS64fitPnijUXp818fUZCZ2zY\nfwLHTheiV/PaxhuO8Np/RNGrKxyMmI9WEtGPRDSciK7xvkKdxMzzAejNJ28CMIWZdyv1zRvqXIye\nIvj27p5Rk0OwlkjdZ8MN1jdvS76q2SQcjO4wXpRzCDl5noX0t+bk4B/frPEdc2qgY+brKzGwfHLZ\nGwtw03tLQ1e0ELcrhZCb1wCkwBPr6GK/MoZnjSESWgFIIqK58OyMfp2ZVWcVscifuzXCnM35aF2v\nrOdR5eREdGtS0yGphEiJdCNitIwQ54q15fTlr9Ywa3qVxt8VJbDz+aGGr/ve/O3YcTiyJETh4Dbr\nDumpTZfJGogRpTCJmcu4EhBRb4uu3Q3AAACVACwmoiXMHOjnBiIaDWA0ADRu3NiCS9vP0I71MbRj\n8I/J+6gM69wAP6zSzh4mlE9i3ZQYyiTzL5VUm1YSmKUtWt+n/11vPngShUUlQV6F363c69u5HsuR\nb42Yj940WGaWvQCmM/MpZj4EYD48KT6DYOaJzJzFzFnp6ekWXDq6+A/IHr6kFQAgo0Ylh6QRnCRa\npoNLX5sfso7RDnXF7mAXUKd8JwLXGuzuerVuc/XeY0FlD3+5WtXl9/Nlu3H5mwstlsw+NGcKRNQT\nQC8A6QHRUKsDSLTg2j8AeIuIKgBIBnABgFctaNd1eKe2X93Z0+edJMQn/p5LdrI9P7QJR6tjDyz/\nWWXnsFlzzVVvL8IDA1qYO8kE/jLnHj+NDxftDDla1zp+xEDsqFD3728++nzZ7pDtuQk981EygKpK\nHf9oqCcA/DlUw0T0OYB+AGoT0V54srclAQAzT2DmjUT0K4A1AErgMVOtC+cmBEHQ56mpwQmJ3pu/\nHTdfEBzRN7DDW7vveMTXX7XnGG7/MDvidozw8JersGT7EVRLMWIdDw8zXmRuW+8Ihea3xszzAMwj\nog+ZeZfZhpn5RgN1XgTwotm2Yw21EVnFClZMtgTBGB8s2hlUtvNwAQoKi1A5Wb/zVNuUNm2ttXGH\nwuXrbM+ubv/gfeeKPb2wWkhyLdQ6+YLCIp+3WFBob5Nylj3X3VrCyJpCRSKaSEQziGi292W7ZOUQ\n/wdvVJ+mGNK+HlY9cYmDEglCWZhZNzXq0u0eBeGW/BdeWfUy6Gnhb+KZrRK64v7PVmqeG2r07/aO\nXw8j86uvAUwAMAmAdtooQRM197QqFSvg3Vu6OSCNIJSyZu9x9GhWmqshJ08/XUq2ybhD0SLSLlht\np7ReDmcznX5gXbebk4zMFIqY+V1mXsbMy70v2yWLcxqmiXeSYD83TFxS5nOhzv4GN8PMYGb8a9oG\nbMs3lgfMf61kUY7JtPMmFppjDSNKYSoR3UNE9Ymopvdlu2RxzqKxF4esc3e/5hguqT8FC3H7KFYL\nIsLeo6fx3oIdpkOPAGUjzqoRNNoP0V55Nx+NVP7/u18ZA2hmvTjlG6OPyZ19jX21Ywa3BgD8d4lp\nPwBBECLAjmx4J8+cw4kzRapWgqOnCpFUIQFVK9rnUeXFSI7mprZLUd4xOZMcN6SNPXIIQjnG7rSl\nuceML2aHYz667I0F2HPktGpYkS7PzESNyklY+cQg0+2axUiO5spE9BgRTVQ+tySiy22XrBxxYQtP\nBMb6qSkOSyII+hzU8TxyOy9O32xr+3d/WrqUGon3kdaRUBnqjoZhFgsHI2sKHwAohGd3MwDsA/Cs\nbRKVQ+7t3wJLxg1Ak1qSxVRwN6M+is4GMzuIJMlPKJjLZq0rYcbwyUvxP788EP78Znbh2kUYUQrN\nmfkFAOcAgJkLEPtxvaJKQgKhns4sYVDbulGURhDKH8z25i0JjI77l4+zsWDrIdzxsboS/Xp56cJ1\n4Kxi+a6j+HXdAd/nYwWhw2pEEyNKoZCIKkGZ9RBRcwCRBZUXyhDO4pF/hMZbe2VaKI0gxB4rdh1F\nXoS5LvS4dsJvZT57d02H4q9frcL6gMiuAHDXJ8v96qyOTDiLMdIbPQngVwCNiOhTAL0B3GqnUEJo\nXvxzR997ibgqxAOfLdUOLLc/jB3NZthy0Njeh0CmrNgXss7sTXnYdCBYcThFyJkCM88EcA08iuBz\nAFnMPNdesQQzjOiZiU4ZqU6LIcQ4JeGmhYsSWik67eJUYejYSVaFEF+9JzgUt1NoKgUiqud9z8yH\nmXkaM/+k5D4IqiNEh66N04LKkisk4L6LWzogjVCemKGxaBqvGNGRzMDK3UeROXZa0LExfulLYwk9\n89HPALqGON9IHSEUJkYbDw1shRenb0b/1nUsu3yFBEKRy0eJgn10eHI6TvlFBBXMsWDrIdXyL7P3\nGG7DTTvJ9ZRCJyLSM3QRPLkVBIt4elg7DGqrP/lqll4FU+/vE1RuJr57IJe2r4dpKolUhPjgpMsT\nyQvRRS+fggT8jxKD2tbFlBW2tUTtAAAck0lEQVT7cEHTWrquq4IguI/vV4ZeTI4ljLikhgURvU9E\neUSkm02NiM4noiIiCpnNrbwyuH19bP3XEJxXr1royhpYNfscf0Vbi1oShPhg+6HQqU9jCduUAoAP\nAQzWq0BEiQD+A2CGjXLEBEmJdv4pjHNrbwl1JQjxjG09ETPPBxBq3/n9AL4FEJz2SPAxcXjoZDxa\na9WVktStgCN7SshtQXALj37vnvT0RgLiNSeiisr7fkT0ABEF+0WahIgaArgawLuRtlXeGdQutOev\nlvmoaW31eEvk52A9IiAng9Y5ghAvtHn816herzjA9euDRTuQOXYaWj36S1TlAIzNFL4FUExELQBM\nBNAIwGcWXPs1AGOYOWSqJyIaTUTZRJSdn59vwaXjByPhhC/wS8cIAO0aVLdLHEGICU6fcy7z8Lp9\nx/HU1A0APJnw1u8/HuIMazES5qKEmYuI6GoAbzLzm0SkndHaOFkAvlBGrLUBXEZERcz8fWBFZp4I\nj0JCVlaWizx67WXpPwegQoKxTQx2+TmveuISdH56pj2NC4IQxOVvLizzeegbCzVq2oMRpXCOiG6E\nJwPbFUpZUqQX9k/eQ0QfAvhJTSHEM3WrO+ue+voNnZFWOdlRGQRBiC5GzEe3AegJ4F/MvIOImgL4\nb6iTiOhzAIsBnEdEe4loFBHdRUR3RSayoEbNKsGd950XNQsrGxVZENClTrWKEbchCEL0MZKOcwOA\nBwCAiGoAqMbM/zFw3o1GhWDmW43WFdTp3rRmmc/dmtTAuMvaYM5mY45dF7VK92WIy6xVGQBQu2r4\nHbtVgcIEQYguIZUCEc0FcKVSdzmAPCJaxMx/tVk2wSTtG1bHun2eyCPesBdGJwof397d9/7BAS3R\nrUkN9FaURDjUrloRB09I2g1BiDWMmI9SmfkEPOGzP2bmCwAMtFcswSrCMR9VSExAv/NKA+7NePgi\nw+dOva8P0qtVxKSRWaavKwiC8xhZaK5ARPUBXAfgUZvlETSoU62iqcxSHPB/JLSsUzVknceGtkGj\nmpXRISMVvz86EEXFIT2NBUFwIUZmCk8DmA5gGzP/TkTNAGy1VywhkFeu64wODVN1PZJuVwlR0a1x\nDdW6Vtn80yp7HNGGdqyPS/022VmxWC0IQvQxknnta2buyMx3K5+3M/Of7BdN8KdPy9qYen8f3RhJ\n13TNwLd39wJQupbw7NXt8cuDF0Z07cAO3j90RsUK7ojZJAiCNRgJc5FBRN8pEU/ziOhbIsqIhnCC\neQIH6BUrJKJN/eAdyuFudjs/U2PmYSZTkCAIrsXIMO8DAD8CaKC8piplgosx2uffcH4jU+0mJSaI\nu6kglGOMKIV0Zv6AmYuU14cA0m2WSwgTX38dYioQScduZJaRaDA8hyAI7sKIUjhMRLcQUaLyugXA\nYbsFE8LDa/8P1W9XSwkvUsnANnXDOk8QhNjAiFK4HR531AMAcgH8GcCtNsokREDd6p5dyBe1VJ/M\n3de/BaY90Af1w0j72aVxGm7rnen7POGWrrguy2N+qpZixLtZEAS3YyTMxS54djT7IKKH4Al9LbiM\n+qmVsGTcAKRrxB5KrpCAdg1SsWavuXC88//eH3WqVwQRoXeLWvjfxjz0b10Hl7arh/suboGKFYyn\n9O6YYf76giBEh3D9CSXEhYupl5oS0qY/pH09tGtQHXf3a26ozca1KiNFcUV988aumPVIX1SskAgi\n0lQIc//WD8seHRBU/tdLWule69ZemYZkMsLVXRpa1pYgxAPhKgVZRYxRvAl00ionY9oDF6JJLfNZ\n1iolJ6J5euhdzpm1q6BOtWAzlZ0b2/q2Kms269o44iSBghBXhKsU4ibRTXljgAMLxUZnI4IgOI+m\nUiCik0R0QuV1Ep79CoJgiDGDW5uqzxalkaulkmNCEAR9NBeamblaNAUR7OXH+3r71gTcTseMNAC7\nIm5n6v19MGvjwcgFEoQ4wrbANUT0vhIWY53G8ZuJaA0RrSWi34iok12yCJ6OtlVdd+h574qCN5he\nYPyka7o2tGQtoFZVmSkI5Yv7Plth+zXsjGb2IYDBOsd3AOjLzB0APANgoo2yCA7TPbNmUJnaYvWb\nN3YBEeHRoW012xrRs0nI6z0woKUpN1lBiAV+WpNr+zVsUwrMPB/AEZ3jvzHzUeXjEgASZK8c8+Wd\nPQB4ZgFevGsH/s5IV3TyLFd1a1IDO58fqtpWWiX13dhd/GYXXRpZ53UUSVpSQYg13BL3eBSAX5wW\nQrAPIsLqJwfhhT91RPemNTGgdR08Pax9WG1VSlZfCnv52k62BOurUlFmHEL84HhsAiLqD49S6KNT\nZzSA0QDQuHHjKEkmWE2qMsKvkAhMvvV8FBQWWdq+Xq6JSOjVvDZ2Hd5tS9uC4DYcnSkQUUcAkwAM\nY2bNIHvMPJGZs5g5Kz1dArTGO6ZmAxqVtUxTakgiISGecOxpJ6LGAKYAGM7MW5ySQ3AevQQ9Xg8l\nAKighO4wohPYxv2VGTUqqZb778fo1byWbdcXBDuxzXxERJ8D6AegNhHtBfAkgCQAYOYJAJ4AUAvA\nO0rYgyJmzrJLHsG9MBgLx/TH2aKSoGPZjw7E3Z+uwMwNB1EvNQV7j57WbCe5QoIl8VeIwstM16JO\nqTeVJCIS7CAaGzJtUwrMfGOI43cAuMOu6wvux3+GkFGjsmqdCokJvhmCWkf9zLB2uKJTAyzZfhh1\nq5sPB25IzoAOXkthXNK2NIRI+4apWJQjaUeE2EOMpYJjhGvi8e+kh/fMRFrlZAxuXz9sOaY90Aft\nGlRHTQtHYX8fdJ5lbQmCl2jMQEUpCI6jt6YQLkbMP5WTPa6mTWpVwbQHLkQdlRwUNSonIa2SeWVR\nwSZPKCHesV8ryJMruJ7ADt6MEtHzHPK24t1E572Of+vjr2wn6wOCa4hG6nNRCkK5JlHp0Y0k2+mQ\nkQoAqFpRf6nNPyWpHoG5He6/uIXv/exH+hpqQxD8EfORIJgkkgQ+z17VHj/c2xsN0tRdTmtUTsLs\nR/qif+s6Qcdu6RG8qfK9EWWd6R4c0NL3vpmBJEWhqBZCeQnlDztMrYGIUhBiDiP9vs8UpFPXq0C8\n1qmUpER0CoiZ5K9kbunRRLMzb12velBZcoDpys6Mc0J8IDMFQQBw0wWeUXinRqm+svRqFQ2ZcdQW\nnM/PrAEAaFLL4wabqPNLIwDdm3oivPZo5tmQVi0leIR+8wUSfkWwn2gMK2T+Kbiei1qlY+fzQzFh\n3jb8vPYAalZJxu+PDjTdznsjstC0dmW0qOPJK/HR7d2xYtdRVNExw7RtUB3N06ti0zODfUmKop13\nOpZoUqsydh0usPUaDdMqYd8x7U2MQmTITEFwDLO7hu/o0xSv39BZd9E4sGtull4FANC1cRouaVvX\npxAAT0jsQe3qabb10/19fDkfrMhaZ7Xa+Nul5vdCtK5nT6KlejZtHBTKcrTgnO3XEKUgOI7RQXaF\nxAQM69zQ1Ki8S+MamPf3frilR+jEPF68yirRgP/fn7pm4P1btaOzLB53seHrDuvcAJufHYxFY42d\nM7JXJpY9OgDPX9PB8DXs4tGhbQBEx7wRz8iagiCY5Omr2uOaLg1xYavavrImtaqYUiTeIHxJiaHP\nefm6Tri4dV3N4/VTSz2ZvBMjNVFqVE7C6zd0QcUKiYau66VOtRRcm9WoTJkVqUzNUnpvohZiHVEK\nguOEE3xOi4ZplfDK9Z0jSsX51k1d8dSV7cqYmqzA212e3yQ4NWmozvSzOy4I2S7gCQne/7xgl1m7\nqaEo0otV3HWthq18YIQgZKFZEAJIr1YRI3tl6ta586JmOHDijKl2ExIIP93fx+f1tHb8IHQYPyPk\neWvHD0K1lCRUSU7EqcLioONmBueNalbGpgMnjZ9gkBqVk7H0nwNQq0oyJi/cYXn7/sSzSojGPExm\nCoLjxKLFYdxlbfD6DV1Mn9e+YSqqpXhG1dVSktCugWd/w63+SigwrEeIL0jr+OiLmqF+atkF4Db1\nq2PO3/qZktkodaunSMyncoDMFATBZv7zpw7YcUjdTbN+agrW7z+B9g2DN795MaszvTolOTEBjWtW\nRu7x0hkNAWhau4rJFkNTIwpx/oXorNmIUhAEm7n+fO2Nbc9d3QEN03JwUUu/OEkBv/vAndG9W9RC\nk1rhdexappch7evhl3UHTLe34elLkXv8DBpqhAaxg3heUohp8xERvU9EeUS0TuM4EdEbRJRDRGuI\nqKtdsgjuJI5/2z7qVE/BU8PalzW7KF9M7aoVsfP5oUgKMMn83/AsPHd1eG6olTT2W1x3fqOyJiyF\nET2bYPJIbZfbyskVfHs5okW4eTiGdtDPufHwwFZRyWwWCbHukvohgME6x4cAaKm8RgN410ZZBCHm\nsKMDCFxj8F0LwLni4HSoKUmJaFXXng1v4WLXTOHBgS3RpXENexq3iH/45QG3C9uUAjPPB3BEp8ow\nAB+zhyUA0ogo/PRZQswRg+vLUSFB2TSXWimpTLk33Wg431tHJSx49UraFuMhKtnr1BTTwjH9w5Cg\nlA9uOz+i883iTZ4U7gzDTazZe8z2azi5ptAQwB6/z3uVslxnxBEEd1C7akU8M6wdBrQpuynu079c\ngMXbDmvGaurdolaZz0SlJrpHBp2HI6fOau5hICL0aVk7uFxFBWnl0zZKhrL+UKdaReSdPGv6fLNd\nu16O71hDQmcrENFoIsomouz8/HynxREsohz8Rm1jeM/MoLwO9VMr4ZquGar1F47pj0kjtEfgyYkJ\nuLpLhmnvFaszfXk3uQFA9YCZ0OOXtzXUhl5UWzV8IdINPHCx6B5tNU4qhX0A/PfnZyhlQTDzRGbO\nYuas9PR0tSpCDFOefoff3dMLU+7pFfXrZtSojEpKzmlvcLq6JoLUaXX+VnaSzWpXweJxA1SPPXNV\ne4zsaSw+lZGYVEL4OKkUfgQwQvFC6gHgODOL6UiIabo0roGuDi9WXpuVgYnDu+Gm7qWusFqdu9fk\npGWWsNJckZKUqBltdniPJoY3vjVIMxeRNaOGZ8aVpJOv24vZWUiFKCuom1Uy/FmNnS6pnwNYDOA8\nItpLRKOI6C4iukup8jOA7QByALwH4B67ZBGEeIKIMKhdPd+CtR6hTCp29HmRmg0n3NJNtTzNzzT1\n+g2dy9R/+6auqFe9YtA5L1/bqcznp69qZ0qWzo2iG3ywfYPU0JUixE7voxuZuT4zJzFzBjNPZuYJ\nzDxBOc7MfC8zN2fmDsycbZcsgjupkpyIYZ0b4IPbujstSrnluavbY2CbuuiiETm1WxPPrKZeatkO\n88pODTxvbDSyh9Ny+4bVUatqcOcOoMx+iRqVS/cb1KiSjKEd1R0br+laNjeHWgIlPd6/7Xx8F0Vz\nYazvUxAEXYgIr9/QxZfuUrCeFnWqYdLILM2osQ8NbIWZD18UFBHW58KqknrUSfTMWaH6S7VF9kjD\nRlRPSbJsb0PPZrVCV4oC7vqLC4IQVRITCC0DNqcN79EEI3tlIoEIw3s2wYHj5qLBOkWo/r15urHQ\nIDdf0BiD2tVDUiIhKTEB105YbIF0ofm/Ed3QMUTUXIl9JAhCVNn5/FDf+9v7NNWs99WdPfHzWnW/\nkMkjszDqI21rsP86xr39m5tKMWlmA9r39/ZGQWGR7/N1WY0w5tu1Ic/7V5ghRCIl2SURZt0hhSAI\nUWVAGMlwGqZV8imN7k1rYvyV6ouygZvutCAC/n5p67DiOH11Z0/f+7b1qyMpkXD/xS3L1OncKA29\nmpduyCOiMkrPDsYOCT8MhVv2SMhMQRDikPdGZKHE4BbfSDqr23pn4oNFO1WPRbLD2H8d6ucHLwyr\njZ/u7xO+ABqECg54TdeGmLJCdTtWVHYrG0FmCoIQhyQkkOF9AZF03k9e0Q6vXu9x+xxhcHOalxu7\nN9I9/uxV7TEugpF5+4b2u3cGcm037XuSmYIgCHHB1V0ycHWX4PAcoTrBNvW1Ew8BwC09zCkZqyDS\nVpRq+aN3Pj8UmWOnAQB6Ntf2MAqlEy5tZ8wsFykyUxAEQZdwRrDh7vQNjAzrNvq2Sse13dTjT5UX\nRCkIgmA5sx7pi/dGaCfn0WL1k4NwixLKQU2tJBjUUHYFW/zg1vPx0MBWIet5c2976dAwFVWU2FSr\nnxjkK3/Yr61ouJsaQcxHgiDoUlUJ1X1BM+ObDJvUqqKZMjTcvAaj+jRVzQ4XTRISKCh6rRr1Uyth\n/f4Tvs9T/Ra1UysnoUWdqsjJ+6PMLCwxgXBj98b4fNluS2U2i8wUBEHQJa1yMmY90hf/vsZa/30t\nbxste/3jl7dFo5qR5XKwG6Pqbkj7ep76ASf8+5oOyGribEBFmSkIghCSaOdhBhCRO04DjbSjAPDZ\nHRdgzb7jZcrMekaFggjof1461gZcxwjJBqK52okoBUEQooo3WF2PUOaoCHxhA0N3+NOrRW30alG6\nqS3cDW3XZ+m7zBoJ9Kim9165rjN6/HtWcN0o7WMQpSAIQlSpWz0Fc//WDw1rqNvmXbLe6kMtbaiW\nIjGqx7x5JVKSgmcF9XRmOdFAlIIgCFEns7Z2cDq35VJeMm4AXpm5BZsPnjR8Tii9NqpPUxQVM0b2\nysRzP2+KTECLEaUgCII7ccmUISGB8LdLz9Ot0zCtEhITyOeKeoU3H4UGKUmJeHBgS906gYTrtWUW\nW5UCEQ0G8DqARACTmPn5gOONAXwEIE2pM5aZf7ZTJkEQBKtZNPZi33u7g+7ZjZ3pOBMBvA1gCIC2\nAG4korYB1R4D8BUzdwFwA4B37JJHEITYI5LYRuWNaC002+n71B1ADjNvZ+ZCAF8AGBZQhwF4t/6l\nAthvozyCIMQAbRUTTNNaVXBn3+YOS+Msl/ulEe3aJDr5oO00HzUEsMfv814AFwTUGQ9gBhHdD6AK\ngIE2yiMIQgxwU/fG6NakBlrX0w+IFw+0qlsNQC4u71gff7mwWVSu6fRC840APmTml4moJ4D/ElF7\nZi7xr0REowGMBoDGjRs7IKYgCNGCiMoohKeubIe61Z1103SaprWrRC02kp1KYR8A/90dGUqZP6MA\nDAYAZl5MRCkAagPI86/EzBMBTASArKwslzmsCYJgJyMdjncUb9i5pvA7gJZE1JSIkuFZSP4xoM5u\nAAMAgIjaAEgBkG+jTIIgCDFHNPdu2DZTYOYiIroPwHR43E3fZ+b1RPQ0gGxm/hHAIwDeI6KH4Vl0\nvpXVslQIgiCUU1rVrYqKFRKdFsOHrWsKyp6DnwPKnvB7vwFAbztlEAQhvujcKDpeOlYx4+G+IetE\ncx+f0wvNgiAIlrHgH/1Rs0qy02LENKIUBEEoN7g930IsIEl2BEEQXIo3iqo3qmo0kJmCIAiCy5g8\nMgvnihn9W6fjdGEJRvVpGrVri1IQBEFwGQPa1PW9NxtNNVLEfCQIgiD4EKUgCIIg+BClIAiCIPgQ\npSAIgiD4EKUgCIIg+BClIAiCIPgQpSAIgiD4EKUgCIIg+KBYi1RNRPkAdoV5em0AhywUx02U13uT\n+4ot5L7cSxNmTg9VKeaUQiQQUTYzZzkthx2U13uT+4ot5L5iHzEfCYIgCD5EKQiCIAg+4k0pTHRa\nABspr/cm9xVbyH3FOHG1piAIgiDoE28zBUEQBEGHuFEKRDSYiDYTUQ4RjXVanlAQ0ftElEdE6/zK\nahLRTCLaqvxfQyknInpDubc1RNTV75yRSv2tRDTSiXvxh4gaEdEcItpAROuJ6EGlPKbvjYhSiGgZ\nEa1W7usppbwpES1V5P+SiJKV8orK5xzleKZfW+OU8s1EdKkzd1QWIkokopVE9JPyOebvi4h2EtFa\nIlpFRNlKWUw/h5bAzOX+BSARwDYAzQAkA1gNoK3TcoWQ+SIAXQGs8yt7AcBY5f1YAP9R3l8G4BcA\nBKAHgKVKeU0A25X/ayjvazh8X/UBdFXeVwOwBUDbWL83Rb6qyvskAEsVeb8CcINSPgHA3cr7ewBM\nUN7fAOBL5X1b5fmsCKCp8twmuuB5/CuAzwD8pHyO+fsCsBNA7YCymH4OrXjFy0yhO4AcZt7OzIUA\nvgAwzGGZdGHm+QCOBBQPA/CR8v4jAFf5lX/MHpYASCOi+gAuBTCTmY8w81EAMwEMtl96bZg5l5lX\nKO9PAtgIoCFi/N4U+f5QPiYpLwZwMYBvlPLA+/Le7zcABhARKeVfMPNZZt4BIAee59cxiCgDwFAA\nk5TPhHJwXxrE9HNoBfGiFBoC2OP3ea9SFmvUZeZc5f0BAN6cfVr35+r7VkwLXeAZVcf8vSkmllUA\n8uDpHLYBOMbMRUoVfxl98ivHjwOoBRfeF4DXAPwDQInyuRbKx30xgBlEtJyIRitlMf8cRorkaI5R\nmJmJKGZdx4ioKoBvATzEzCc8g0kPsXpvzFwMoDMRpQH4DkBrh0WKGCK6HEAeMy8non5Oy2MxfZh5\nHxHVATCTiDb5H4zV5zBS4mWmsA9AI7/PGUpZrHFQmbJC+T9PKde6P1feNxElwaMQPmXmKUpxubg3\nAGDmYwDmAOgJj5nBO/jyl9Env3I8FcBhuO++egO4koh2wmN2vRjA64j9+wIz71P+z4NHiXdHOXoO\nwyVelMLvAFoqHhPJ8CyA/eiwTOHwIwCvd8NIAD/4lY9QPCR6ADiuTIGnAxhERDUUL4pBSpljKPbl\nyQA2MvMrfodi+t6IKF2ZIYCIKgG4BJ71kjkA/qxUC7wv7/3+GcBs9qxc/gjgBsWLpymAlgCWRecu\ngmHmccycwcyZ8PxuZjPzzYjx+yKiKkRUzfsenudnHWL8ObQEp1e6o/WCx3tgCzx23kedlseAvJ8D\nyAVwDh475Sh4bLOzAGwF8D8ANZW6BOBt5d7WAsjya+d2eBb1cgDc5oL76gOPLXcNgFXK67JYvzcA\nHQGsVO5rHYAnlPJm8HR+OQC+BlBRKU9RPucox5v5tfWocr+bAQxx+m/mJ1c/lHofxfR9KfKvVl7r\nvX1CrD+HVrxkR7MgCILgI17MR4IgCIIBRCkIgiAIPkQpCIIgCD5EKQiCIAg+RCkIgiAIPkQpCHEN\nERUrUTJXE9EKIuoVon4aEd1joN25RGQ4py8Rfa7so3mIiG40ep4gWI0oBSHeOc3MnZm5E4BxAP4d\non4aPJFArSaTPYHi+gKYb0P7gmAIUQqCUEp1AEcBT2wmIpqlzB7WEpE3qu7zAJors4sXlbpjlDqr\nieh5v/auJU+OhS1EdKHaBYnoUyLaAKC1EkxvEIBpRHSHbXcpCDpIQDwh3qmkdMYp8OR6uFgpPwPg\navYE66sNYAkR/QhPjP32zNwZAIhoCDxhlS9g5gIiqunXdgVm7k5ElwF4EsDAwIsz881EdC2AxvCE\nmn6Jma+151YFITSiFIR457RfB98TwMdE1B6esAbPEdFF8ISMbojSMMr+DATwATMXAAAz++fA8Ab7\nWw4gU0eGrvCEVugIT9gFQXAMUQqCoMDMi5VZQTo88ZjSAXRj5nNKlNAUk02eVf4vhspvTZlBPAdP\nJrLLleudIqIBzNw/vLsQhMiQNQVBUCCi1vCkbj0MT8jnPEUh9AfQRKl2Ep40ol5mAriNiCorbfib\nj3Rh5p8BdIMn5WoHeAKzdRGFIDiJzBSEeMe7pgB4TEYjmbmYiD4FMJWI1gLIBrAJAJj5MBEtIqJ1\nAH5h5r8TUWcA2URUCOBnAP80cf0uAFYrId2TmPmEVTcmCOEgUVIFQRAEH2I+EgRBEHyIUhAEQRB8\niFIQBEEQfIhSEARBEHyIUhAEQRB8iFIQBEEQfIhSEARBEHyIUhAEQRB8/D+49nWK/caHXwAAAABJ\nRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x2yvOAZPi9PP",
        "colab_type": "text"
      },
      "source": [
        "# Custom Gradients"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sg8yOYfoqVSa",
        "colab_type": "text"
      },
      "source": [
        "Now we consider the example given in the official documentation. Consider the function \n",
        "\n",
        "$$\n",
        "\\text{log1pexp}(x) = \\log(1+e^x)\n",
        "$$\n",
        "\n",
        "we can evaluate the gradients easily without calculating it. Note that if you calculate it we have\n",
        "\n",
        "$$\n",
        "\\frac{d\\text{log1pexp}(x)}{dx} = 1-\\frac{1}{1+e^x}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0eO8OQ3cyBf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def log1pexp(x):\n",
        "  return tf.math.log(1 + tf.exp(x))\n",
        "\n",
        "def grad_log1pexp(x):\n",
        "  with tf.GradientTape() as tape:\n",
        "    tape.watch(x)\n",
        "    value = log1pexp(x)\n",
        "  return tape.gradient(value, x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X6o4B7FyjAcN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "42f07073-87c9-4d12-954e-1db47d49157f"
      },
      "source": [
        "grad_log1pexp(tf.constant(0.)).numpy()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.5"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zdc7Np93q6Ga",
        "colab_type": "text"
      },
      "source": [
        "The problem, due to numerical issues, is that evaluating the function for big values of $x$, for example $x=100$ will give the wrong results. Knowing the function we can define a custom gradient!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3rCpPfxRjC5e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c3ab0d15-548b-4d22-ed7a-5be084a2b38a"
      },
      "source": [
        "grad_log1pexp(tf.constant(100.)).numpy()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "nan"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DadtsYlLk_8C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def log1pexp(x):\n",
        "  return tf.math.log(1 + tf.exp(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_mbs0H-PjE1t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "@tf.custom_gradient\n",
        "def log1pexp(x):\n",
        "  e = tf.exp(x)\n",
        "  def grad(dy):\n",
        "    return dy * (1 - 1 / (1 + e))\n",
        "  return tf.math.log(1 + e), grad\n",
        "\n",
        "def grad_log1pexp(x):\n",
        "  with tf.GradientTape() as tape:\n",
        "    tape.watch(x)\n",
        "    value = log1pexp(x)\n",
        "  return tape.gradient(value, x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90MmHyuIkhr0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4eafbe69-c340-4c7e-9f14-2b3eea0a520b"
      },
      "source": [
        "grad_log1pexp(tf.constant(100.)).numpy()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9IGkvnjRkjaN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5b3421b5-18ba-4f86-f457-f410617e21c5"
      },
      "source": [
        "log1pexp(1.0)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: id=19, shape=(), dtype=float32, numpy=1.3132616>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y6-bnTk6rFAK",
        "colab_type": "text"
      },
      "source": [
        "# Using GPU and CPU in eager mode"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5E7s9wppk8U8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "\n",
        "def measure(x, steps):\n",
        "  tf.matmul(x, x)\n",
        "  start = time.time()\n",
        "  for i in range(steps):\n",
        "    x = tf.matmul(x, x)\n",
        "  _ = x.numpy()\n",
        "  end = time.time()\n",
        "  return end - start"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DDOToWDxl8JC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "80b94ee6-faeb-480d-e96e-688b7e905377"
      },
      "source": [
        "shape = (1000, 1000)\n",
        "steps = 200\n",
        "print(\"Time to multiply a {} matrix by itself {} times:\".format(shape, steps))\n",
        "\n",
        "# Run on CPU:\n",
        "with tf.device(\"/cpu:0\"):\n",
        "  print(\"CPU: {} secs\".format(measure(tf.random.normal(shape), steps)))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Time to multiply a (1000, 1000) matrix by itself 200 times:\n",
            "CPU: 7.359170198440552 secs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jfVsrs-wl_bR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c89005f6-2227-4a4a-ef19-9d80af009300"
      },
      "source": [
        "# Run on GPU, if available:\n",
        "if tf.test.is_gpu_available():\n",
        "  with tf.device(\"/gpu:0\"):\n",
        "    print(\"GPU: {} secs\".format(measure(tf.random.normal(shape), steps)))\n",
        "else:\n",
        "  print(\"GPU: not found\")"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU: 0.259415864944458 secs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NziKgdfGmKmm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}