Super_Basic_Image_Classification_part1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jansoe/AICA/blob/main/Super_Basic_Image_Classification_part1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kJLb0L98ur9d"
      },
      "source": [
        "# A simple Artifical Neural Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2D8gzaqdFm3J"
      },
      "source": [
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OOh2DRwrLoJN"
      },
      "source": [
        "## The Data\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7aOlul7618kS"
      },
      "source": [
        "Our toy data: The MNIST dastaset (handwritten digit recognition)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6FK0XF6p1tJv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "197d86a8-8ed5-41e6-ecd6-f2407784676d"
      },
      "source": [
        "mnist = tf.keras.datasets.mnist\n",
        "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
        "\n",
        "# scaling the pixel values to the range (-1, 1) \n",
        "train_images = (train_images/255 - 0.5) * 2\n",
        "test_images = (test_images/255 - 0.5) * 2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11490434/11490434 [==============================] - 2s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ipAYhZWdjaKs",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 452
        },
        "outputId": "c1bc4344-023c-4ba8-fca2-e54299f115f3"
      },
      "source": [
        "_ = plt.imshow(test_images[2], cmap=plt.cm.gray)\n",
        "_ = plt.title('Label: %s'%test_labels[2])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGzCAYAAABpdMNsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeWklEQVR4nO3de3CU5dnH8d+CsCAmGwPkJBASEEE5aMHEVECUDCEeKkhbtc4UHAcHDA5CPTSdcrDtTCqtSkGKztQSreIBR0CtTQeDCVMboCCU0kokTCihkCCR7IYggZL7/YPXrSsJuGGTK4fvZ+aeMbvPnb14usO3T3bZeJxzTgAAtLIu1gMAADonAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEXKT9+/fL4/Ho17/+dcS+Z1FRkTwej4qKiiL2PYG2hgChU8rPz5fH49G2bdusR2kRpaWlmjdvnr797W+rR48e8ng82r9/v/VYQAgCBHRAJSUlWrZsmWprazVs2DDrcYBGESCgA/rOd76jmpoa/eMf/9B9991nPQ7QKAIENOHUqVNauHChRo8eLZ/Pp169emncuHH68MMPm9zz7LPPKjk5WT179tRNN92k3bt3n3PMnj179N3vflexsbHq0aOHxowZo3feeeeC85w4cUJ79uzR0aNHL3hsbGysoqKiLngcYIkAAU0IBAL63e9+pwkTJuipp57S4sWL9dlnnykrK0s7d+485/iXX35Zy5YtU05OjnJzc7V7927dcsstqqqqCh7zz3/+UzfccIM++eQT/fjHP9bTTz+tXr16acqUKVq7du1559m6dauGDRum5557LtJ/VMDEJdYDAG3V5Zdfrv3796t79+7B22bOnKmhQ4dq+fLlevHFF0OOLysr0969e3XFFVdIkiZPnqz09HQ99dRTeuaZZyRJc+fO1YABA/S3v/1NXq9XkvTQQw9p7NixeuKJJzR16tRW+tMB9rgCAprQtWvXYHwaGhr0+eef67///a/GjBmjjz/++Jzjp0yZEoyPJKWlpSk9PV3vv/++JOnzzz/Xxo0b9f3vf1+1tbU6evSojh49qurqamVlZWnv3r36z3/+0+Q8EyZMkHNOixcvjuwfFDBCgIDzeOmllzRy5Ej16NFDvXv3Vt++ffXHP/5Rfr//nGOvvPLKc24bMmRI8O3PZWVlcs5pwYIF6tu3b8hatGiRJOnIkSMt+ucB2hJ+BAc04ZVXXtGMGTM0ZcoUPfbYY4qLi1PXrl2Vl5enffv2hf39GhoaJEmPPvqosrKyGj1m8ODBFzUz0J4QIKAJb731llJTU/X222/L4/EEb//yauXr9u7de85tn376qQYOHChJSk1NlSR169ZNmZmZkR8YaGf4ERzQhK5du0qSnHPB27Zs2aKSkpJGj1+3bl3Iazhbt27Vli1blJ2dLUmKi4vThAkT9MILL+jw4cPn7P/ss8/OO084b8MG2gOugNCp/f73v1dBQcE5t8+dO1e333673n77bU2dOlW33XabysvL9fzzz+vqq6/W8ePHz9kzePBgjR07VrNnz1Z9fb2WLl2q3r176/HHHw8es2LFCo0dO1YjRozQzJkzlZqaqqqqKpWUlOjgwYP6+9//3uSsW7du1c0336xFixZd8I0Ifr9fy5cvlyR99NFHkqTnnntOMTExiomJ0Zw5c77J6QFaFAFCp7Zy5cpGb58xY4ZmzJihyspKvfDCC/rzn/+sq6++Wq+88orWrFnT6IeE/vCHP1SXLl20dOlSHTlyRGlpaXruueeUmJgYPObqq6/Wtm3b9OSTTyo/P1/V1dWKi4vTddddp4ULF0bsz3Xs2DEtWLAg5Lann35akpScnEyA0CZ43Fd/vgAAQCvhNSAAgAkCBAAwQYAAACYIEADABAECAJggQAAAE23u3wE1NDTo0KFDioqKCvn4EwBA++CcU21trZKSktSlS9PXOW0uQIcOHVL//v2txwAAXKSKigr169evyfvb3I/g+DXCANAxXOjv8xYL0IoVKzRw4ED16NFD6enp2rp16zfax4/dAKBjuNDf5y0SoDfeeEPz58/XokWL9PHHH2vUqFHKysril20BAP7HtYC0tDSXk5MT/PrMmTMuKSnJ5eXlXXCv3+93klgsFovVzpff7z/v3/cRvwI6deqUtm/fHvILt7p06aLMzMxGf49KfX29AoFAyAIAdHwRD9DRo0d15swZxcfHh9weHx+vysrKc47Py8uTz+cLLt4BBwCdg/m74HJzc+X3+4OroqLCeiQAQCuI+L8D6tOnj7p27aqqqqqQ26uqqpSQkHDO8V6vV16vN9JjAADauIhfAXXv3l2jR49WYWFh8LaGhgYVFhYqIyMj0g8HAGinWuSTEObPn6/p06drzJgxSktL09KlS1VXV6f777+/JR4OANAOtUiA7r77bn322WdauHChKisrde2116qgoOCcNyYAADovj3POWQ/xVYFAQD6fz3oMAMBF8vv9io6ObvJ+83fBAQA6JwIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGDiEusBALScIUOGNGvfnj17wt4zd+7csPcsX7487D3oOLgCAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBM8GGkQAd23XXXNWtfQ0ND2HsOHjzYrMdC58UVEADABAECAJiIeIAWL14sj8cTsoYOHRrphwEAtHMt8hrQNddcow8++OB/D3IJLzUBAEK1SBkuueQSJSQktMS3BgB0EC3yGtDevXuVlJSk1NRU3XfffTpw4ECTx9bX1ysQCIQsAEDHF/EApaenKz8/XwUFBVq5cqXKy8s1btw41dbWNnp8Xl6efD5fcPXv3z/SIwEA2qCIByg7O1vf+973NHLkSGVlZen9999XTU2N3nzzzUaPz83Nld/vD66KiopIjwQAaINa/N0BMTExGjJkiMrKyhq93+v1yuv1tvQYAIA2psX/HdDx48e1b98+JSYmtvRDAQDakYgH6NFHH1VxcbH279+vv/71r5o6daq6du2qe++9N9IPBQBoxyL+I7iDBw/q3nvvVXV1tfr27auxY8dq8+bN6tu3b6QfCgDQjkU8QK+//nqkvyWAZrr22mubta+uri7sPWvXrm3WY6Hz4rPgAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATLf4L6QBExvDhw8PeM2fOnGY91h/+8Idm7QPCwRUQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATPBp2EA7MXTo0LD39OrVq1mP9cYbbzRrHxAOroAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMe55yzHuKrAoGAfD6f9RhAm7N169aw9/Tt27dZjzV8+PCw99TV1TXrsdBx+f1+RUdHN3k/V0AAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgIlLrAcAOqOBAweGvWfMmDFh7/n000/D3iPxwaJoHVwBAQBMECAAgImwA7Rp0ybdcccdSkpKksfj0bp160Lud85p4cKFSkxMVM+ePZWZmam9e/dGal4AQAcRdoDq6uo0atQorVixotH7lyxZomXLlun555/Xli1b1KtXL2VlZenkyZMXPSwAoOMI+00I2dnZys7ObvQ+55yWLl2qn/70p7rzzjslSS+//LLi4+O1bt063XPPPRc3LQCgw4joa0Dl5eWqrKxUZmZm8Dafz6f09HSVlJQ0uqe+vl6BQCBkAQA6vogGqLKyUpIUHx8fcnt8fHzwvq/Ly8uTz+cLrv79+0dyJABAG2X+Lrjc3Fz5/f7gqqiosB4JANAKIhqghIQESVJVVVXI7VVVVcH7vs7r9So6OjpkAQA6vogGKCUlRQkJCSosLAzeFggEtGXLFmVkZETyoQAA7VzY74I7fvy4ysrKgl+Xl5dr586dio2N1YABA/TII4/oF7/4ha688kqlpKRowYIFSkpK0pQpUyI5NwCgnQs7QNu2bdPNN98c/Hr+/PmSpOnTpys/P1+PP/646urq9OCDD6qmpkZjx45VQUGBevToEbmpAQDtnsc556yH+KpAICCfz2c9BtCipk+fHvaeVatWhb3no48+CnuPJI0bN65Z+4Cv8vv9531d3/xdcACAzokAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmwv51DAAu3ogRI1rlcZYsWdIqjwM0B1dAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJPowUuEg33HBD2Hvuv//+sPfs2LEj7D0bNmwIew/QWrgCAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBM8GGkwEXKzMwMe09sbGzYewoKCsLec/LkybD3AK2FKyAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQfRgpcpFGjRoW9xzkX9p633nor7D1AW8YVEADABAECAJgIO0CbNm3SHXfcoaSkJHk8Hq1bty7k/hkzZsjj8YSsyZMnR2peAEAHEXaA6urqNGrUKK1YsaLJYyZPnqzDhw8H12uvvXZRQwIAOp6w34SQnZ2t7Ozs8x7j9XqVkJDQ7KEAAB1fi7wGVFRUpLi4OF111VWaPXu2qqurmzy2vr5egUAgZAEAOr6IB2jy5Ml6+eWXVVhYqKeeekrFxcXKzs7WmTNnGj0+Ly9PPp8vuPr37x/pkQAAbVDE/x3QPffcE/zvESNGaOTIkRo0aJCKioo0ceLEc47Pzc3V/Pnzg18HAgEiBACdQIu/DTs1NVV9+vRRWVlZo/d7vV5FR0eHLABAx9fiATp48KCqq6uVmJjY0g8FAGhHwv4R3PHjx0OuZsrLy7Vz507FxsYqNjZWTz75pKZNm6aEhATt27dPjz/+uAYPHqysrKyIDg4AaN/CDtC2bdt08803B7/+8vWb6dOna+XKldq1a5deeukl1dTUKCkpSZMmTdLPf/5zeb3eyE0NAGj3PK45n4rYggKBgHw+n/UY6KSa8+/Xdu7cGfaeY8eOhb1n2LBhYe8BLPn9/vO+rs9nwQEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMBExH8lN9CezZgxI+w9cXFxYe/505/+FPYeoKPhCggAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMMGHkQJfkZyc3CqPc+zYsVZ5HKAt4woIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADDBh5ECX3H77be3yuO8++67rfI4QFvGFRAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIPI0WHNHbs2GbtS0hIiPAkAJrCFRAAwAQBAgCYCCtAeXl5uv766xUVFaW4uDhNmTJFpaWlIcecPHlSOTk56t27ty677DJNmzZNVVVVER0aAND+hRWg4uJi5eTkaPPmzdqwYYNOnz6tSZMmqa6uLnjMvHnz9O6772rNmjUqLi7WoUOHdNddd0V8cABA+xbWmxAKCgpCvs7Pz1dcXJy2b9+u8ePHy+/368UXX9Tq1at1yy23SJJWrVqlYcOGafPmzbrhhhsiNzkAoF27qNeA/H6/JCk2NlaStH37dp0+fVqZmZnBY4YOHaoBAwaopKSk0e9RX1+vQCAQsgAAHV+zA9TQ0KBHHnlEN954o4YPHy5JqqysVPfu3RUTExNybHx8vCorKxv9Pnl5efL5fMHVv3//5o4EAGhHmh2gnJwc7d69W6+//vpFDZCbmyu/3x9cFRUVF/X9AADtQ7P+IeqcOXP03nvvadOmTerXr1/w9oSEBJ06dUo1NTUhV0FVVVVN/gM/r9crr9fbnDEAAO1YWFdAzjnNmTNHa9eu1caNG5WSkhJy/+jRo9WtWzcVFhYGbystLdWBAweUkZERmYkBAB1CWFdAOTk5Wr16tdavX6+oqKjg6zo+n089e/aUz+fTAw88oPnz5ys2NlbR0dF6+OGHlZGRwTvgAAAhwgrQypUrJUkTJkwIuX3VqlWaMWOGJOnZZ59Vly5dNG3aNNXX1ysrK0u//e1vIzIsAKDj8DjnnPUQXxUIBOTz+azHQDv39NNPN2vfvHnzwt6zY8eOsPekpaWFvefMmTNh7wEs+f1+RUdHN3k/nwUHADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAE836jahAa7r00kvD3nPrrbe2wCSNe+utt8LewydbA1wBAQCMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAm+DBStHmnT58Oe8+xY8ea9VjvvPNO2Ht+85vfNOuxgM6OKyAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwITHOeesh/iqQCAgn89nPQYA4CL5/X5FR0c3eT9XQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMBEWAHKy8vT9ddfr6ioKMXFxWnKlCkqLS0NOWbChAnyeDwha9asWREdGgDQ/oUVoOLiYuXk5Gjz5s3asGGDTp8+rUmTJqmuri7kuJkzZ+rw4cPBtWTJkogODQBo/y4J5+CCgoKQr/Pz8xUXF6ft27dr/PjxwdsvvfRSJSQkRGZCAECHdFGvAfn9fklSbGxsyO2vvvqq+vTpo+HDhys3N1cnTpxo8nvU19crEAiELABAJ+Ca6cyZM+62225zN954Y8jtL7zwgisoKHC7du1yr7zyirviiivc1KlTm/w+ixYtcpJYLBaL1cGW3+8/b0eaHaBZs2a55ORkV1FRcd7jCgsLnSRXVlbW6P0nT550fr8/uCoqKsxPGovFYrEufl0oQGG9BvSlOXPm6L333tOmTZvUr1+/8x6bnp4uSSorK9OgQYPOud/r9crr9TZnDABAOxZWgJxzevjhh7V27VoVFRUpJSXlgnt27twpSUpMTGzWgACAjimsAOXk5Gj16tVav369oqKiVFlZKUny+Xzq2bOn9u3bp9WrV+vWW29V7969tWvXLs2bN0/jx4/XyJEjW+QPAABop8J53UdN/Jxv1apVzjnnDhw44MaPH+9iY2Od1+t1gwcPdo899tgFfw74VX6/3/znliwWi8W6+HWhv/s9/x+WNiMQCMjn81mPAQC4SH6/X9HR0U3ez2fBAQBMECAAgAkCBAAwQYAAACYIEADABAECAJggQAAAEwQIAGCCAAEATBAgAIAJAgQAMEGAAAAmCBAAwAQBAgCYIEAAABMECABgggABAEwQIACACQIEADBBgAAAJggQAMAEAQIAmCBAAAATBAgAYIIAAQBMtLkAOeesRwAARMCF/j5vcwGqra21HgEAEAEX+vvc49rYJUdDQ4MOHTqkqKgoeTyekPsCgYD69++viooKRUdHG01oj/NwFufhLM7DWZyHs9rCeXDOqba2VklJSerSpenrnEtacaZvpEuXLurXr995j4mOju7UT7AvcR7O4jycxXk4i/NwlvV58Pl8Fzymzf0IDgDQORAgAICJdhUgr9erRYsWyev1Wo9iivNwFufhLM7DWZyHs9rTeWhzb0IAAHQO7eoKCADQcRAgAIAJAgQAMEGAAAAmCBAAwES7CdCKFSs0cOBA9ejRQ+np6dq6dav1SK1u8eLF8ng8IWvo0KHWY7W4TZs26Y477lBSUpI8Ho/WrVsXcr9zTgsXLlRiYqJ69uypzMxM7d2712bYFnSh8zBjxoxznh+TJ0+2GbaF5OXl6frrr1dUVJTi4uI0ZcoUlZaWhhxz8uRJ5eTkqHfv3rrssss0bdo0VVVVGU3cMr7JeZgwYcI5z4dZs2YZTdy4dhGgN954Q/Pnz9eiRYv08ccfa9SoUcrKytKRI0esR2t111xzjQ4fPhxcf/nLX6xHanF1dXUaNWqUVqxY0ej9S5Ys0bJly/T8889ry5Yt6tWrl7KysnTy5MlWnrRlXeg8SNLkyZNDnh+vvfZaK07Y8oqLi5WTk6PNmzdrw4YNOn36tCZNmqS6urrgMfPmzdO7776rNWvWqLi4WIcOHdJdd91lOHXkfZPzIEkzZ84MeT4sWbLEaOImuHYgLS3N5eTkBL8+c+aMS0pKcnl5eYZTtb5Fixa5UaNGWY9hSpJbu3Zt8OuGhgaXkJDgfvWrXwVvq6mpcV6v17322msGE7aOr58H55ybPn26u/POO03msXLkyBEnyRUXFzvnzv5v361bN7dmzZrgMZ988omT5EpKSqzGbHFfPw/OOXfTTTe5uXPn2g31DbT5K6BTp05p+/btyszMDN7WpUsXZWZmqqSkxHAyG3v37lVSUpJSU1N133336cCBA9YjmSovL1dlZWXI88Pn8yk9Pb1TPj+KiooUFxenq666SrNnz1Z1dbX1SC3K7/dLkmJjYyVJ27dv1+nTp0OeD0OHDtWAAQM69PPh6+fhS6+++qr69Omj4cOHKzc3VydOnLAYr0lt7tOwv+7o0aM6c+aM4uPjQ26Pj4/Xnj17jKaykZ6ervz8fF111VU6fPiwnnzySY0bN067d+9WVFSU9XgmKisrJanR58eX93UWkydP1l133aWUlBTt27dPP/nJT5Sdna2SkhJ17drVeryIa2ho0COPPKIbb7xRw4cPl3T2+dC9e3fFxMSEHNuRnw+NnQdJ+sEPfqDk5GQlJSVp165deuKJJ1RaWqq3337bcNpQbT5A+J/s7Ozgf48cOVLp6elKTk7Wm2++qQceeMBwMrQF99xzT/C/R4wYoZEjR2rQoEEqKirSxIkTDSdrGTk5Odq9e3eneB30fJo6Dw8++GDwv0eMGKHExERNnDhR+/bt06BBg1p7zEa1+R/B9enTR127dj3nXSxVVVVKSEgwmqptiImJ0ZAhQ1RWVmY9ipkvnwM8P86VmpqqPn36dMjnx5w5c/Tee+/pww8/DPn9YQkJCTp16pRqampCju+oz4emzkNj0tPTJalNPR/afIC6d++u0aNHq7CwMHhbQ0ODCgsLlZGRYTiZvePHj2vfvn1KTEy0HsVMSkqKEhISQp4fgUBAW7Zs6fTPj4MHD6q6urpDPT+cc5ozZ47Wrl2rjRs3KiUlJeT+0aNHq1u3biHPh9LSUh04cKBDPR8udB4as3PnTklqW88H63dBfBOvv/6683q9Lj8/3/3rX/9yDz74oIuJiXGVlZXWo7WqH/3oR66oqMiVl5e7jz76yGVmZro+ffq4I0eOWI/Wompra92OHTvcjh07nCT3zDPPuB07drh///vfzjnnfvnLX7qYmBi3fv16t2vXLnfnnXe6lJQU98UXXxhPHlnnOw+1tbXu0UcfdSUlJa68vNx98MEH7lvf+pa78sor3cmTJ61Hj5jZs2c7n8/nioqK3OHDh4PrxIkTwWNmzZrlBgwY4DZu3Oi2bdvmMjIyXEZGhuHUkXeh81BWVuZ+9rOfuW3btrny8nK3fv16l5qa6saPH288eah2ESDnnFu+fLkbMGCA6969u0tLS3ObN2+2HqnV3X333S4xMdF1797dXXHFFe7uu+92ZWVl1mO1uA8//NBJOmdNnz7dOXf2rdgLFixw8fHxzuv1uokTJ7rS0lLboVvA+c7DiRMn3KRJk1zfvn1dt27dXHJysps5c2aH+z9pjf35JblVq1YFj/niiy/cQw895C6//HJ36aWXuqlTp7rDhw/bDd0CLnQeDhw44MaPH+9iY2Od1+t1gwcPdo899pjz+/22g38Nvw8IAGCizb8GBADomAgQAMAEAQIAmCBAAAATBAgAYIIAAQBMECAAgAkCBAAwQYAAACYIEADABAECAJj4P6gnqaxw5r1qAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c4gvGiq8qxOo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a45eb42-8407-41fb-bf1f-f9989a0f31d8"
      },
      "source": [
        "train_images.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60000, 28, 28)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lRX-plXXq5Ta"
      },
      "source": [
        "The training data conatins 60.000 samples of handwritten digits. \n",
        "The data is inherently 2D (28x28 pixel).\n",
        "\n",
        "For our first network we will create now a simple 1D-Vector by flattening the images and thus discarding the 2D neighbourhood structure."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9RaVzw2Vq4d8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ddc92a34-5841-4532-8dff-9d5e0a6aa924"
      },
      "source": [
        "train_images_flat = train_images.reshape((-1, 28*28))\n",
        "test_images_flat = test_images.reshape((-1, 28*28))\n",
        "\n",
        "train_images_flat.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60000, 784)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 64
        },
        "id": "2yGULPVcKjQt",
        "outputId": "85c47977-a583-40e3-bcfe-6d9c8e9b1ede"
      },
      "source": [
        "plt.figure(figsize=(20,0.1))\n",
        "_ = plt.imshow(test_images_flat[2].reshape(1, -1), cmap=plt.cm.gray, aspect='auto')\n",
        "_ = plt.title('Label: %s'%test_labels[2])\n",
        "_ = plt.yticks([])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 2000x10 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABiUAAABJCAYAAABSHSVhAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAXEElEQVR4nO3deVDU9/3H8dfCssu54gWIt5WoxGOoVDTqaKMT4phWbZpraKuNTSaKrYmp1WTGo3/kh0lrYj0GrcYjta2pjtocHiUepEkJeJ+RoMV4i0qQBeTcz++PDDvZYJMguOsuz8fMzuj382Hn/fH7eX8/3y9vv9+vxRhjBAAAAAAAAAAAcJcF+ToAAAAAAAAAAADQMlCUAAAAAAAAAAAAXkFRAgAAAAAAAAAAeAVFCQAAAAAAAAAA4BUUJQAAAAAAAAAAgFdQlAAAAAAAAAAAAF5BUQIAAAAAAAAAAHgFRQkAAAAAAAAAAOAVFCUAAAAAAAAAAIBXUJQAAAAAIEk6e/asLBaL/vjHPzbbd+7du1cWi0V79+5ttu8EAAAA4L8oSgAAAAB+bO3atbJYLNq/f7+vQ7kr8vPz9cILL+iBBx5QaGioLBaLzp496+uwAAAAANwhihIAAAAA7lk5OTlavHixnE6n+vTp4+twAAAAADQRRQkAAAAA96wf//jHKikp0bFjx5SWlubrcAAAAAA0EUUJAAAAIMBVV1dr7ty5GjhwoFq1aqWIiAgNHz5ce/bs+Z8/88Ybb6hr164KCwvTiBEjdPz48QZ9Tp06pZ/+9Kdq06aNQkNDlZycrHfeeedb46moqNCpU6d0/fr1b+3bpk0bRUVFfWs/AAAAAP6BogQAAAAQ4EpLS7Vq1SqNHDlSr776qubPn69r164pNTVVhw8fbtD/rbfe0uLFi5Wenq6XXnpJx48f14MPPqirV6+6+5w4cUKDBw/Wp59+qtmzZ2vhwoWKiIjQ+PHjtWXLlm+MJy8vT3369NHSpUube6gAAAAA7nFWXwcAAAAA4O5q3bq1zp49K5vN5t72zDPPqHfv3lqyZInefPNNj/6nT59WQUGBOnbsKEl6+OGHlZKSoldffVWvv/66JGn69Onq0qWL9u3bJ7vdLkmaOnWqhg0bplmzZmnChAleGh0AAAAAf8KdEgAAAECACw4OdhckXC6XiouLVVtbq+TkZB08eLBB//Hjx7sLEpI0aNAgpaSkaNu2bZKk4uJi7d69W48//ricTqeuX7+u69ev68aNG0pNTVVBQYEuXrz4P+MZOXKkjDGaP39+8w4UAAAAwD2PogQAAADQAqxbt079+/dXaGio2rZtq/bt2+v999/XzZs3G/RNSEhosO2+++7T2bNnJX15J4UxRnPmzFH79u09PvPmzZMkFRUV3dXxAAAAAPBPPL4JAAAACHDr16/XpEmTNH78eM2cOVMxMTEKDg5WRkaGzpw50+jvc7lckqTf/va3Sk1NvW2fnj17NilmAAAAAIGJogQAAAAQ4DZt2qQePXpo8+bNslgs7u31dzV8XUFBQYNtn332mbp16yZJ6tGjhyQpJCREo0ePbv6AAQAAAAQsHt8EAAAABLjg4GBJkjHGvS03N1c5OTm37b9161aPd0Lk5eUpNzdXY8aMkSTFxMRo5MiRWrFihS5fvtzg569du/aN8VRUVOjUqVO6fv16o8cCAAAAwL9xpwQAAAAQAFavXq0dO3Y02D59+nQ98sgj2rx5syZMmKCxY8eqsLBQy5cvV2JiosrKyhr8TM+ePTVs2DBNmTJFVVVVWrRokdq2bavf/e537j7Lli3TsGHD1K9fPz3zzDPq0aOHrl69qpycHF24cEFHjhz5n7Hm5eXphz/8oebNm/etL7u+efOmlixZIkn6+OOPJUlLly5VdHS0oqOjNW3atO/yzwMAAADgHkFRAgAAAAgAmZmZt90+adIkTZo0SVeuXNGKFSu0c+dOJSYmav369dq4caP27t3b4Gd+8YtfKCgoSIsWLVJRUZEGDRqkpUuXqkOHDu4+iYmJ2r9/v37/+99r7dq1unHjhmJiYpSUlKS5c+c227i++OILzZkzx2PbwoULJUldu3alKAEAAAD4GYv56j3cAAAAAAAAAAAAdwnvlAAAAAAAAAAAAF5BUQIAAAAAAAAAAHgFRQkAAAAAAAAAAOAVFCUAAAAAAAAAAIBXUJQAAAAAAAAAAABeQVECAAAAAAAAAAB4hbWxP+ByuXTp0iVFRUXJYrHcjZgAAAAAAAAAAICfMMbI6XQqPj5eQUHffC9Eo4sSly5dUufOne84OAAAAAAAAAAAEHjOnz+vTp06fWOfRj++KSoq6o4DAuB/goOD1blzZw0fPlyHDh3SmTNn9PLLL+vxxx8PiOPBgAED9Oijj2rz5s26ceOGXnvtNT3yyCPfevD0BwkJCXr55Ze1evVqnT17VsuXL1dISIivw2qSoKAgdevWTYMHD9auXbtUWFioNWvWaN68eYqPj/d1eE3Wq1cvjRs3Tm+99ZZKSkr0xhtvaOjQoQExtvvvv19bt27Vv//9bx05ckSrV69Wu3btZLfbfR1ak7Rr1069evXS+++/r+LiYp04cUJ79uxRUlKSr0NrsoSEBD355JNauXKlSkpKdPLkSe3YsUOTJk3ydWhN1r59e82cOVPLly/XhQsXdPDgQT355JMaPHiwXx8nu3fvrsGDB+svf/mLCgsLtX//fn3wwQcaNWqUbDabX9/l3Lt3b82YMcN9fDx8+LBWrVqltLQ0X4fWZG3atNErr7yiTZs26fz588rPz9fkyZM1cuRIhYWF+Tq8OxYZGanY2FhlZGSooKBAR44cUW5ursaNG+fr0JqsS5cumjp1qjIzM3Xt2jUdP35cK1as0OTJkxUcHOzr8JokPDxcv/71r5WZmamCggJdvHhRK1eu1OzZsxUXF+fr8Jps6tSpOnnypC5cuKCSkhItWrRIw4cPV8eOHX0d2h2LjIxUcnKyJk+erMLCQhUXF6ukpETZ2dlKTU1V3759v/V/q96rrFarfv7zn+vVV1/V0aNHVVxcrIMHD2rbtm1KSkry23UtODhYoaGhSktLU15enk6fPq2SkhJt27ZNM2bM0AMPPODrEJskODhYPXr00N/+9jdt375db7/9tjIyMhQbGyubzebr8AB4wXf5fWGj75Tw14M+gDtjsVgUFBQkq9WqqKgoRUVFKTQ01O9/uVEvODhYISEhioiIkMPhUFhYmEJCQvz2xP2r6k92w8PD3WPz93321fkYGRmpqKgohYeHKzQ0NGD2WUhIiMc+s1qtATO2iIgIj/0WFBTk13Oyfj7Wj83hcKisrEwVFRV+/0sp6ct9ZrPZ3PPR6XQqIiIiIC4mg4KCZLfbFRYWJofDoVu3bslms8lqtfr1nKw/Ptbvs6qqKtXU1LjHZbFYZIzxdZh3JDg4WHa73T22+uNI/fmIv45L+nK/fXW9rs89f5+P9cfI0NBQRUVFKTg4WFar1a8Lf/W+fgypqKhQWFhYQJwfWywW99iioqIUERGh8PBw2e32gDgfsdvtioqKksPhUGRkZECca1ksFlmtVtlsNjkcDjkcDgUFBSkyMlIhISEKDg7223lpsVhks9nc87F+v0VERLjH5Y/H//o12Wazuc+NHQ6HIiIiZLfbZbU2+ld195T64394eLgiIiJUW1vrvl7z17kIoHG+S67778oLAAAAAAAAAAD8CkUJAAAAAAAAAADgFRQlAAAAAAAAAACAVzT6QXX++Lw+AHfOGCOXy6Xa2lo5nU5ZLBZVVlaquro6II4HdXV1qqmpUXl5uUpLS3Xr1i3V1NTI5XL5OrQmq6urU2VlpSoqKtxj8/d99tX5WFZWJqfTqYqKClVWVgbMPqupqfHYZ7W1tQEztvLycvfzZCsqKuRyufx6TtbPx/qxlZaWyul0qqysTHV1db4Or8nq6upUXV3tno9Op1Pl5eWqrq72dWhN5nK5VFVVpVu3brnHVl1drdraWr+ek/XHx/p9VlZWpvLycve4/HlsdXV1qqqq8piPFRUVAXE+4nK5PNbr+nH5+3ysP0ZWVlZ6rNc1NTW+Dq3JbncMuXXrVkDMR2OMe2xOp1N1dXWqqKhQVVVVQJyPVFVVua9pXC5XQJxrGWNUW1ur6upqlZaWup/nX1ZWppqaGtXV1fntvDTGqLq62j0fv7q2+fu46sdWVlYmm80mu92u8vJyVVVVqba21tchNkn98b+iokJ2u93jes1f9xmAxvkuuW4xjTwiXLhwQZ07d77joAAAAAAAAAAAQOA5f/68OnXq9I19Gl2UcLlcys/PV2Jios6fPy+Hw9GkIIFAUVpaqs6dO5MXwFeQF4AncgJoiLwAGiIvgIbIC8ATOYF7jTFGTqdT8fHxCgr65rdGNPrxTUFBQerYsaMkyeFwMOmBryEvgIbIC8ATOQE0RF4ADZEXQEPkBeCJnMC9pFWrVt+pHy+6BgAAAAAAAAAAXkFRAgAAAAAAAAAAeMUdFSXsdrvmzZsnu93e3PEAfou8ABoiLwBP5ATQEHkBNEReAA2RF4AncgL+rNEvugYAAAAAAAAAALgTPL4JAAAAAAAAAAB4BUUJAAAAAAAAAADgFRQlAAAAAAAAAACAV1CUAAAAAAAAAAAAXkFRAgAAAAAAAAAAeMUdFSWWLVumbt26KTQ0VCkpKcrLy2vuuIB7wocffqgf/ehHio+Pl8Vi0datWz3ajTGaO3euOnTooLCwMI0ePVoFBQUefYqLi5WWliaHw6Ho6GhNnjxZZWVlXhwF0LwyMjL0gx/8QFFRUYqJidH48eOVn5/v0aeyslLp6elq27atIiMj9eijj+rq1asefc6dO6exY8cqPDxcMTExmjlzpmpra705FKBZZGZmqn///nI4HHI4HBoyZIi2b9/ubicfAGnBggWyWCx6/vnn3dvIDbQ08+fPl8Vi8fj07t3b3U5OoCW6ePGifvazn6lt27YKCwtTv379tH//fnc719xoabp169ZgrbBYLEpPT5fEWoHA0eiixNtvv60ZM2Zo3rx5OnjwoAYMGKDU1FQVFRXdjfgAnyovL9eAAQO0bNmy27a/9tprWrx4sZYvX67c3FxFREQoNTVVlZWV7j5paWk6ceKEsrKy9N577+nDDz/Us88+660hAM0uOztb6enp+uSTT5SVlaWamho99NBDKi8vd/d54YUX9O6772rjxo3Kzs7WpUuX9JOf/MTdXldXp7Fjx6q6ulr/+c9/tG7dOq1du1Zz5871xZCAJunUqZMWLFigAwcOaP/+/XrwwQc1btw4nThxQhL5AOzbt08rVqxQ//79PbaTG2iJ7r//fl2+fNn9+eijj9xt5ARami+++EJDhw5VSEiItm/frpMnT2rhwoVq3bq1uw/X3Ghp9u3b57FOZGVlSZIee+wxSawVCCCmkQYNGmTS09Pdf6+rqzPx8fEmIyOjsV8F+BVJZsuWLe6/u1wuExcXZ/7whz+4t5WUlBi73W7+/ve/G2OMOXnypJFk9u3b5+6zfft2Y7FYzMWLF70WO3A3FRUVGUkmOzvbGPNlHoSEhJiNGze6+3z66adGksnJyTHGGLNt2zYTFBRkrly54u6TmZlpHA6Hqaqq8u4AgLugdevWZtWqVeQDWjyn02kSEhJMVlaWGTFihJk+fboxhrUCLdO8efPMgAEDbttGTqAlmjVrlhk2bNj/bOeaGzBm+vTp5nvf+55xuVysFQgojbpTorq6WgcOHNDo0aPd24KCgjR69Gjl5OQ0V50E8AuFhYW6cuWKRz60atVKKSkp7nzIyclRdHS0kpOT3X1Gjx6toKAg5ebmej1m4G64efOmJKlNmzaSpAMHDqimpsYjN3r37q0uXbp45Ea/fv0UGxvr7pOamqrS0lL3/y4H/FFdXZ02bNig8vJyDRkyhHxAi5eenq6xY8d65IDEWoGWq6CgQPHx8erRo4fS0tJ07tw5SeQEWqZ33nlHycnJeuyxxxQTE6OkpCStXLnS3c41N1q66upqrV+/Xk8//bQsFgtrBQJKo4oS169fV11dncfElqTY2FhduXKlWQMD7nX1c/6b8uHKlSuKiYnxaLdarWrTpg05g4Dgcrn0/PPPa+jQoerbt6+kL+e9zWZTdHS0R9+v58btcqe+DfA3x44dU2RkpOx2u5577jlt2bJFiYmJ5ANatA0bNujgwYPKyMho0EZuoCVKSUnR2rVrtWPHDmVmZqqwsFDDhw+X0+kkJ9Ai/fe//1VmZqYSEhK0c+dOTZkyRb/5zW+0bt06SVxzA1u3blVJSYkmTZokifMnBBarrwMAAPiv9PR0HT9+3ON5yEBL1KtXLx0+fFg3b97Upk2bNHHiRGVnZ/s6LMBnzp8/r+nTpysrK0uhoaG+Dge4J4wZM8b95/79+yslJUVdu3bVP/7xD4WFhfkwMsA3XC6XkpOT9X//93+SpKSkJB0/flzLly/XxIkTfRwd4HtvvvmmxowZo/j4eF+HAjS7Rt0p0a5dOwUHBzd4q/vVq1cVFxfXrIEB97r6Of9N+RAXF9fgJfC1tbUqLi4mZ+D3pk2bpvfee0979uxRp06d3Nvj4uJUXV2tkpISj/5fz43b5U59G+BvbDabevbsqYEDByojI0MDBgzQn/70J/IBLdaBAwdUVFSk73//+7JarbJarcrOztbixYtltVoVGxtLbqDFi46O1n333afTp0+zXqBF6tChgxITEz229enTx/1YM6650ZJ9/vnn+uCDD/SrX/3KvY21AoGkUUUJm82mgQMHateuXe5tLpdLu3bt0pAhQ5o9OOBe1r17d8XFxXnkQ2lpqXJzc935MGTIEJWUlOjAgQPuPrt375bL5VJKSorXYwaagzFG06ZN05YtW7R79251797do33gwIEKCQnxyI38/HydO3fOIzeOHTvmcQGRlZUlh8PR4MIE8Ecul0tVVVXkA1qsUaNG6dixYzp8+LD7k5ycrLS0NPefyQ20dGVlZTpz5ow6dOjAeoEWaejQocrPz/fY9tlnn6lr166SuOZGy7ZmzRrFxMRo7Nix7m2sFQgojX0z9oYNG4zdbjdr1641J0+eNM8++6yJjo72eKs7ECicTqc5dOiQOXTokJFkXn/9dXPo0CHz+eefG2OMWbBggYmOjjb//Oc/zdGjR824ceNM9+7dza1bt9zf8fDDD5ukpCSTm5trPvroI5OQkGCeeuopXw0JaLIpU6aYVq1amb1795rLly+7PxUVFe4+zz33nOnSpYvZvXu32b9/vxkyZIgZMmSIu722ttb07dvXPPTQQ+bw4cNmx44dpn379uall17yxZCAJpk9e7bJzs42hYWF5ujRo2b27NnGYrGYf/3rX8YY8gGoN2LECDN9+nT338kNtDQvvvii2bt3ryksLDQff/yxGT16tGnXrp0pKioyxpATaHny8vKM1Wo1r7zyiikoKDB//etfTXh4uFm/fr27D9fcaInq6upMly5dzKxZsxq0sVYgUDS6KGGMMUuWLDFdunQxNpvNDBo0yHzyySfNHRdwT9izZ4+R1OAzceJEY4wxLpfLzJkzx8TGxhq73W5GjRpl8vPzPb7jxo0b5qmnnjKRkZHG4XCYX/7yl8bpdPpgNEDzuF1OSDJr1qxx97l165aZOnWqad26tQkPDzcTJkwwly9f9vies2fPmjFjxpiwsDDTrl078+KLL5qamhovjwZouqefftp07drV2Gw20759ezNq1Ch3QcIY8gGo9/WiBLmBluaJJ54wHTp0MDabzXTs2NE88cQT5vTp0+52cgIt0bvvvmv69u1r7Ha76d27t/nzn//s0c41N1qinTt3GkkN5roxrBUIHBZjjPHJLRoAAAAAAAAAAKBFadQ7JQAAAAAAAAAAAO4URQkAAAAAAAAAAOAVFCUAAAAAAAAAAIBXUJQAAAAAAAAAAABeQVECAAAAAAAAAAB4BUUJAAAAAAAAAADgFRQlAAAAAAAAAACAV1CUAAAAAAAAAAAAXkFRAgAAAAAAAAAAeAVFCQAAAAAAAAAA4BUUJQAAAAAAAAAAgFf8P/riCZtWZDalAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FOOqKQWLL2A0"
      },
      "source": [
        "## The most simple Network "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FVjU-G023yYc"
      },
      "source": [
        "All features (pixel values) are mapped to probabilities for all 10 labels\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mc5JrZik2cqa"
      },
      "source": [
        "# A Dense Layer ist an all-to-all conection of input and output neurons\n",
        "out_layer = tf.keras.layers.Dense(\n",
        "    input_shape=(784,),             \n",
        "    units=10,                      # output: 10 units for 10 categories\n",
        "    activation='softmax',          # softmax-activation, so that probabilities sum up to 1\n",
        "    name = 'probabilities'\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C9vs1_u279O2"
      },
      "source": [
        "model1 = tf.keras.models.Sequential()\n",
        "model1.add(out_layer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tGv2G8eN_5tD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201
        },
        "outputId": "80558c9d-450d-414e-e345-99aee1227eff"
      },
      "source": [
        "tf.keras.utils.plot_model(model1, show_shapes=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAC4CAIAAACn5ypdAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO2de1RTV77H94GEnCQkEOQpAhLAIohyqbYSZSm6lr3qkgpIpSOdK04r+ALKYygPKSJYUhxkYaVWpdw1OioqFqyKzoWWWhbqtVe4PKwIKK9GBEVIAolAOPePPXNuJpAYSEgi3Z+/OPv52498OWfvfc4PIwgCIBAIhAYY6dsABALxxoN0BIFAaArSEQQCoSlIRxAIhKZQ5C9u376dm5urL1MQCMSbgq+vb2xsLHn5L/cjXV1dly5d0rlJCF1w586dO3fu6NuKGaG7uxvNW11y586d27dvy4dQJia6ePGiruxB6I6QkBAwSwf3woULW7dunZVNM0zgXJIHrY8gEAhNQTqCQCA0BekIAoHQFKQjCARCU5COIBAITdGbjixbtszY2Njb21tbWa5fv25mZvb9998DAA4fPmxtbY1h2PHjxyfGqs6rRWaoWJ3xptuvQGRkJPZPwsLC5KMqKiqSkpJKSkq4XC5M8NFHH8knWLduHYvFMjY29vT0vH//vi7NXr16NTYBU1NTGHv27Nlly5axWCwnJ6fw8PCenp6JJUilUnd399TUVADAlStX+Hy+TCYjY0tLS8liLS0tp2ek3nTk3r17/v7+Wswi/+JyfHx8TU2NsljVebXIm/4u9Ztu/0QsLCzKy8ubm5sLCwvJwM8//zw/Pz85OTk4OPjx48cuLi5z5sw5c+bMtWvXyDR///vfL168uGnTpqamJh8fH33Y/i+sXLkSAFBcXLxt27aQkJDu7u6ysrJbt26tX79+bGxMIXFKSkpzczP8OyAgAMfxtWvXDgwMwJD333+/u7v71q1bGzZsmLY9en6uwTBMW1k2btw4ODi4adMmdWIlEgmPx1Mz77SZoWJJFFqhdd50+ydCp9P//d//fcGCBTQaDYZkZ2efP3/+woULLBaLTJafn29kZBQRETE4OKhL8yYFx3GhUEjIERER8ec//xkA8M0338ydOzchIcHMzMzb2zs2Nrauru7u3bvy2WtqahobG+VDoqOjlyxZsmHDBqg4GIbZ29v7+fm5ublN20g96wiVStVBlokUFhb29vZqXo5+edNboXf7W1tb9+/ff+DAARzH5cN5PF5MTMxvv/0WHx+vL9tIbty4Ia9xXV1djY2Na9asgX/b2dmR/1kdHBwAAB0dHWRiiUSSkJCQl5enUGZ6enpdXd3E8GkzZR3Jz8/Hcdza2joyMtLOzg7HcR6PByXwyy+/ZDAYLBart7c3Li7O3t6+ubmZIIjc3NyFCxfSaDQOh7N58+aHDx+SpbW2trq7uzOZTDqd7ufnV11dDcN//vlnDw8PMzMzHMe9vLxu3rypOkt1dbWjoyOGYV999dVEmxViY2Ji4uLi2traMAxzdXWdmFcmk6WlpTk6OtLp9MWLFxcXF8Pwn3766Z133mEwGGw228vLSygUqugohWILCgqYTCaDwSgrK1u/fj2bzZ43b965c+dUd2lUVJSJiYmtrS0sc8+ePUwmE8Ow58+fK7RiquP4WnRv/40bN9hsdlZWltbbooz8/HyCIAICAiZGZWZmLliw4NSpUxUVFRNjlc1qFb0ElM+rKZGdnR0dHQ3/5nK58kIMF0e4XC4ZkpKSsmfPHisrK4VCOBzOqlWr8vLytPboKn+/BBtGvI6IiAgmk/ngwQOpVNrU1ASXeTo7OwmCSElJAQBER0cfPXo0KCjo119/TUtLMzExOX369MDAQH19vY+Pj6WlZU9PD0EQa9eu5XK5T548GR0dbWxsfPfdd3Ecf/ToEUEQFy9eTE9P7+/vf/HixfLly+fMmQOrVpGlq6sLAHD06FGYsqWlBQDw9ddfw0uF2ODgYBcXF7JFCrHx8fE0Gu3SpUsvX75MTk42MjK6d++eWCxms9l8Pl8ikfT09AQFBfX19anuKIViYedUVlYODg729vb6+fkxmcyRkRHVXbpt2zYbGxuyzJycHAAArFqhFarZsmXLli1b1EysF/uvXr3KYrEyMjKmZCQxlXlrb28vH8Llcj08PBSSubi4PHnyhCCImpoaIyOj+fPni8VigiDKy8vff/99mEbFrFbRS5POqym1tLu728PDQyaTwcuqqioqlZqfny8UChsbGxcuXPjee++RiaurqwMCAgiC6Ovrg5oiX1RSUhIAoLa2lgyJjo4mf2iqmTiXpvlcQ6FQoBh7eHgUFBSIRKKioiIyNjs7e+/evSUlJU5OTrm5uUFBQWFhYWZmZl5eXsePH3/+/PmJEydgShaLNX/+fAqF4unpefLkSalUCqO2bNny+eefczgcCwuLgICAFy9ewL5QkUVbSKXSgoKCwMDA4OBgc3Pz1NRUKpVaVFTU3t4uFAo9PT1xHLexsSkpKZne4jaPx2Oz2VZWVqGhoUNDQ52dnTBcdZcaDjNn/8aNG4VC4f79+2fA6kkYGhp68uSJi4uLsgS+vr6ffvppe3v7Z599Jh8ukUhUz2owWS8pm1dTsjk7O3vfvn1GRv/42a5atSoxMTEqKorNZi9atEgkEp06dYo0MiYmpqCgQFlRcDWkoaFhSgYoQwvrI0uXLmUwGPJPKyRNTU1isXjp0qVkyLJly0xMTBSWgiBeXl5mZmb19fUK4XBBRH6n6rVZNKG5uXl4eHjRokXwkk6n29raPnz4kMvlWltbh4WFpaent7e3a16RiYkJAGB0dHRilIouNRzedPt7e3sJgmAwGCrSZGZmvvXWW8eOHSOfuMEUZzXZS8rmlfoGCwSCK1eubN++nQxJSUk5ceJEZWWlWCx+/Pgxj8fz9fWFd5HJyck7d+60t7dXVhps+LNnz9Q3QAXaWWel0Wjk/YI8cG+J3OuGmJubi0SiScuhUqlwXl67dm316tVWVlY0Gg0uTSuDzKIthoaGAACpqankpnpHR8fw8DCdTv/hhx9WrlyZlZXF5XJDQ0MlEokW61VAWZe+KRi+/VKpFABAbtxMCo7jRUVFGIbt2LGDHO6pzmqIsnmlvsF8Pv+TTz4hl4SfPn3K5/N37ty5Zs0aJpPp7Ox88uRJgUCQk5NTXV3d0NDw8ccfqyiNTqeDf3aC5mhBR0ZHRwcGBubNmzcxytzcHACg0L/KEo+NjfX39zs6OnZ2dgYGBtra2t69e3dwcJDP5yurmsyicSP+H7godeTIEfnHP/i1BU9Pz++//14gECQmJhYXFx8+fFiL9cqjokvfCN4I++EPadL7XHngB3taWloOHjwIQ6Y0q0lUzCt16OnpOXv27O7du8mQlpYWmUw2d+5cMoTNZltYWDQ1NRUWFlZWVhoZGUHBglVnZWVhGPbLL7/AxCMjI2QnaI4WdKSqqoogiOXLl0+MWrRokampKWk6AODu3bsjIyNvv/32xMQ//vjj+Pi4j49PQ0PD6Ojo7t27uVwujuMqzpiQWTRvBYmDgwOO43V1dQrhAoHgwYMHAAArK6svvvjCx8cHXs4E8l1KoVC0e8OlA94I++GJZ3VOiBw8eNDd3b22thZeTmlWkyibV2rC5/PDwsIsLCzIEChbT58+JUNEIlF/f7+Dg0NRUZG8Wsmvs5KPY7DhNjY207NHgWnqyPj4+MuXL8fGxurr62NiYhwdHeUf20hwHI+Li7t8+fKZM2eEQmFDQ8OuXbvs7OwiIiJggpGRkcHBwbGxsfv370dFRTk5OW3fvh3eX1RUVEil0paWFoXHzkmzTNV+CwsLgUDQ3t4uEokUZjmO4+Hh4efOnSsoKBAKhTKZrLu7++nTpwKBIDIy8uHDhyMjI7W1tR0dHZNK57RR1qWurq79/f2lpaWjo6N9fX3ypwNUtEL3aG5/eXm5Lvd9GQwGl8vt7u5+bUr4dGNsbExeqp7VygqZdF4BAEJDQ21sbFQct3/27Nm333776aefygc6Ozv7+/ufPHny1q1bEomkq6sLGvCnP/1JnebDhnt5eamT+PXI65b6+2dUKtXe3p5CobDZ7M2bN7e1tREEwefz4W2Sg4PD6dOnYeLx8fGcnBw3NzcqlcrhcAIDA+GhEoIgioqK/P39ra2tKRTKnDlzPvzww46ODhiVmJhoYWFhbm4eEhICzy+4uLh0dnYqy3L06FF4SIHBYAQEBPzlL3+BQstkMoOCghRiCYK4f/++k5MTnU5fuXJlamqqQuyrV68SExMdHR0pFIqVlVVwcHBTU1N7ezuPx+NwOMbGxnPnzk1JSRkbG1PRSwqVHjt2DK5subm5tbW1nThxgs1mAwCcnJwePXqkrEsJgnjx4oW/vz+O487Ozvv27UtISAAAuLq6dnZ2yrcCbjqqYKr7vrq3//r16ywWKzMzU30jIdPe942KiqJSqcPDw/Dy8uXLcPvG0tJy7969CtkTEhLIfV9ls1p1L006rwiCCAwMBACkpaUpszw2NjYsLGxiODyG4+rqSqPRTE1NV6xY8d13301MNum+78aNG+3t7cfHx8kQTfZ9p6kjFhYW6tSHUBMddOk0zo+oj36nxLR1pKWlhUKhkP/z9IVMJvPz8yssLNRZjc+fP8dx/PDhw/KBejg/8trVKcRUedO79I2wXyKR3Lx5s6WlBa4yurq6ZmRkZGRkiMVifZkkk8lKS0tFIlFoaKjOKk1PT/f29o6KigIAEAQhEAiqq6tbW1unXSD6/sj0efjw4cQXukl0OS0QatLf3w/f09uxYwcMSUpKCgkJCQ0N1dcreVVVVSUlJeXl5apPsmiR3Nzcurq669evw5NZZWVl8D09+febp4z8zYk694dJSUnwaM38+fMvXryo1l0UQiW66dKZe67R+5RQ87lGBTdv3kxMTNSWPYZMaWnpoUOHVC/tvZaJcwkj5F7Ugd/vJ2bdVycQ4HfgdwLNW50xcS6h5xoEAqEpSEcQCISmIB1BIBCagnQEgUBoCtIRBAKhKZP4CZ/Gt5cRbwqzeHBncdMMkC1btshfTqIj0/tsJMLAOXLkCABA4V2v2cHt27fz8vLQvNUZcC7JM4mOfPDBBzoxBqFT4G7/bB3cvLy82do0A2TiKSS0PoJAIDQF6QgCgdAUpCMIBEJTkI4gEAhNQTqCQCA0xSB05M6dOwsXLoSft7axscnMzJzpGktKSrhcLvxQiK2tbVhY2EzXiNA7kZGR5NdhFEa8oqIiKSlJflZ89NFH8gnWrVvHYrGMjY09PT1VfEh1Jli9evXEr9uQXi/Onj0LvRc6OTmFh4dD15wKSKVSd3f31NRUAMCVK1f4fL78R6dKS0vJYqfn2g2AaX2fdYZ47733AAAvX77UWY0uLi5mZmY6q06/zOh3FfXLlL4HWl5e3tzcLJVKyfC0tLRNmzYJhUJ46eLiMmfOHADA1atX5bPL++XUJatWrZr4s4X+N8+fPw8A4PP5AwMDtbW1XC7X29t7dHRUoYTY2Fgg933WvLy8VatWkT+08fHx7u7uW7dubdiwQdffVXwTkUgkPB5P31bMTrTStzoYIDqdDr+HRrq/ys7OPn/+/IULF1gsFpksPz/fyMgoIiJCXx9JkwfHcVLjIBEREdA53DfffDN37tyEhAQzMzNvb+/Y2Ni6ujoF/wo1NTWNjY3yIdHR0UuWLNmwYcPY2BgAAMMw+D006KlzevyOdKSwsFDeOTtCi2ilb3U/QK2trfv37z9w4ADppA7C4/FiYmJ+++23+Ph4XdozKTdu3JDXuK6ursbGxjVr1sC/7ezsyBcCHBwcAADynj0kEklCQkJeXp5Cmenp6XV1dRPDp42B6khBQQGTyWQwGGVlZevXr2ez2fPmzTt37hwAID8/H8dxa2vryMhIOzs7HMd5PB7U4KioKBMTE+gtAQCwZ88eJpOJYRj8PH9cXFxbWxuGYa6ururY8PPPP3t4eJiZmeE47uXldfPmTQDAxx9/DJ8kXVxcoGOk8PBwBoNhZmZ25coVmUyWlpbm6OhIp9MXL14M77e//PJLBoPBYrF6e3vj4uLs7e2bm5tnqN80hyCI3Nxc6PGbw+Fs3rwZ+qBVv2+1NUA3btyYaXc2+fn5BEEEBARMjMrMzFywYMGpU6cqKiomxirrJRXzFgAw6fSYKtnZ2dHR0fBvLpcrr7xwcYTL5ZIhKSkpe/bsgf705OFwOKtWrcrLyyO09RE5+fslg1ofSUlJAQBUVlYODg729vb6+fkxmcyRkRGCICIiIphM5oMHD6RSaVNTE1xn6uzsJAhi27ZtNjY2ZJk5OTkAgL6+PoIggoODXVxc5GtUvT5y8eLF9PT0/v7+Fy9eLF++nHx0DA4ONjY2/u2338iUf/jDH65cuUIQRHx8PI1Gu3Tp0suXL5OTk42MjO7du0e2JTo6+ujRo0FBQb/++quW+mwKqLk+kpaWZmJicvr06YGBgfr6eh8fH0tLS+gcR/2+1coAXb16lcViZWRkvNbmafud4HK5Hh4eCslcXFyePHlCEERNTY2RkdH8+fPFYjHxr+sjKnpJxbxVNj3Up7u728PDQyaTwcuqqioqlZqfny8UChsbGxcuXAjXTSDV1dXQH9Ok/muSkpIAALW1tWSIHvxO6Awej8dms62srEJDQ4eGhjo7O2E4hUKB/w08PDwKCgpEIlFRUZF2q96yZcvnn3/O4XAsLCwCAgJevHgBx2PXrl0ymYysTigU3rt3b8OGDVKptKCgIDAwMDg42NzcPDU1lUqlyluVnZ29d+/ekpISd3d37ZqqLSQSSW5ublBQUFhYmJmZmZeX1/Hjx58/f37ixImpFqX5AG3cuFEoFO7fv3+qVavJ0NDQkydPoOOrSfH19f3000/b29s/++wz+XB1emnivH3t9FCH7Ozsffv2GRn942e7atWqxMTEqKgoNpu9aNEikUh06tQp0siYmJiCggJlRcHVkIaGhikZoAxD1xES+EXySb1PLl26lMFgwBvLGQJ+oR/ulq1Zs2bBggXffvstQRAAgPPnz4eGhhobGzc3Nw8PDy9atAhmodPptra2M2qV1mlqahKLxaQLWADAsmXLTExMFJbupooOBmga9Pb2EgSh2ttDZmbmW2+9dezYserqajJwSr1EzlvNp4dAILhy5Yq8F9qUlJQTJ05UVlaKxeLHjx/zeDxfX9+uri4AQHJy8s6dO+3t7ZWVBhv+7Nkz9Q1QwRujI6qh0WjwZkGLXLt2bfXq1VZWVjQaDS6PQzAMi4yMfPz4cWVlJQDgr3/9K/SoOjQ0BABITU0ld+M7OjqGh4e1a9WMMjAwAAAgzyZAzM3NRSKRhiXPxABpiFQqBQCQGzeTAj37Yhi2Y8cOiUQCA6fXS5pPDz6f/8knn5BLwk+fPuXz+Tt37lyzZg2TyXR2dj558qRAIMjJyamurm5oaPj4449VlAZd6MJO0JzZoCOjo6MDAwPQ/brm3Lp168iRI52dnYGBgba2tnfv3h0cHOTz+fJptm/fjuP4qVOnmpub2Wy2k5MTAACuZh05ckT+ufH27dtasUo3mJubAwAUfg+a9612B0hbwB/Sa90A+vr6xsbGtrS0HDx4EIZMr5c0nB49PT1nz57dvXs3GdLS0iKTyebOnUuGsNlsCwuLpqamwsLCyspKeLATwzBYdVZWFoZhv/zyC0wMPQrCTtCc2aAjVVVVBEEsX74cAEChUCZ99lGf//mf/2EymQ0NDaOjo7t37+ZyuTiOK3xri8PhbN26tbS09PDhw5988gkMdHBwwHG8rq5Ok9r1y6JFi0xNTcmpBgC4e/fuyMjI22+/DTToW+0OkLawtrbGMEydEyIHDx50d3eH23Pgdb2kDA2nB5/PDwsLs7CwIEOgbD19+pQMEYlE/f39Dg4ORUVF8molv85KPo7BhtvY2EzPHgXeVB0ZHx9/+fLl2NhYfX19TEyMo6MjfG50dXXt7+8vLS0dHR3t6+uT30u3sLAQCATt7e0ikWjSqTw6Ovrs2bOqqiomk+no6AgAqKiokEqlLS0tEx99d+3a9erVq6tXr27atAmG4DgeHh5+7ty5goICoVAok8m6u7vlh9nwwXE8Li7u8uXLZ86cEQqFDQ0Nu3btsrOzi4iIAFPsW80HqLy8fEb3fRkMBpfL7e7ufm1K+HRjbGxMXqroJRWFKJseoaGhNjY2Ko7bP3v27Ntvv1X4lp2zs7O/v//Jkydv3bolkUi6urqgAfAp+7XAhnt5eamT+PXI65a+9n3v3Lnj6ekJV6FtbW2zsrKOHTsG14Hc3Nza2tpOnDjBZrMBAE5OTo8ePYqIiKBSqfb29hQKhc1mb968ua2tDRb14sULf39/HMednZ337duXkJAAAHB1de3s7Lx//76TkxOdTl+5cuXXX3+tYqH+8uXLBEEkJiZaWFiYm5uHhIR89dVXAAAXFxe4eQn5t3/7t6SkJPmGvHr1KjEx0dHRkUKhWFlZBQcHNzU18fl8ePfo4OCgR9f2au77jo+P5+TkuLm5UalUDocTGBjY3NwMo9Ts256eHs0HqKen5/r16ywWKzMz87U2T3vfNyoqikqlDg8Pw8vLly/DWWFpabl3716F7AkJCeS+r7JeUj1vJ50eBEEEBgYCANLS0pRZHhsbGxYWNjEcnrtxdXWl0WimpqYrVqz47rvvJiabdN9348aN9vb24+PjZIgm+74GoSNTBb4ooW8riA0bNjx+/FjfVqiLLt+v0fEATVtHWlpaKBSKHsUdIpPJ/Pz8CgsLdVbj8+fPcRw/fPiwfOBsPj+ijNcuj80Q5ANRfX09/KeqFzMMH30NkGokEsnNmzdbWlrgKqOrq2tGRkZGRoZYLNaXSTKZrLS0VCQShYaG6qzS9PR0b2/vqKgoAABBEAKBoLq6urW1ddoFvqk6oi8SExNbWloePXoUHh5OLuAj3hT6+/vhe3o7duyAIUlJSSEhIaGhofp6Ja+qqqqkpKS8vFz1SRYtkpubW1dXd/36dXgqqqysDL6nd+3atekXKn9z8kY81yQlJcGzPfPnz7948aKOa09JSTEyMnJwcIAH4d8gdPZco/sB0nze3rx5MzExUVv2GDKlpaWHDh0aGxvTpJCJcwkj5F7UuXDhwtatWwltvbqDMCRCQkLAZB4DZgFo3uqYiXMJPdcgEAhNQTqCQCA0BekIAoHQFKQjCARCUybx73vhwgXd24GYaeA56Fk5uPBtt1nZNMOku7tb8aVE+c0b5LEdgUCog6p9XwRiIhiGFRcXf/DBB/o2BGG4oPURBAKhKUhHEAiEpiAdQSAQmoJ0BIFAaArSEQQCoSlIRxAIhKYgHUEgEJqCdASBQGgK0hEEAqEpSEcQCISmIB1BIBCagnQEgUBoCtIRBAKhKUhHEAiEpiAdQSAQmoJ0BIFAaArSEQQCoSlIRxAIhKYgHUEgEJqCdASBQGgK0hEEAqEpSEcQCISmIB1BIBCagnQEgUBoCtIRBAKhKUhHEAiEpiAdQSAQmoJ0BIFAaArSEQQCoSlIRxAIhKYgHUEgEJqCdASBQGgK0hEEAqEpFH0bgDA4Tp482d/fLx9SVlb25MkT8jI8PNza2lrndiEMF4wgCH3bgDAsIiMjv/nmGxqNNjFqdHSUw+H09PRQKOg/EOL/Qc81CEU+/PBDAMCryTA2Nv7DH/6ARAShALofQShCEIS9vf3Tp08nja2pqfH19dWxSQgDB92PIBTBMGzbtm0mJiYTo+bOnbt8+XLdm4QwcJCOICbhww8/HBkZUQg0MTH5j//4DwzD9GISwpBBzzWIyXFzc2ttbVUIrK+v9/Ly0os9CEMG3Y8gJicsLIxKpcqHuLq6IhFBTArSEcTkhIWFjY2NkZdUKjU8PFyP9iAMGfRcg1CKt7d3fX09nCEYhrW1tTk7O+vbKIQhgu5HEEr54x//aGxsDADAMOztt99GIoJQBtIRhFI+/PDD8fFxAICxsfEf//hHfZuDMFyQjiCUYmdnt2LFCgzDxsfHQ0JC9G0OwnBBOoJQxUcffUQQxOrVq21tbfVtC8KAIXSCvluJQPxOKS4u1sEPXHcvXMXExKD3MrTF1q1bddafR44c2blzJ5PJ1EFdsDoAwKeffqqb6mY3W7du1U1FutMRX1/fDz74QGfVzW62bt2qs/5cuXLl3LlzdVAR5OLFiwAANFW0gs50BK2PIF6DLkUE8YaCdASBQGgK0hEEAqEpSEcQCISmIB1BIBCaMtt0ZNmyZcbGxt7e3trKcv36dTMzs++//x4AcPjwYWtrawzDjh8/PjFWdV69Y1DGaJGKioqkpKSSkhIul4thGIZhH330kXyCdevWsVgsY2NjT0/P+/fv69K21atXYxMwNTWFsWfPnl22bBmLxXJycgoPD+/p6ZlYglQqdXd3T01NBQBcuXKFz+fLZDJdNkFNZpuO3Lt3z9/fX4tZ5A/RxcfH19TUKItVnVfvGJQx2uLzzz/Pz89PTk4ODg5+/Pixi4vLnDlzzpw5c+3aNTLN3//+94sXL27atKmpqcnHx0eP1kJWrlwJACguLt62bVtISEh3d3dZWdmtW7fWr18v/6EGSEpKSnNzM/w7ICAAx/G1a9cODAzo2ujXMdt0BDKNb/8py7Jx48bBwcFNmzapEyuRSHg8npp5dcxMG6PQdh2QnZ19/vz5CxcusFgsMjA/P9/IyCgiImJwcFCXxkwKjuNCoVD+3GdERMSf//xnAMA333wzd+7chIQEMzMzb2/v2NjYurq6u3fvymevqalpbGyUD4mOjl6yZMmGDRsmKo5+mZ06ovAhrxnKMpHCwsLe3l7Ny3kT0XHbW1tb9+/ff+DAARzH5cN5PF5MTMxvv/0WHx+vM2OUcePGDXmN6+rqamxsXLNmDfzbzs6O/O/l4OAAAOjo6CATSySShISEvLw8hTLT09Pr6uomhusXQ9GR/Px8HMetra0jIyPt7OxwHOfxeFCev/zySwaDwWKxent74+Li7O3tm5ubCYLIzc1duHAhjUbjcDibN29++PAhWVpra6u7uzuTyaTT6X5+ftXV1TD8559/9vDwMDMzw3Hcy8vr5s2bqrNUV1c7OjpiGPbVV8BjmIsAABRBSURBVF9NtFkhNiYmJi4urq2tDcMwV1fXiXllMllaWpqjoyOdTl+8eHFxcTEM/+mnn9555x0Gg8Fms728vIRCoda7V8GYgoICJpPJYDDKysrWr1/PZrPnzZt37tw51QMRFRVlYmJCvrC3Z88eJpOJYdjz588V2g4AuHHjBpvNzsrK0npbIPn5+QRBBAQETIzKzMxcsGDBqVOnKioqJsYqmzkq+gQoH7spkZ2dHR0dDf/mcrnysgsXR7hcLhmSkpKyZ88eKysrhUI4HM6qVavy8vIM60FVB+/wwAa/9n2hiIgIJpP54MEDqVTa1NQEl6A6OzsJgkhJSQEAREdHHz16NCgo6Ndff01LSzMxMTl9+vTAwEB9fb2Pj4+lpWVPTw9BEGvXruVyuU+ePBkdHW1sbHz33XdxHH/06BFBEBcvXkxPT+/v73/x4sXy5cvnzJkDq1aRpaurCwBw9OhRmLKlpQUA8PXXX8NLhdjg4GAXFxeyRQqx8fHxNBrt0qVLL1++TE5ONjIyunfvnlgsZrPZfD5fIpH09PQEBQX19fVppT8VUDAGdmllZeXg4GBvb6+fnx+TyRwZGVE9ENu2bbOxsSHLzMnJAQBAgxXafvXqVRaLlZGRMSUjCYLYsmXLli1bXpuMy+V6eHgoBLq4uDx58oQgiJqaGiMjo/nz54vFYoIgysvL33//fZhGxcxR0SeTjt2U2tXd3e3h4SGTyeBlVVUVlUrNz88XCoWNjY0LFy587733yMTV1dUBAQEEQfT19QEAUlJS5ItKSkoCANTW1r620mnMk+lhWDpiZmZGXt67dw8AcODAAeKfAyyRSGDU8PCwqalpaGgomfi///u/AQBw1q5du3bJkiVkVH19PQAgPj5eobpDhw4BAHp7e1Vn0ZaOSCQSBoNB2jw8PEyj0Xbv3g0fgK9evaq6cxTQlo6QXXrs2DEAQGtrK6FyINTXkWmjjo6IxWIMwzZt2qQQTuoIQRBxcXEAgL179xJyOqJ65ijrE2VjN6V27d27l5w2ELgLA5k3b15XVxdZ/tKlS7u7uwklOvLtt98CAP7617++tlKd6YihPNdMZOnSpQwGQ/5phaSpqUksFi9dupQMWbZsmYmJicIyFcTLy8vMzAxKgzxwQWTSXTRlWTShubl5eHh40aJF8JJOp9va2j58+JDL5VpbW4eFhaWnp7e3t2uxxikBvV6Njo5OjFIxEPoCqj+DwVCRJjMz86233jp27Bj5VAumOHPIPlE2duobLBAIrly5sn37djIkJSXlxIkTlZWVYrH48ePHPB7P19cXan1ycvLOnTvt7e2VlQYb/uzZM/UNmGkMV0cAADQaDeqxAnDfi9yHh5ibm4tEoknLoVKp8Bdy7dq11atXW1lZ0Wg0uGyuDDKLthgaGgIApKamkucIOjo6hoeH6XT6Dz/8sHLlyqysLC6XGxoaKpFItFivVlA2EPpCKpUCACb1ZE6C43hRURGGYTt27CC7dKozB6Js7NQ3mM/nf/LJJ+SS8NOnT/l8/s6dO9esWcNkMp2dnU+ePCkQCHJycqqrqxsaGj7++GMVpdHpdPDPTjAQDFdHRkdHBwYG5s2bNzHK3NwcAKAw9soSj42N9ff3Ozo6dnZ2BgYG2tra3r17d3BwkM/nK6uazKJxI/4fuGB25MgR+bvB27dvAwA8PT2///57gUCQmJhYXFx8+PBhLdarOSoGQl/AH9JrT2T5+vrGxsa2tLQcPHgQhkxp5pCoGDt16OnpOXv27O7du8mQlpYWmUwm/yI1m822sLBoamoqLCysrKw0MjKCggWrzsrKwjDsl19+gYmhq0PYCQaC4epIVVUVQRCTepNdtGiRqakp2a0AgLt3746MjLz99tsTE//444/j4+M+Pj4NDQ2jo6O7d+/mcrk4jqs4Y0Jm0UpDIA4ODjiO19XVKYQLBIIHDx4AAKysrL744gsfHx94aTjIDwSFQtHubdr0gKeK1TkhcvDgQXd399raWng5pZlDomzs1ITP54eFhVlYWJAhULbkPbGLRKL+/n4HB4eioiJ5tZJfHyEfx2DDbWxspmfPTGBYOjI+Pv7y5cuxsbH6+vqYmBhHR0f5R0oSHMfj4uIuX7585swZoVDY0NCwa9cuOzu7iIgImGBkZGRwcHBsbOz+/ftRUVFOTk7bt2+H9xcVFRVSqbSlpUXhkXjSLFO138LCQiAQtLe3i0Qihd8bjuPh4eHnzp0rKCgQCoUymay7u/vp06cCgSAyMvLhw4cjIyO1tbUdHR2G4Ihb2UC4urr29/eXlpaOjo729fXJn3dQaHt5efnM7fsyGAwul9vd3f3alPDpBnrPAGrMHGWFTDp2AIDQ0FAbGxsVx+2fPXv27bffKnzezdnZ2d/f/+TJk7du3ZJIJF1dXdCAP/3pT+o0HzbcsHwbzvA67j8A6u3XUKlUe3t7CoXCZrM3b97c1tZGEASfz4e3cA4ODqdPn4aJx8fHc3Jy3NzcqFQqh8MJDAyEh0oIgigqKvL397e2tqZQKHPmzPnwww87OjpgVGJiooWFhbm5eUhICDxJ4eLi0tnZqSzL0aNH4XEJBoMREBDwl7/8Bf4TYDKZQUFBCrEEQdy/f9/JyYlOp69cuTI1NVUh9tWrV4mJiY6OjhQKxcrKKjg4uKmpqb29ncfjcTgcY2PjuXPnpqSkjI2NaaU/5VEw9dixY3Ctzs3Nra2t7cSJE2w2GwDg5OT06NEjZQNBEMSLFy/8/f1xHHd2dt63b19CQgIAwNXVtbOzU77tPT09169fZ7FYmZmZ6hsJUXPfNyoqikqlDg8Pw8vLly+7uLgAACwtLeEejTwJCQnkvq+ymaO6TyYdO4IgAgMDAQBpaWnK7IyNjQ0LC5sYDg/duLq60mg0U1PTFStWfPfddxOTTbpfs3HjRnt7+/Hx8df20lTnybQxLB2xsLDQjT1vOjM6P/Q7EGrqSEtLC4VCIf+v6AuZTObn51dYWKizGp8/f47j+OHDh9VJrDMdMaznGsN8l/F3iOEPhKura0ZGRkZGhlgs1pcNMpmstLRUJBKFhobqrNL09HRvb++oqCid1agOhqUjCIT6JCUlhYSEhIaG6uuVvKqqqpKSkvLyctUnWbRIbm5uXV3d9evXtfI6mBYxFB1JTk4uKioaHBx0dna+dOmSvs35/fJmDURWVlZUVNQXX3yhl9rXrl37t7/9TWcewsrKyl69elVVVcXhcHRTo/pghE7e9sEwrLi4GDkT0BazuD+hA1DofQKhITqbJ4ZyP4JAIN5ckI4gEAhNQTqCQCA0BekIAoHQFN3591X/vSaEOszW/oSHvi9cuKBvQxBTQQdn3XSzJYRAICaim/Osursfma37lHoB7fsi1GEajhOmB1ofQSAQmoJ0BIFAaArSEQQCoSlIRxAIhKYgHUEgEJqCdASBQGjKrNKRkpISLpeLyWFiYmJtbb169eqcnJyXL1/q20DEzFJRUZGUlCQ/DT766CP5BOvWrWOxWMbGxp6enio+qjpzjI+PHzlyZKJD9erq6hUrVjAYDDs7u8TExFevXgEArly5wufzDf+bUgAY0vdZtYWLiwt0Bwc/Vvzjjz9u374dwzA7O7up+lI0WHTZnzpGze8qTiQtLW3Tpk1CoRBeuri4zJkzB0zwVSjvo1PHPHr0aMWKFQAAeeeNBEE0NjbS6fT9+/eLxeKamhpLS8vw8HAYlZeXt2rVqpcvX06vRp3Nk1l1P6IAhmHm5uarV68uKiq6cOHCs2fPNm7cqK9vZ70pSCSSif8t9VLIlMjOzj5//vyFCxdYLBYZmJ+fb2RkFBERYQiD/r//+7+fffbZrl27vL29FaIOHjxoa2t74MABJpPp6+ubmJj4n//5n9BfX3R09JIlSzZs2DA2NqYPq9VlNuuIPFu2bNm+fXtvb+/x48f1bYtBU1hY2NvbawiFqE9ra+v+/fsPHDhAOqyD8Hi8mJiY3377LT4+XmfGKGPJkiUlJSXbtm1TcAM4NjZ27dq1VatWkWdP169fTxBEWVkZvExPT6+rq8vLy9O1xVPh96IjAADogaW8vBwAIJPJ0tLSHB0d6XT64sWLi4uLAQAFBQVMJpPBYJSVla1fv57NZs+bN+/cuXMw+08//fTOO+8wGAw2m+3l5SUUCpWVYyAQBJGbm7tw4UIajcbhcDZv3gz/xUVFRZmYmJBfA9yzZw+TycQwDHpCiIuLa2trwzDM1dU1Pz8fx3Fra+vIyEg7Ozscx3k8HvT7o34hAIAbN27MnC8bAEB+fj5BEAEBAROjMjMzFyxYcOrUqYqKCvW7SPVM0O6gP378WCwWyztvhA40SPfSHA5n1apVeXl5hCG/p6aDZydCT+sjCsBfvoODA0EQ8fHxNBrt0qVLL1++TE5ONjIygksn0ON8ZWXl4OBgb2+vn58fk8kcGRkRi8VsNpvP50skkp6enqCgoL6+PhXlzDTq9GdaWpqJicnp06cHBgbq6+t9fHwsLS17enoIgti2bZuNjQ2ZMicnBwAAWxQcHOzi4kJGRUREMJnMBw8eSKXSpqamZcuWsViszs7OKRVy9epVFouVkZGhTtOmsT7C5XI9PDwUAl1cXJ48eUIQRE1NjZGR0fz588ViMfGv6yMqukjZTCA0HvR3331Xfn3kp59+AgDk5OTIp6HT6WvXriUvk5KSAAC1tbVT6RWCQOsjMwGLxcIwTCQSSaXSgoKCwMDA4OBgc3Pz1NRUKpVaVFREpuTxeGw228rKKjQ0dGhoqLOzs729XSgUenp64jhuY2NTUlJiaWn52nL0iEQiyc3NDQoKCgsLMzMz8/LyOn78+PPnz0+cODHVoigUCvyP7eHhUVBQIBKJptrGjRs3CoXC/fv3T7VqdRgaGnry5An8Hz4pvr6+n376aXt7+2effSYfrk4XTZwJWh90uDVDevyDUKlUeXfxbm5uAICGhoZp1zLT/I50ZGhoiCAINpvd3Nw8PDy8aNEiGE6n021tbeENrQImJiYAgNHRUS6Xa21tHRYWlp6e3t7eDmPVL0f3NDU1icVi0iMsAGDZsmUmJiYK3kinytKlSxkMhoG0EdLb20sQhGrPD5mZmW+99daxY8eqq6vJwCl1ETkTtD7ocE1HYRl1ZGRE3g04bN2zZ8+mXctM8zvSkUePHgEA3N3dh4aGAACpqankMZOOjo7h4WEVeel0+g8//LBy5cqsrCwulxsaGiqRSKZRjs4YGBgAAJiamsoHmpubi0QiDUum0WjQWaSBIJVKAQAKi5cKQC+/GIbt2LGD/D8/vS7S+qDDNSb40A0ZHh6WSqV2dnZkCNQU2FLD5HekIzdu3AAArF+/3srKCgBw5MgR+Qe8135ezNPT8/vvvxcIBImJicXFxYcPH55eObrB3NwcAKDwkxgYGICe7qfN6Oio5oVoF/gbe+1hLV9f39jY2JaWloMHD8KQ6XWR1gfd2dmZxWLJe1xvbW0FACxevJgMGRkZAf9sqWHye9GRnp6eI0eOzJs3b8eOHQ4ODjiO19XVqZ9dIBA8ePAAAGBlZfXFF1/4+Pg8ePBgGuXojEWLFpmamv7yyy9kyN27d0dGRt5++20AAIVCGR0dnUaxVVVVBEEsX75ck0K0i7W1NYZh6pwQOXjwoLu7e21tLbxU3UXK0PqgUyiUDRs23Lp1a3x8HIaUl5djGCa//QRbB33UGyazU0cIghCLxdAhe19fX3Fx8YoVK4yNjUtLS9lsNo7j4eHh586dKygoEAqFMpmsu7v76dOnKgoUCASRkZEPHz4cGRmpra3t6OhYvnz5NMrRGTiOx8XFXb58+cyZM0KhsKGhYdeuXXZ2dhEREQAAV1fX/v7+0tLS0dHRvr4++X+GFhYWAoGgvb1dJBJBmYDHgsfGxurr62NiYhwdHeEOuvqFlJeXz9y+L4PB4HK58KuuqoFPN+SKpuouUlGIskEPDQ21sbGZxnH7/fv3P3v27PPPPx8aGrp9+3ZOTs727dvfeustMgFsnZeX11RL1h0zvB/0D4BO9p+uXLmyePFiBoNhYmJiZGQE/nmk9Z133snIyHjx4gWZ8tWrV4mJiY6OjhQKxcrKKjg4uKmp6dixY3BBy83Nra2t7cSJE2w2GwDg5OT0X//1Xzwej8PhGBsbz507NyUlZWxsTFk5M91MQr3+HB8fz8nJcXNzo1KpHA4nMDCwubkZRr148cLf3x/HcWdn53379iUkJAAAXF1dOzs779+/7+TkRKfTV65c2dPTExERQaVS7e3tKRQKm83evHlzW1vbVAu5fv06i8XKzMxUp2nT2PeNioqiUqnDw8Pw8vLly3D7xtLScu/evQqJExISyH1fZV2kYiY8evRI2aAHBgYCANLS0iY18vbt2ytWrCBXPWxtbXk83k8//QRj4ekkGo1mZ2eXkJAglUrl827cuNHe3h7+X5wSuvndEQQxq3Tk94PO+jMiIsLCwkIHFZFMQ0daWlooFMrp06dnyCQ1kclkfn5+hYWF2i32+fPnOI4fPnx4Gnl1Nk9m53MNQosY/vumrq6uGRkZGRkZYrFYXzbIZLLS0lKRSBQaGqrdktPT0729vaOiorRbrHZBOoKYDSQlJYWEhISGhurrlbyqqqqSkpLy8nLVJ1mmSm5ubl1d3fXr16lUqhaL1TpIRxBKSU5OLioqGhwcdHZ2vnTpkr7NeQ1ZWVlRUVFffPGFXmpfu3bt3/72N/KFI61QVlb26tWrqqoqDoejxWJnAt35r0G8cRw6dOjQoUP6tmIKrFu3bt26dfq2Qmu8//7777//vr6tUAt0P4JAIDQF6QgCgdAUpCMIBEJTkI4gEAhN0d0665EjR5DzZy0yW/vzzp074J/ewhFvChihk4+1oWmBQOiF2NhYX1/fma5FRzqCQCBmMWh9BIFAaArSEQQCoSlIRxAIhKYgHUEgEJryf+LLyNXphiOUAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YUuXfMJEMVdt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cb1932a9-ee0c-4719-a5cd-1cb52e259e62"
      },
      "source": [
        "model1.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " probabilities (Dense)       (None, 10)                7850      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 7,850\n",
            "Trainable params: 7,850\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dow5veHRA1wX"
      },
      "source": [
        "For training the model we need to specify the loss to minimize, and the procedure to do this.\n",
        "\n",
        "For our case of `multiclass-classification` we can use the loss of `categorical_crossentropy`, which measures how many bits of uncertainty are left to obtain the true labels given our prediction.\n",
        "\n",
        "We use **sparse**`_categorical_crossentropy` because our labels are encoded sparse i.e. as integers 0,1,..,9 and not as binary **one-hot** labels: (1,0,0,0,0,0,0,0,0,0,0), (0,1,0,0,0,0,0,0,0,0,0), ...,  (0,0,0,0,0,0,0,0,0,0,1)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D3Nad9XbA8XW"
      },
      "source": [
        "model1.compile(\n",
        "    loss = 'sparse_categorical_crossentropy', \n",
        "    optimizer = 'SGD',      # optimization procdure\n",
        "    metrics = ['accuracy']  # Monitoring training\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MB4gy14O-LzI"
      },
      "source": [
        "Now we can fit our data to the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SVZUPHhm8LOD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5c17e81d-ee9b-4d37-8223-0080a8817ba7"
      },
      "source": [
        "history = model1.fit(\n",
        "    x = train_images_flat, \n",
        "    y = train_labels,\n",
        "    batch_size = 16,\n",
        "    validation_data = (test_images_flat, test_labels),\n",
        "    epochs = 20  # one epoche is one sweep through the training data\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "3750/3750 [==============================] - 14s 3ms/step - loss: 0.4501 - accuracy: 0.8689 - val_loss: 0.3201 - val_accuracy: 0.9085\n",
            "Epoch 2/20\n",
            "3750/3750 [==============================] - 9s 2ms/step - loss: 0.3301 - accuracy: 0.9046 - val_loss: 0.3034 - val_accuracy: 0.9114\n",
            "Epoch 3/20\n",
            "3750/3750 [==============================] - 11s 3ms/step - loss: 0.3114 - accuracy: 0.9105 - val_loss: 0.3087 - val_accuracy: 0.9125\n",
            "Epoch 4/20\n",
            "3750/3750 [==============================] - 10s 3ms/step - loss: 0.3010 - accuracy: 0.9143 - val_loss: 0.2893 - val_accuracy: 0.9162\n",
            "Epoch 5/20\n",
            "3750/3750 [==============================] - 10s 3ms/step - loss: 0.2946 - accuracy: 0.9156 - val_loss: 0.2873 - val_accuracy: 0.9171\n",
            "Epoch 6/20\n",
            "3750/3750 [==============================] - 10s 3ms/step - loss: 0.2894 - accuracy: 0.9185 - val_loss: 0.2871 - val_accuracy: 0.9193\n",
            "Epoch 7/20\n",
            "3750/3750 [==============================] - 9s 2ms/step - loss: 0.2872 - accuracy: 0.9193 - val_loss: 0.2820 - val_accuracy: 0.9177\n",
            "Epoch 8/20\n",
            "3729/3750 [============================>.] - ETA: 0s - loss: 0.2828 - accuracy: 0.9203"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8N-PpsB1Gm4h"
      },
      "source": [
        "plt.plot(history.history['accuracy'], label='Training')\n",
        "plt.plot(history.history['val_accuracy'], label='Validation')\n",
        "plt.ylim([.8, .95])\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HkyCMsdtgJrW"
      },
      "source": [
        "## Getting Deeper  \n",
        "\n",
        "Additional `hidden layers` increase complexity and thus expressiveness of the network."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8L0RpGCv-c3e"
      },
      "source": [
        "hidden_layer = tf.keras.layers.Dense( \n",
        "    input_shape = (28*28,), \n",
        "    units=200,            # 200 units als intermediate-output\n",
        "    activation = 'relu',  # one of the most popular activation functions \n",
        "    name = 'hidden'\n",
        ") "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q6jmSNym-cxC"
      },
      "source": [
        "# unser outputlayer wie oben\n",
        "out_layer = tf.keras.layers.Dense( \n",
        "    units=10,                      \n",
        "    activation='softmax',  \n",
        "    name= 'out'\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eSx3lZlr-cpQ"
      },
      "source": [
        "model2 = tf.keras.models.Sequential()\n",
        "\n",
        "model2.add(hidden_layer)\n",
        "model2.add(out_layer)\n",
        "\n",
        "model2.compile(\n",
        "    loss = 'sparse_categorical_crossentropy',  # Klassifizierungsfehler\n",
        "    metrics = ['accuracy'],  # Anteil korrekt klassifizierter Samples des Validierungsdatensatzes\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-GOOX3F3Et-t"
      },
      "source": [
        "tf.keras.utils.plot_model(model2, show_shapes=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LuniAJBoMEbs"
      },
      "source": [
        "model2.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2P0A9eM7FLs-"
      },
      "source": [
        "history = model2.fit(\n",
        "    x = train_images_flat, \n",
        "    y = train_labels,\n",
        "    validation_data = (test_images_flat, test_labels),\n",
        "    epochs = 20\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lxYWbfMrozxz"
      },
      "source": [
        "\n",
        "### Experiments Suggestions\n",
        "Observe the train and validation error \n",
        "- in/decrease the number of neurons in the `hidden layer` (getting wider)\n",
        "- with a second `hidden layer 2` (getting deeper) \n"
      ]
    }
  ]
}