Adversarial_attacks_on_MNIST.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Adversarial attacks on MNIST.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMXDDbbFkcZp1Y5RBuuzsQn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/GiuliaLanzillotta/exercises/blob/master/Adversarial_attacks_on_MNIST.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QV1lcyF1W7NC"
      },
      "source": [
        "# Adversarial attacks on MNIST\n",
        "\n",
        "Today we're going to do 2 things: \n",
        "- Traine a naive CNN on MNIST\n",
        "- Attack it with different techniques\n",
        "\n",
        "Let's start!\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JIYOGHD-YOYF"
      },
      "source": [
        "# uncomment if not already installed  \n",
        "#!pip install tensorboardX"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-B2QCeWTXtA4",
        "outputId": "2e4e0021-cb41-4380-aed0-9aaa4aae162f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Imports + constants\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "import numpy as np\n",
        "import time\n",
        "from torchvision import datasets, transforms\n",
        "\n",
        "use_cuda = False\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
        "batch_size = 64\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Reproducibility \n",
        "# Notice that complete reproducibility is not guaranteed anyway \n",
        "# (for example due to the non perfect associativity of floating point addition)\n",
        "# Look here for more : https://pytorch.org/docs/stable/notes/randomness.html\n",
        "np.random.seed(42)\n",
        "torch.manual_seed(42)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7f50a3821600>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0a0LqBJ0XfMA"
      },
      "source": [
        "## MNIST classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zG5HZqtSZHSO"
      },
      "source": [
        "### The architecture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ffWO65hoWxMo"
      },
      "source": [
        "class Net(nn.Module):\n",
        "  \"\"\"2 layers feed-forward classifier for MNIST images\"\"\"\n",
        "  def __init__(self):\n",
        "        super(Net, self).__init__()\n",
        "        self.fc = nn.Linear(28*28, 200)\n",
        "        self.fc2 = nn.Linear(200,10)\n",
        "\n",
        "  def forward(self, x):\n",
        "      x = x.view((-1, 28*28))\n",
        "      x = F.relu(self.fc(x))\n",
        "      x = self.fc2(x)\n",
        "      return x"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GYohHAM5Zu9Y"
      },
      "source": [
        "class ConvNet(nn.Module):\n",
        "  \"\"\"Pretty basic CNN classifier for MNIST images.\"\"\"\n",
        "  def __init__(self):\n",
        "      # We'll use 6 convolutional layers with decreasing convolution window \n",
        "      # and increasing number of channels \n",
        "      # + ReLU after each layer \n",
        "      # + batch normalization\n",
        "      # + dropout and 2 fully connected layers as a classification head \n",
        "      super(ConvNet, self).__init__()\n",
        "      self.conv1 = nn.Conv2d(1, 32, kernel_size=(5, 5))\n",
        "      self.bn1 = nn.BatchNorm2d(32)\n",
        "      self.conv2 = nn.Conv2d(32, 32, kernel_size=(5, 5))\n",
        "      self.bn2 = nn.BatchNorm2d(32)\n",
        "      self.conv3 = nn.Conv2d(32, 64, kernel_size=(3, 3))\n",
        "      self.bn3 = nn.BatchNorm2d(64)\n",
        "      self.conv4 = nn.Conv2d(64, 64, kernel_size=(3, 3))\n",
        "      self.bn4 = nn.BatchNorm2d(64)\n",
        "      self.conv5 = nn.Conv2d(64, 128, kernel_size=(3, 3))\n",
        "      self.bn5 = nn.BatchNorm2d(128)\n",
        "      self.conv6 = nn.Conv2d(128, 128, kernel_size=(1, 1))\n",
        "      self.bn6 = nn.BatchNorm2d(128)\n",
        "      self.conv2_drop = nn.Dropout2d(p=0.2)\n",
        "      self.fc1 = nn.Linear(128, 100)\n",
        "      self.fc2 = nn.Linear(100, 10)\n",
        "\n",
        "  def forward(self, x):\n",
        "      ##  CONVOLUTIONAL LAYERS \n",
        "      x = F.relu(self.conv1(x))\n",
        "      x = self.bn1(x)\n",
        "      x = F.relu(self.conv2(x))\n",
        "      x = self.conv2_drop(F.max_pool2d(self.bn2(x), 2))\n",
        "      x = F.relu(self.conv3(x))\n",
        "      x = self.bn3(x)\n",
        "      x = F.relu(self.conv4(x))\n",
        "      x = self.bn4(x)\n",
        "      x = F.max_pool2d(x, 2)\n",
        "      x = self.conv2_drop(x)\n",
        "      x = F.relu(self.conv5(x))\n",
        "      x = self.bn5(x)\n",
        "      x = F.relu(self.conv6(x))\n",
        "      x = self.bn6(x)\n",
        "      ## CLASSIFICATION HEAD \n",
        "      size = x.size()[1] * x.size()[2] * x.size()[3]\n",
        "      # flattening \n",
        "      x = x.view(-1, size)\n",
        "      x = F.relu(self.fc1(x))\n",
        "      x = self.fc2(x)\n",
        "      return x\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_nEnzUryZJyu"
      },
      "source": [
        "### The data "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a7RBlTLCZMQI"
      },
      "source": [
        "# Here do 2 things: \n",
        "# 1. Download the MNIST dataset (already divided into train and test)\n",
        "# 2. normalize the input s.t. we have a certain mean and sd (note that the provided\n",
        "# mean and sd are the empirical ones from the data)\n",
        "train_dataset = datasets.MNIST('mnist_data/', train=True, download=True, \n",
        "                               transform=transforms.Compose([transforms.ToTensor(), \n",
        "                                                             transforms.Normalize((0.1307,), (0.3081,))]))\n",
        "test_dataset = datasets.MNIST('mnist_data/', train=False, download=True, \n",
        "                              transform=transforms.Compose([transforms.ToTensor(), \n",
        "                                                            transforms.Normalize((0.1307,), (0.3081,))]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1o1ci2cDcpIc"
      },
      "source": [
        "# Use a DataLoader to avoid iterating through the data yourself\n",
        "# Notice the batch_size=64 that we defined above \n",
        "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
        "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pvrpr-PKZMhM"
      },
      "source": [
        "### Training "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8l4njkbmc2t4"
      },
      "source": [
        "# (This has any effect only if the model is not already there)\n",
        "ff_model = Net().to(device)\n",
        "conv_model = ConvNet().to(device)\n",
        "\n",
        "#This has any effect only on certain modules\n",
        "# (e.g. Dropout, BatchNorm) which behave differently \n",
        "# in train and test mode.\n",
        "ff_model.train()\n",
        "conv_model.train()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r55wbv2rZM35"
      },
      "source": [
        "learning_rate = 0.0001\n",
        "num_epochs = 5\n",
        "# Optimizers:\n",
        "\"\"\" SGD vs Adam \"\"\"\n",
        "#opt = optim.SGD(params=model.parameters(), lr=learning_rate)\n",
        "opt = optim.Adam(params=conv_model.parameters(), lr=learning_rate)\n",
        "# Loss: \n",
        "ce_loss = torch.nn.CrossEntropyLoss()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hgwHF22UdOjs",
        "outputId": "09a8725c-eb31-4058-9bd2-cc7d222d8b86",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "tot_steps = 0\n",
        "for epoch in range(1,num_epochs+1):\n",
        "  print(\"-------------- Epoch \"+str(epoch)+\" -------------\")\n",
        "  t1 = time.time()\n",
        "  for batch_idx, (x_batch, y_batch) in enumerate(train_loader):\n",
        "    x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n",
        "    tot_steps += 1\n",
        "    opt.zero_grad()\n",
        "    out = conv_model(x_batch)\n",
        "    batch_loss = ce_loss(out, y_batch)\n",
        "\n",
        "    # show accuracy every 100 steps\n",
        "    if batch_idx % 100 == 0:\n",
        "      pred = torch.max(out, dim=1)[1] # predictions\n",
        "      acc = pred.eq(y_batch).sum().item() / float(batch_size) # accuracy\n",
        "      print(\"Batch \"+str(batch_idx)+\": \"+ str(acc))\n",
        "\n",
        "    batch_loss.backward()\n",
        "    opt.step() \n",
        "  t2 = time.time()\n",
        "  print(\"Time = %.2lf seconds\"%(t2-t1))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-------------- Epoch 1 -------------\n",
            "Batch 0: 0.078125\n",
            "Batch 100: 0.78125\n",
            "Batch 200: 0.90625\n",
            "Batch 300: 0.96875\n",
            "Batch 400: 0.96875\n",
            "Batch 500: 0.9375\n",
            "Batch 600: 0.96875\n",
            "Batch 700: 0.953125\n",
            "Batch 800: 1.0\n",
            "Batch 900: 0.984375\n",
            "Time = 163.63 seconds\n",
            "-------------- Epoch 2 -------------\n",
            "Batch 0: 1.0\n",
            "Batch 100: 0.96875\n",
            "Batch 200: 0.9375\n",
            "Batch 300: 0.984375\n",
            "Batch 400: 1.0\n",
            "Batch 500: 0.984375\n",
            "Batch 600: 0.96875\n",
            "Batch 700: 1.0\n",
            "Batch 800: 0.96875\n",
            "Batch 900: 1.0\n",
            "Time = 162.66 seconds\n",
            "-------------- Epoch 3 -------------\n",
            "Batch 0: 0.984375\n",
            "Batch 100: 1.0\n",
            "Batch 200: 0.953125\n",
            "Batch 300: 1.0\n",
            "Batch 400: 0.96875\n",
            "Batch 500: 0.953125\n",
            "Batch 600: 0.9375\n",
            "Batch 700: 1.0\n",
            "Batch 800: 0.984375\n",
            "Batch 900: 1.0\n",
            "Time = 162.59 seconds\n",
            "-------------- Epoch 4 -------------\n",
            "Batch 0: 0.984375\n",
            "Batch 100: 0.96875\n",
            "Batch 200: 0.984375\n",
            "Batch 300: 1.0\n",
            "Batch 400: 0.984375\n",
            "Batch 500: 0.953125\n",
            "Batch 600: 0.96875\n",
            "Batch 700: 0.984375\n",
            "Batch 800: 0.984375\n",
            "Batch 900: 1.0\n",
            "Time = 162.23 seconds\n",
            "-------------- Epoch 5 -------------\n",
            "Batch 0: 0.984375\n",
            "Batch 100: 0.984375\n",
            "Batch 200: 0.984375\n",
            "Batch 300: 1.0\n",
            "Batch 400: 0.9375\n",
            "Batch 500: 0.984375\n",
            "Batch 600: 1.0\n",
            "Batch 700: 0.984375\n",
            "Batch 800: 0.96875\n",
            "Batch 900: 0.984375\n",
            "Time = 162.38 seconds\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AQ1kXDiFeP3U",
        "outputId": "0de582a9-e650-4497-9485-87a4c728931c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Evaluate on the test set \n",
        "tot_test, tot_acc = 0.0, 0.0\n",
        "for batch_idx, (x_batch, y_batch) in enumerate(test_loader):\n",
        "    x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n",
        "    out = conv_model(x_batch)\n",
        "    pred = torch.max(out, dim=1)[1]\n",
        "    acc = pred.eq(y_batch).sum().item()\n",
        "    tot_acc += acc\n",
        "    tot_test += x_batch.size()[0]\n",
        "acc = tot_acc/tot_test\n",
        "print('Accuracy %.5lf'%(acc))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy 0.98860\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3oFGmjJkQkpU"
      },
      "source": [
        "Here I'll save the result of some trials:\n",
        "\n",
        "    1) Net + Adam x 5 epochs = 0.9575 test accuracy / time x epoch around 11 s\n",
        "    2) ConvNet + Adam x 5 epochs = 0.98870 test accuracy/ time x epoch around 170 s\n",
        "\n",
        "Notice that this result are obtained by training on CPU"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gy7kacRpXhkA"
      },
      "source": [
        "## Attacks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gSN-q5MeOVy5"
      },
      "source": [
        "# decide which model to try your attacks on\n",
        "model = conv_model"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Ew0qQPjXr7y"
      },
      "source": [
        "# We first load the test dataset again. \n",
        "# Notice that this time we don't want to normalize the input right away \n",
        "# (as we want to be able to search for adversarial examples in the original \n",
        "# image domain)\n",
        "test_dataset = datasets.MNIST('mnist_data/', train=False, download=True, \n",
        "                              transform=transforms.Compose([transforms.ToTensor()]))\n",
        "# In order to be able for our trained model to function with this un-normalized\n",
        "# input we need to insert a normalization layer first \n",
        "class Normalize(nn.Module):\n",
        "    def forward(self, x):\n",
        "        return (x - 0.1307)/0.3081\n",
        "model = nn.Sequential(Normalize(), model)\n",
        "# and here we also create a version of the model that outputs the class probabilities\n",
        "model_to_prob = nn.Sequential(model, nn.Softmax())\n",
        "# we put the neural net into evaluation mode (this disables features like dropout)\n",
        "model.eval()\n",
        "model_to_prob.eval()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HGY0oZO_SqxR"
      },
      "source": [
        "#### The attacks \n",
        "We're now going to implement 4 different attacks (actually 2, each one of them in its targeted and untargeted version). \n",
        "\n",
        "1) **FGSM** : fast gradient sign method. One shot algorithm, meaning that the output is obtained in only one step. \n",
        "\n",
        "2) **PGD** : projected gradient descent (method) - an iterative evolution of the previous one\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CddvlDufgEPG"
      },
      "source": [
        "def fgsm(model, x, lbl, eps, targeted=True):\n",
        "    \"\"\" Implementing both targeted and untargeted FGSM algorithm.\"\"\"\n",
        "\n",
        "    # detatch the input from the graph of previous computation \n",
        "    input_ = x.clone().detach_()\n",
        "    # tell pytorch to track the gradients wrt input x\n",
        "    input_.requires_grad_()\n",
        "    # loss here \n",
        "    loss = ce_loss(model(input_), torch.tensor([lbl], dtype=torch.long))\n",
        "    # gradient of the loss \n",
        "    loss.backward() \n",
        "    # now comes the difference in logic between targeted and untargeted attacks\n",
        "    \n",
        "    # The idea here is to minimise the model's loss \n",
        "    # with respect to the target label \n",
        "    if targeted: x_adv = input_ - eps*torch.sign(input_.grad)\n",
        "    # The idea here is to maximise the model's loss wrt \n",
        "    # the original label ('label')\n",
        "    else: x_adv = input_ + eps*torch.sign(input_.grad)\n",
        "\n",
        "    return x_adv\n",
        "\n",
        "\n",
        "def pgd(model, x, lbl, k, eps, eps_step, targeted=True, clipping=True):\n",
        "    # The idea here is to search for adversarial example \n",
        "    # inside a ball around the original example x \n",
        "    # As projection is easier in this case, we use an L-infinity \n",
        "    # ball of radius eps \n",
        "    # if clipping is set to True we restrict the outputs to \n",
        "    # the [0,1]^n box if clipping is set to True\n",
        "\n",
        "    # random initialization \n",
        "    eta = torch.rand_like(x)*2*eps - eps\n",
        "    x_adv = x + eta\n",
        "    # iteration \n",
        "    for steps in range(1,k+1):\n",
        "      # computing fgsm step \n",
        "      next_x = fgsm(model, x_adv, lbl, eps_step, targeted=targeted)\n",
        "\n",
        "      # projecting the step into the ball \n",
        "      delta = next_x - x\n",
        "      delta = torch.clamp(delta, min=-1*eps, max=eps)  \n",
        "\n",
        "      # taking the step    \n",
        "      x_adv = x + delta\n",
        "\n",
        "      if clipping: \n",
        "        # projecting the output to the [0,1] box\n",
        "        x_adv = torch.clamp(x_adv, min=0, max=1)\n",
        "\n",
        "      # check if we have an adversarial example \n",
        "      out = model(x_adv)\n",
        "      pred = torch.max(out, dim=1)\n",
        "      if pred==lbl and targeted: return x_adv\n",
        "      if pred!=lbl and not targeted: return x_adv\n",
        "\n",
        "    return x_adv\n"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5PhkBBzqgBal"
      },
      "source": [
        "# define a show function that displays the original image together with the \n",
        "# adversarial example and the model predictions\n",
        "def show(original, adv, model_to_prob):\n",
        "    p0 = model_to_prob(original).detach().numpy()\n",
        "    p1 = model_to_prob(adv).detach().numpy()\n",
        "    f, axarr = plt.subplots(1,2)\n",
        "    axarr[0].imshow(original.detach().numpy().reshape(28, 28), cmap='gray')\n",
        "    axarr[0].set_title(\"Original, class: \" + str(p0.argmax()))\n",
        "    axarr[1].imshow(adv.detach().numpy().reshape(28, 28), cmap='gray')\n",
        "    axarr[1].set_title(\"Adversarial, class: \" + str(p1.argmax()))"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JlXY6rRpUiiJ"
      },
      "source": [
        "Trying out attacks! "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "91afM6GNUnB2",
        "outputId": "83786bd6-9dce-48dd-b170-ef5dc22b98eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        }
      },
      "source": [
        "original = torch.unsqueeze(test_dataset[1][0], dim=0)\n",
        "f, axarr = plt.subplots(1,1)\n",
        "axarr.imshow(original[0][0], cmap='gray')"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f504cd21550>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANYElEQVR4nO3df4hd9ZnH8c9n3QTEFk0iOwxG1hr1j7iolVEWVxaX2uiKJgakJshiqTD9o0LF+CNkhQiLKLvb3T8DUxoatWvTkJjGumzqhvpjwQRHiTHRtBpJbMIkQzZgE0Rqkmf/mDPLVOeeOznn3ntu8rxfMNx7z3PvOQ9XPzm/7jlfR4QAnPv+rOkGAPQGYQeSIOxAEoQdSIKwA0n8eS8XZptD/0CXRYSnm15rzW77dtu/tf2R7ZV15gWgu1z1PLvt8yT9TtK3JR2U9Jak5RHxfslnWLMDXdaNNfuNkj6KiI8j4o+Sfi5pSY35AeiiOmG/RNLvp7w+WEz7E7aHbY/aHq2xLAA1df0AXUSMSBqR2IwHmlRnzX5I0qVTXs8vpgHoQ3XC/pakK21/w/ZsScskbelMWwA6rfJmfESctP2gpK2SzpO0NiL2dKwzAB1V+dRbpYWxzw50XVd+VAPg7EHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASPb2VNKp55JFHSuvnn39+y9o111xT+tl77rmnUk+T1qxZU1p/8803W9aee+65WsvGmWHNDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJcHfZPrB+/frSet1z4U3at29fy9qtt95a+tlPPvmk0+2kwN1lgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJrmfvgSbPo+/du7e0vnXr1tL65ZdfXlq/6667SusLFixoWbvvvvtKP/v000+X1nFmaoXd9n5JxyWdknQyIoY60RSAzuvEmv3vIuJoB+YDoIvYZweSqBv2kPRr22/bHp7uDbaHbY/aHq25LAA11N2MvzkiDtn+C0mv2N4bEa9PfUNEjEgakbgQBmhSrTV7RBwqHsclvSjpxk40BaDzKofd9gW2vz75XNIiSbs71RiAzqqzGT8g6UXbk/P5j4j4r450dZYZGio/47h06dJa89+zZ09pffHixS1rR4+Wnyg5ceJEaX327Nml9e3bt5fWr7322pa1efPmlX4WnVU57BHxsaTW/yUB9BVOvQFJEHYgCcIOJEHYgSQIO5AEl7h2wODgYGm9OD3ZUrtTa7fddltpfWxsrLRex4oVK0rrCxcurDzvl19+ufJnceZYswNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEpxn74CXXnqptH7FFVeU1o8fP15aP3bs2Bn31CnLli0rrc+aNatHnaAu1uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATn2XvgwIEDTbfQ0qOPPlpav+qqq2rNf8eOHZVq6DzW7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCOidwuze7cwSJLuvPPO0vqGDRtK6+2GbB4fHy+tl10P/9prr5V+FtVExLQDFbRds9tea3vc9u4p0+bafsX2h8XjnE42C6DzZrIZ/1NJt39p2kpJ2yLiSknbitcA+ljbsEfE65K+fF+kJZLWFc/XSbq7w30B6LCqv40fiIjJAcYOSxpo9Ubbw5KGKy4HQIfUvhAmIqLswFtEjEgakThABzSp6qm3I7YHJal4LD8kC6BxVcO+RdL9xfP7Jf2yM+0A6Ja2m/G2X5B0i6SLbR+UtFrSM5J+YfsBSQckfaebTaK6oaGh0nq78+jtrF+/vrTOufT+0TbsEbG8RelbHe4FQBfxc1kgCcIOJEHYgSQIO5AEYQeS4FbS54DNmze3rC1atKjWvJ999tnS+hNPPFFr/ugd1uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kAS3kj4LDA4OltbffffdlrV58+aVfvbo0aOl9Ztuuqm0vm/fvtI6eq/yraQBnBsIO5AEYQeSIOxAEoQdSIKwA0kQdiAJrmc/C2zcuLG03u5cepnnn3++tM559HMHa3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSILz7H1g8eLFpfXrr7++8rxfffXV0vrq1asrzxtnl7ZrdttrbY/b3j1l2pO2D9neWfzd0d02AdQ1k834n0q6fZrp/x4R1xV//9nZtgB0WtuwR8Trko71oBcAXVTnAN2DtncVm/lzWr3J9rDtUdujNZYFoKaqYV8jaYGk6ySNSfpRqzdGxEhEDEXEUMVlAeiASmGPiCMRcSoiTkv6saQbO9sWgE6rFHbbU+9tvFTS7lbvBdAf2p5nt/2CpFskXWz7oKTVkm6xfZ2kkLRf0ve72ONZr9315qtWrSqtz5o1q/Kyd+7cWVo/ceJE5Xnj7NI27BGxfJrJP+lCLwC6iJ/LAkkQdiAJwg4kQdiBJAg7kASXuPbAihUrSus33HBDrflv3ry5ZY1LWDGJNTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJOGI6N3C7N4trI98/vnnpfU6l7BK0vz581vWxsbGas0bZ5+I8HTTWbMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJcz34OmDt3bsvaF1980cNOvurTTz9tWWvXW7vfH1x44YWVepKkiy66qLT+8MMPV573TJw6dapl7fHHHy/97GeffVZpmazZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJzrOfA3bt2tV0Cy1t2LChZa3dtfYDAwOl9XvvvbdST/3u8OHDpfWnnnqq0nzbrtltX2r7N7bft73H9g+L6XNtv2L7w+JxTqUOAPTETDbjT0paERELJf21pB/YXihppaRtEXGlpG3FawB9qm3YI2IsIt4pnh+X9IGkSyQtkbSueNs6SXd3q0kA9Z3RPrvtyyR9U9IOSQMRMbnTdVjStDtYtoclDVdvEUAnzPhovO2vSdoo6aGI+MPUWkzctXLam0lGxEhEDEXEUK1OAdQyo7DbnqWJoP8sIjYVk4/YHizqg5LGu9MigE5oeytp29bEPvmxiHhoyvR/kfS/EfGM7ZWS5kbEY23mlfJW0ps2bSqtL1mypEed5HLy5MmWtdOnT9ea95YtW0rro6Ojlef9xhtvlNa3b99eWm91K+mZ7LP/jaR/kPSe7Z3FtFWSnpH0C9sPSDog6TszmBeAhrQNe0T8j6Rp/6WQ9K3OtgOgW/i5LJAEYQeSIOxAEoQdSIKwA0kwZHMfeOyx0p8n1B7SuczVV19dWu/mZaRr164tre/fv7/W/Ddu3Niytnfv3lrz7mcM2QwkR9iBJAg7kARhB5Ig7EAShB1IgrADSXCeHTjHcJ4dSI6wA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkmgbdtuX2v6N7fdt77H9w2L6k7YP2d5Z/N3R/XYBVNX25hW2ByUNRsQ7tr8u6W1Jd2tiPPYTEfGvM14YN68Auq7VzStmMj77mKSx4vlx2x9IuqSz7QHotjPaZ7d9maRvStpRTHrQ9i7ba23PafGZYdujtkdrdQqglhnfg8721yS9JumpiNhke0DSUUkh6Z80san/vTbzYDMe6LJWm/EzCrvtWZJ+JWlrRPzbNPXLJP0qIv6qzXwIO9BllW84aduSfiLpg6lBLw7cTVoqaXfdJgF0z0yOxt8s6Q1J70k6XUxeJWm5pOs0sRm/X9L3i4N5ZfNizQ50Wa3N+E4h7ED3cd94IDnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEm1vONlhRyUdmPL64mJaP+rX3vq1L4nequpkb3/ZqtDT69m/snB7NCKGGmugRL/21q99SfRWVa96YzMeSIKwA0k0HfaRhpdfpl9769e+JHqrqie9NbrPDqB3ml6zA+gRwg4k0UjYbd9u+7e2P7K9sokeWrG93/Z7xTDUjY5PV4yhN25795Rpc22/YvvD4nHaMfYa6q0vhvEuGWa80e+u6eHPe77Pbvs8Sb+T9G1JByW9JWl5RLzf00ZasL1f0lBENP4DDNt/K+mEpGcnh9ay/c+SjkXEM8U/lHMi4vE+6e1JneEw3l3qrdUw499Vg99dJ4c/r6KJNfuNkj6KiI8j4o+Sfi5pSQN99L2IeF3SsS9NXiJpXfF8nSb+Z+m5Fr31hYgYi4h3iufHJU0OM97od1fSV080EfZLJP1+yuuD6q/x3kPSr22/bXu46WamMTBlmK3DkgaabGYabYfx7qUvDTPeN99dleHP6+IA3VfdHBHXS/p7ST8oNlf7Ukzsg/XTudM1khZoYgzAMUk/arKZYpjxjZIeiog/TK01+d1N01dPvrcmwn5I0qVTXs8vpvWFiDhUPI5LelETux395MjkCLrF43jD/fy/iDgSEaci4rSkH6vB764YZnyjpJ9FxKZicuPf3XR99ep7ayLsb0m60vY3bM+WtEzSlgb6+ArbFxQHTmT7AkmL1H9DUW+RdH/x/H5Jv2ywlz/RL8N4txpmXA1/d40Pfx4RPf+TdIcmjsjvk/SPTfTQoq/LJb1b/O1pujdJL2his+4LTRzbeEDSPEnbJH0o6b8lze2j3p7TxNDeuzQRrMGGertZE5vouyTtLP7uaPq7K+mrJ98bP5cFkuAAHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4k8X98jzceoKWtgAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F_ZBtAPNUkcE"
      },
      "source": [
        "# try out our attacks\n",
        "original = torch.unsqueeze(test_dataset[0][0], dim=0)\n",
        "adv = fgsm(model, original, lbl=, eps=0.08, targeted=True)\n",
        "show(original, adv, model_to_prob)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vUVVUNLHgF0M",
        "outputId": "fd651d00-453c-450f-9729-3e34fb242c58",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        }
      },
      "source": [
        "# try out our attacks\n",
        "original = torch.unsqueeze(test_dataset[0][0], dim=0)\n",
        "adv = pgd(model, original, lbl=, k=100, eps=0.08, eps_step=0.05, targeted=True)\n",
        "show(original, adv, model_to_prob)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/container.py:117: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
            "  input = module(input)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAADHCAYAAAAAoQhGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAag0lEQVR4nO3dfbRdVX3u8e9DEl5iogGBjBgCqYC9YCsvRkrlLQy0IJYbKJQr12octkS4yJWCXBncoVCHtUiFtl4udOCFy1tEQRQUkfKq2ILUwEUhBDTQAIEEAiFyIEJOcn73j7Wim7Pnytnr7Jdz5s7zGeOM7D333GvNtfdvzay95psiAjMzy88WY10AMzMbHVfgZmaZcgVuZpYpV+BmZplyBW5mlilX4GZmmXIFPoyksyX9n07nbWFbIWm3NrdxrqRrOlEeG98kXSHpS2NdjpFIWixpbot5fQ7U1NcVuKRPSHpY0lpJKyVdImnapt4TEV+OiL9qZft18vY7STtLenXYX0g6Y6zLlhNJP5L0sqStxrosnRAR746IH411OXphLM6Bvq3Ayw/tK8CZwNuA/YFdgNslbVnxnom9K2F/iYinI2LKxj/gD4Eh4IYxLlo2JM0GDgIC+M9jsH9J6kidsDmeS2NxDvRlBS7prcDfAKdGxK0RMRgRy4DjgdnAX5T5zpX0bUnXSHoF+MTwn2CSPi7pKUkvSfq8pGWSPtDw/mvKx7PL/23nS3pa0ouS/mfDdvaTdJ+kNZJWSLqo6j+SFo7v3ZJul7Ra0vOSzq7Id335y+PXku6R9O6G146U9KikAUnPSvpsmb69pJvLcq6W9JNRntQfB+4pP3drzceBnwJXAPMbX5C0j6QHy+/rW8DWDa8tkfSnDc8nSlolad/y+f6S7i2/05833tIor/j/VtK/AWuBd5a/XJ8s9/Ufkj5a5t1V0l3lufCipIWNv2jLc+Nzkn4BvFaWo/F88TnQYX1ZgQPvpwjw7zQmRsSrwC3ABxuS5wHfBqYBCxvzS9oTuBj4KDCD4kp+5gj7PhD4feAw4AuS9ijTNwB/DWwP/HH5+n+reVxImgrcAdwKvAPYDbizIvsPgd2BHYEHefPxXQZ8KiKmAn8A3FWmnwEsB3YApgNnU1wRIuliSRe3UEZRBO+VdY7N+DjFd7QQOFzSdICykrsRuBrYDrgeOLbhfdcCJzQ8Pxx4MSIelDQT+AHwpfK9nwVukLRDQ/6PAQuAqcAq4GvAh8rYeD/wUJlPwN9RxN0ewCzg3GHHcALwYWBaRKwf9prPgU6LiL77o7jCXlnx2nnA7eXjcyn+h2x8/VzgmvLxF4BrG16bDKwDPpDIO5viS96pIf+/Ax+pKMdpwHcbngewWwvHdgLw/ype+215Eq9NK/fxtvL508CngLcOy/dF4KZWyrKJMh4EvApMGetYyOWP4j/+QWD78vljwF+Xjw8GngPUkP9e4Evl492AAWBy+Xwh8IXy8eeAq4ft61+A+eXjHwFfbHjtLcAaiv8gthmhzEc3xiKwDPjksDzLNp4viff7HGjzr1+vwF8Etq+4DzejfH2jZzaxnXc0vh4Ra4GXRtj3yobHa4EpAJLeVf4sW1nervkyxZVIXbOAJ0bKJGmCpPMkPVHub1n50sZ9HgscCTwl6ceS/rhM/3tgKXBb+TP6rFGUcT5wQxS/eKw184HbImJjbH6D391GeQfwbJQ1Q+mpjQ8iYimwBDhK0mSK++ffKF/eBfjz8nbAGklrKP6zmNGwrcYYfw34L8BJwApJP5D0nwAkTZf0zfJ2wyvANTTHcOX55HOg8/q1Ar8PeAP4s8ZESVOAD/Hmn1ubmo5xBbBTw/u3Ad4+yjJdQnFVtXtEvJXiZ5lGsZ1ngHe2kO+/Utwe+gDFrZ/ZZboAIuJnETGP4qfljcB1ZfpARJwREe+kqAhOl3RYq4UrP6M/x7dPWlZ+ZscDh5SV20qKWw17SdqLIg5nlj/LN9p52GY23kaZBzxaVupQxMvVETGt4e8tEXFew3vfdA5ExL9ExAcpKvnHgK+XL325zPuHZQz/Bc0xvKnzyedAh/VlBR4Rv6ZoxPxfko6QNElFC/91FPe2rm5xU9+muKp5f3kf8lxGF3BQ3F98BXi1vKI5uSpj2Yi0rOLlm4EZkk6TtJWkqZL+qGJ/b1D8YphMcfJt3P6Wkj4q6W0RMViWa6h87U8l7VZWFr+muG85VOM4jwFeBu6u8Z7N3dEUn/OewN7l3x7ATyjuo94HrAf+exnLfwbsN2wb3wT+hCKuvtGQfg1FDB9eXpFuLWmupJ1IKK+y50l6C0X8vMrvvv+p5fNfl/fWz6x5nD4HOqwvK3CAiDif4n/4r1J8OfdT/M99WES80eI2FgOnUpwcKyiC9wWKoKjrsxRXBAMUVzTf2kTeWcC/VZRpgKIR9iiK2zW/Ag5NZL2K4mf2s8CjFL0bGn0MWFb+tDyJoqEWigafOyiO9T7g4oi4G0DSP0v6500eZfHT8ephP/dt0+YD/zeKbmgrN/4BF1F8L0MUvyY/AaymuMUxvIF+BcX39X4aYisinqG4Cj2booHyGYqKt+rc3wI4neKe+2rgEH5X0f4NsC9FpfaD4WVogc+BDpPPs9aVt2DWUPwE/I8u7uc24DMRsaRb+zAbz3wOtMYV+AgkHUVxz1zABcAfAfv6CtPMxlrf3kLpoHkUPyefo/hp9RFX3mY2HvgK3MwsU74CNzPLVFsVeNlF73FJS0fZ2d1sXHJsWw5GfQtF0gTglxTdeZYDPwNOiIhHN/Ee36+xroqI0fbT/61OxfbEiWM/Id/QULr7cuq878Tt1NQxb9iwoSltwoQJbe9r/frhU61Uf+bdyltH6nOo85mnYrudUu0HLI2IJwEkfZNyFFgb2zQbDzoS29OmbXLq+Z54/fXXk+mDg4NNaW+8MZrhDW+WOuaBgYGmtKlTp7a9rxdffLEpreoz71beOlKfQ7ufeTu3UGby5nkPlpOYqU/SAkmLJC1qY19mveTYtix0/TdeRFwKXAq+hWL9xbFtY62dCvxZiuGuG+1UppnlrnZsT5w4selnduqneJUpU6Yk0199tXkyu+23T0/gV2d/qW1U3dZI3YbZeuutEznTUtutKmvVsbWqartbbdW8Ql2dvFXq3BZJHdukSZOSeYff4lq3bl0yXzu3UH4G7C7p98qJnj4CfK+N7ZmNF45ty8Kor8AjYr2kT1NMDj8BuLyc/Mksa45ty0Vb98Aj4haKJcrM+opj23LgkZhmZplyBW5mlqmeTmblrlbWbZ0YiTka3YrtdntlVGm390SdAUKpnhap3jVV+6ojdVzQ/sChqu3WGYiTOrZWe+OsWbOGwcHBptj2FbiZWaZcgZuZZcoVuJlZplyBm5llauznuzTrA+0Opa/TeFdnCHhVI1tq6H6dBrmqofStDrGvypc6tjqfTSdmOayz3dRn1omG5+GNxFXTAvsK3MwsU67Azcwy5QrczCxTrsDNzDLlCtzMLFPuhWLWARs2bKgcbt2KTvRYabcHR51FGrq1IEO3pg5I6daw+yp1vuPh30XV1AW+Ajczy5QrcDOzTLkCNzPLlCtwM7NMtTUfuKRlwACwAVgfEXNGyO/5wK2rOjUfeN3YnjRpUgwfSl+l3bm4x7N2G1K7pU4DYmqagar5y+vkbVcqtjvRC+XQiGj90zHLh2PbxjXfQjEzy1S7FXgAt0l6QNKCThTIbJxwbNu41+4tlAMj4llJOwK3S3osIu5pzFAGv08Ay02t2N5iC/+Ytd5rK+oi4tny3xeA7wL7JfJcGhFzRmoEMhtP6sa2K3AbC6O+Apf0FmCLiBgoH/8J8MWOlcxsjHQ7tussnFCn90Qdqd4TdYbSVxkPPU5S6qwIn+pFklosoypvL7VzC2U68F1JG7fzjYi4tSOlMhtbjm3Lwqgr8Ih4Etirg2UxGxcc25YL37gzM8uUK3Azs0xtdvOBH3fccU1pJ554YjLvc88915RWNS/vwoULm9JWrlyZzLt06dJNFdEylJoPvFvzSlc55JBDmtIWLEj34K0T2w888EBT2t13353MO2nSpE0V8beq5uKu08DbS90qV5253VN8BW5mlilX4GZmmXIFbmaWKVfgZmaZcgVuZpapthZ0qL2zcbCgw5NPPtmUNnv27K7sq6qlffHixV3ZXzcsX748mX7++ec3pS1atKjbxRlRpxZ0qKvOgg6pHgZVQ7VTvR+qei7ccsstTWndiu0ddtghmf7973+/4/vaY489Ws67ZMmSlvNWxfZXvvKVprSnnnqq5e12YzqBNWvWMDg42BTbvgI3M8uUK3Azs0y5Ajczy5QrcDOzTG12Q+lTw+bf8573JPOmGkSqGlT23XffprS5c+cm8+6///5Nac8880xT2qxZs5Lvr2P9+vVNaatWrUrmnTFjRsvbffrpp5vSxkMjZq6qhmpXNW6mXHDBBU1pe+2VnlTx0UcfbUorp89tkortk08+OZn3qKOOakr7zW9+05S2zTbbJN+fml87NXd5lV122SWZ3uoQf0jH9nnnndfy+6ukpipodw52X4GbmWXKFbiZWaZcgZuZZcoVuJlZpkaswCVdLukFSY80pG0n6XZJvyr/3ba7xTTrPMe25W7EofSSDgZeBa6KiD8o084HVkfEeZLOAraNiM+NuLNxMJS+l7bdNn3u77333k1pqUnz3/e+97VdhlTL9y9/+ctk3lSvm+222y6Z95RTTmlKu+SSS2qWrvPqDKXvdmxX9Z4YHBxsSqta/CE17L7OIgBVvVjaXWyiasGB448/vikt1TupKrZTMbjFFunrzLVr1zalVcX2Y4891pRWFdsXXXRRU9rFF1+czJtStTBGuyvYp2J7xCvwiLgHWD0seR5wZfn4SuDotkpmNgYc25a70d4Dnx4RK8rHK4HpHSqP2VhzbFs22h7IExGxqVsjkhYA6YX5zMYxx7aNd6O9An9e0gyA8t8XqjJGxKURMSci5oxyX2a95Ni2bLQ0H7ik2cDNDQ09fw+81NDQs11E/I8WtrNZNWKOV8cee2wy/brrrmtKe+SRRxI54dBDD21KW716+O3k3qs7H3inYrvOfOBV88Sn1GlsTG236v2pvJ1YeT3VwJpq1Ktq0Es1utYp18EHH5xMv/7665vSqmJ73rx5TWntDnmHdMNvqw3d69atY2hoqH4jpqRrgfuA35e0XNJfAucBH5T0K+AD5XOzrDi2LXcj3gOPiBMqXjqsw2Ux6ynHtuXOIzHNzDLlCtzMLFOuwM3MMrXZrUq/udlxxx2b0h5++OGW8x533HHJvDfccEN7BeuSsVqVPhXbVUPe6/TKSPVSaHdIdpU6Q/Srek+kytaNVdoBXnrppaa01GIMADvttFNT2kknnZTMm4rtquHxKZ34fob3xhl1LxQzMxufXIGbmWXKFbiZWaZcgZuZZWqzW5V+c5Oat3uHHXZI5n355Zeb0h5//PGOl2lz0Yl5oVN5q+b4rjPkvE7DYipv1Xzg3WqwTEnN251qrKxSJ7a70TAJ1d9Zq9+lr8DNzDLlCtzMLFOuwM3MMuUK3MwsUx6J2ScOOOCAZPpdd93VlDZp0qRk3rlz5zal3XPPPW2Vq9fG00jMOjrRMJnaRtV33Yn5rbsh1Tj6rne9K5k31Qi5atWqZN7UPOGphY6r1Bml2olFp4d/b2vXrmXDhg0eiWlm1i9cgZuZZcoVuJlZplyBm5llqpU1MS+X9IKkRxrSzpX0rKSHyr8ju1tMs85zbFvuWhlKfwVwEXDVsPR/iIivdrxENipHHpmuZ1K9EO68885k3vvuu6+jZcrAFYyT2O7EivCpbdTZbi97pgwMDLSc98Mf/nAyPdXjpCq26/Q4SamaOqBO3jrTDLS6vxGvwCPiHmB1y3s2y4Rj23LXzj3wT0v6RfkzdNuOlchs7Dm2LQujrcAvAXYF9gZWABdUZZS0QNIiSYtGuS+zXnJsWzZGVYFHxPMRsSEihoCvA/ttIu+lETEnIuaMtpBmveLYtpyMaj5wSTMiYkX59BjgkU3lt87aZpttmtKOOOKIZN5169Y1pZ1zzjnJvIODg+0VrA90MrarFv5N6dZCxXX218tGzKlTpybTU42uVbGdyvv5z3++vYJ1UarhtupzaNWIFbika4G5wPaSlgPnAHMl7Q0EsAz4VFulMBsDjm3L3YgVeESckEi+rAtlMespx7blziMxzcwy5QrczCxTrsDNzDLlVekzdOaZZzal7bPPPsm8t956a1Pavffe2/Eybe4mTpzItGnT3pRWZ/h1VY+VVG+ROgsG1FFnCPjrr7+ezNtuT5aTTz65Ka1ObC9durSt/UP6u6jqoZWaqqIqb51pDYZ/5mvWrEnm8xW4mVmmXIGbmWXKFbiZWaZcgZuZZcqr0o9jVfMg33jjjU1pr732WjJvahjyT3/60/YKNo6N1ar0EyZMiMmTJ7eUN9UwWbU6ebtDrau0u5p6uyuv77XXXsn333TTTU1pdWK7qrGvqtG1XalG2zpznacaQSEdI6nY9hW4mVmmXIGbmWXKFbiZWaZcgZuZZcoVuJlZpjyUfpx4+9vf3pT2ta99LZl3woQJTWm33HJLMm8/9zgZTyKiaQh1VQ+DOur0AKmjE9todbupY7jjjjuS70/1uqnKm+pxUqe3SZ1h/1XbrfP9dKMnjK/Azcwy5QrczCxTrsDNzDLlCtzMLFMjDqWXNAu4CphOsdDrpRHxT5K2A74FzKZY/PX4iHh5hG15KD3pRshUY+N73/ve5PufeOKJprSqlbtTeftZnaH0ucZ2ndXue7nSPKQb9X74wx82pR144IHJ92/YsKEp7fDDD0/mXbx4ccvl6vXn0Glr1qxhcHBwVEPp1wNnRMSewP7AKZL2BM4C7oyI3YE7y+dmOXFsW9ZGrMAjYkVEPFg+HgCWADOBecCVZbYrgaO7VUizbnBsW+5q9QOXNBvYB7gfmB4RK8qXVlL8DE29ZwGwYPRFNOs+x7blqOVGTElTgBuA0yLilcbXoriRnrwHGBGXRsSciJjTVknNusSxbblqqQKXNIkiwBdGxHfK5OclzShfnwG80J0imnWPY9tyNuItFEkCLgOWRMSFDS99D5gPnFf+2zwTuyXtuuuuTWlVPU5STj/99Ka0za23SSd0MrZTq9JXSfXUqLMqfZ0eFXVWmq+jarvbbrttU1pVD6mUT37yk01p999/f8vv79YUAXVUDZnvRk+YVu6BHwB8DHhY0kNl2tkUwX2dpL8EngKO73jpzLrLsW1ZG7ECj4h/Bar61h7W2eKY9Y5j23LnkZhmZplyBW5mlinPB95Fu+yySzL9tttua+n9Z555ZjL95ptvHnWZbOzVaWhLNWJ2a19VjW+pMkyePDmZd/ny5S3ta/78+cn0q666qimtztQB3Wq0rWP4vPAbdaMR01fgZmaZcgVuZpYpV+BmZplyBW5mlilX4GZmmXIvlC5asCA9Ud3OO+/c0vt//OMfJ9NHWoTDem9oaKipF0cneh30svdEnfJeeOGFyfRU75RVq1Y1pVXFdrvlGg8LN0ydOrVn+/IVuJlZplyBm5llyhW4mVmmXIGbmWXKjZgdklpl+9RTTx2DkthYGBoaahpyXtWglhruXaexss5w8aq8qeHpVcP2jznmmKa0E088MZk31WDZrqpjSOlEo2+d76fd77JdvgI3M8uUK3Azs0y5Ajczy5QrcDOzTI1YgUuaJeluSY9KWizpM2X6uZKelfRQ+Xdk94tr1jmObctdK71Q1gNnRMSDkqYCD0i6vXztHyLiq90rXj4OOuigprQ6E9GnVpVvdzJ/G9GYxHad3iJ11NlGKraqek+keljVkYrtmTNnJvO+9tprTWmdWKRhYGCgKW3SpEktv79qsYtu9TgZvr+hoaFkvlYWNV4BrCgfD0haAqQ/fbOMOLYtd7XugUuaDewD3F8mfVrSLyRdLmnbivcskLRI0qK2SmrWRY5ty1HLFbikKcANwGkR8QpwCbArsDfFVcwFqfdFxKURMSci5nSgvGYd59i2XLVUgUuaRBHgCyPiOwAR8XxEbIiIIeDrwH7dK6ZZdzi2LWcj3gOXJOAyYElEXNiQPqO8hwhwDPBId4rYf37+8583pR122GFNaatXr+5FcTZbnYxtSWy55ZZvSutEw2QvVZV3yZIlTWlVQ+ZTsZ0ait+JaQbq5H3jjTdaSqvaRtVnkzqOqgbPOnOVt9qBoZVeKAcAHwMelvRQmXY2cIKkvYEAlgGfarl0ZuODY9uy1kovlH8FlHjpls4Xx6x3HNuWO4/ENDPLlCtwM7NMuQI3M8uUernCuSQvp25dFRGpe9pdl4rtOr0n6ujEdrfaaqumtKpeGb3Ubrmqpq9I9epI7QvSQ+yreoXUWRijXanY9hW4mVmmXIGbmWXKFbiZWaZcgZuZZarXjZirgKfKp9sDeY01bo2Pa+zsEhE7jMWOG2I7h89ptPr12HI4rmRs97QCf9OOpUX9OIubj2vz1s+fU78eW87H5VsoZmaZcgVuZpapsazALx3DfXeTj2vz1s+fU78eW7bHNWb3wM3MrD2+hWJmlqmeV+CSjpD0uKSlks7q9f47qVzw9gVJjzSkbSfpdkm/Kv9NLog7nkmaJeluSY9KWizpM2V69sfWTf0S247rfI6tpxW4pAnA/wY+BOxJsfLJnr0sQ4ddARwxLO0s4M6I2B24s3yem/XAGRGxJ7A/cEr5PfXDsXVFn8X2FTius9DrK/D9gKUR8WRErAO+CczrcRk6JiLuAYYvXDkPuLJ8fCVwdE8L1QERsSIiHiwfDwBLgJn0wbF1Ud/EtuM6n2PrdQU+E3im4fnyMq2fTG9YEHclMH0sC9MuSbOBfYD76bNj67B+j+2++u77Ja7diNlFUXTxybabj6QpwA3AaRHxSuNruR+bjV7u330/xXWvK/BngVkNz3cq0/rJ85JmAJT/vjDG5RkVSZMognxhRHynTO6LY+uSfo/tvvju+y2ue12B/wzYXdLvSdoS+AjwvR6Xodu+B8wvH88HbhrDsoyKJAGXAUsi4sKGl7I/ti7q99jO/rvvx7ju+UAeSUcC/whMAC6PiL/taQE6SNK1wFyK2cyeB84BbgSuA3ammJ3u+IgY3iA0rkk6EPgJ8DAwVCafTXG/MOtj66Z+iW3HdT7H5pGYZmaZciOmmVmmXIGbmWXKFbiZWaZcgZuZZcoVuJlZplyBm5llyhW4mVmmXIGbmWXq/wP3l5VXJtabZwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6lrK0p32bqx4"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}