Forgetting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code for papers:\n",
    "1) Forgetting Outside the Box: Scrubbing Deep Networks of Information Accessible from Input-Output Observations (https://arxiv.org/abs/2003.02960) <br>\n",
    "2) Eternal Sunshine of the Spotless Net : Selective forgetting in Deep Networks (https://arxiv.org/abs/1911.04933)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/aditya/anaconda3/envs/pytorch/lib/python3.7/site-packages/tqdm/autonotebook.py:17: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  \" (e.g. in jupyter console)\", TqdmExperimentalWarning)\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "import os\n",
    "os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID'\n",
    "os.environ['CUDA_VISIBLE_DEVICES']='0'\n",
    "import variational\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "import os\n",
    "import time\n",
    "import math\n",
    "import pandas as pd\n",
    "from collections import OrderedDict\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "    \n",
    "import copy\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from typing import List\n",
    "import itertools\n",
    "from tqdm.autonotebook import tqdm\n",
    "from models import *\n",
    "import models\n",
    "from logger import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pdb():\n",
    "    import pdb\n",
    "    pdb.set_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parameter_count(model):\n",
    "    count=0\n",
    "    for p in model.parameters():\n",
    "        count+=np.prod(np.array(list(p.shape)))\n",
    "    print(f'Total Number of Parameters: {count}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize_params(model):\n",
    "    param = []\n",
    "    for p in model.parameters():\n",
    "        param.append(p.data.view(-1).cpu().numpy())\n",
    "    return np.concatenate(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_param_shape(model):\n",
    "    for k,p in model.named_parameters():\n",
    "        print(k,p.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pre-training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#%run main.py --dataset lacuna100 --model allcnn --filters 1.0 --lr 0.1 --lossfn ce --num-classes 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint name: small_cifar5_resnet_0_4_forget_None_lr_0_01_bs_128_ls_ce_wd_0_1_seed_1\n",
      "[Logging in small_cifar5_resnet_0_4_forget_None_lr_0_01_bs_128_ls_ce_wd_0_1_seed_1_training]\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Number of Classes: 5\n",
      "[0] train metrics:{\"loss\": 1.7039970436096192, \"error\": 0.7780000026226044}\n",
      "Learning Rate : 0.01\n",
      "[0] test metrics:{\"loss\": 1.4608981723785401, \"error\": 0.6460000026226044}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.48 sec\n",
      "[1] train metrics:{\"loss\": 1.3020475568771361, \"error\": 0.5219999997615814}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.27 sec\n",
      "[2] train metrics:{\"loss\": 0.9928716106414794, \"error\": 0.3160000019073486}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.27 sec\n",
      "[3] train metrics:{\"loss\": 0.8012013492584229, \"error\": 0.24000000524520873}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.36 sec\n",
      "[4] train metrics:{\"loss\": 0.6915865902900695, \"error\": 0.2119999957084656}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.32 sec\n",
      "[5] train metrics:{\"loss\": 0.6329783720970154, \"error\": 0.19800000762939454}\n",
      "Learning Rate : 0.01\n",
      "[5] test metrics:{\"loss\": 0.8008310027122497, \"error\": 0.2580000019073486}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.34 sec\n",
      "[6] train metrics:{\"loss\": 0.5734132161140442, \"error\": 0.1660000033378601}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.36 sec\n",
      "[7] train metrics:{\"loss\": 0.5120970392227173, \"error\": 0.13199999904632567}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.37 sec\n",
      "[8] train metrics:{\"loss\": 0.46339017057418824, \"error\": 0.10000000381469727}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[9] train metrics:{\"loss\": 0.4212678904533386, \"error\": 0.07599999952316285}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.37 sec\n",
      "[10] train metrics:{\"loss\": 0.38448141384124757, \"error\": 0.044000004291534424}\n",
      "Learning Rate : 0.01\n",
      "[10] test metrics:{\"loss\": 0.8366646966934204, \"error\": 0.238}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.61 sec\n",
      "[11] train metrics:{\"loss\": 0.35193701601028443, \"error\": 0.036000004291534424}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.38 sec\n",
      "[12] train metrics:{\"loss\": 0.3325947813987732, \"error\": 0.018000006675720215}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[13] train metrics:{\"loss\": 0.31253176617622375, \"error\": 0.01599999761581421}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[14] train metrics:{\"loss\": 0.2986357855796814, \"error\": 0.012}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.42 sec\n",
      "[15] train metrics:{\"loss\": 0.2899319808483124, \"error\": 0.01}\n",
      "Learning Rate : 0.01\n",
      "[15] test metrics:{\"loss\": 0.912347505569458, \"error\": 0.24000000429153442}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.54 sec\n",
      "[16] train metrics:{\"loss\": 0.2756882109642029, \"error\": 0.003999997615814209}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.36 sec\n",
      "[17] train metrics:{\"loss\": 0.2711794512271881, \"error\": 0.004}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[18] train metrics:{\"loss\": 0.26516263842582705, \"error\": 0.002}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[19] train metrics:{\"loss\": 0.2617307143211365, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.41 sec\n",
      "[20] train metrics:{\"loss\": 0.25729798460006714, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "[20] test metrics:{\"loss\": 0.9364876155853271, \"error\": 0.2459999952316284}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.62 sec\n",
      "[21] train metrics:{\"loss\": 0.25371763491630556, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.36 sec\n",
      "[22] train metrics:{\"loss\": 0.2502923436164856, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.4 sec\n",
      "[23] train metrics:{\"loss\": 0.24791007125377654, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.37 sec\n",
      "[24] train metrics:{\"loss\": 0.2451861720085144, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.41 sec\n",
      "[25] train metrics:{\"loss\": 0.24318906199932097, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "[25] test metrics:{\"loss\": 0.9425990428924561, \"error\": 0.2399999952316284}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.6 sec\n",
      "[26] train metrics:{\"loss\": 0.24067750763893128, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.33 sec\n",
      "[27] train metrics:{\"loss\": 0.23890034687519074, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.39 sec\n",
      "[28] train metrics:{\"loss\": 0.23704768562316894, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[29] train metrics:{\"loss\": 0.2354983172416687, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[30] train metrics:{\"loss\": 0.23443228912353514, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "[30] test metrics:{\"loss\": 0.9481057176589965, \"error\": 0.24800000667572022}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.48 sec\n"
     ]
    }
   ],
   "source": [
    "%run main.py --dataset small_cifar5 --model resnet --filters 0.4 --lr 0.01\\\n",
    "--resume checkpoints/cifar100_resnet_0_4_forget_None_lr_0_1_bs_128_ls_ce_wd_0_0005_seed_1_25.pt --disable-bn\\\n",
    "--weight-decay 0.1 --batch-size 128 --epochs 31 --seed 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint name: small_cifar5_resnet_0_4_forget_0_num_25_lr_0_01_bs_128_ls_ce_wd_0_1_seed_1\n",
      "[Logging in small_cifar5_resnet_0_4_forget_0_num_25_lr_0_01_bs_128_ls_ce_wd_0_1_seed_1_training]\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Replacing indexes [26 86  2 55 75 93 16 73 54 95 53 92 78 13  7 30 22 24 33  8 43 62  3 71\n",
      " 45]\n",
      "Number of Classes: 5\n",
      "[0] train metrics:{\"loss\": 1.6633460474014283, \"error\": 0.7600000004768371}\n",
      "Learning Rate : 0.01\n",
      "[0] test metrics:{\"loss\": 1.4610265054702758, \"error\": 0.6440000002384185}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.33 sec\n",
      "[1] train metrics:{\"loss\": 1.2611491651535034, \"error\": 0.5060000023841857}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.24 sec\n",
      "[2] train metrics:{\"loss\": 0.9654477944374085, \"error\": 0.33000000619888303}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.27 sec\n",
      "[3] train metrics:{\"loss\": 0.7829683752059936, \"error\": 0.23600000047683717}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.36 sec\n",
      "[4] train metrics:{\"loss\": 0.6663924803733826, \"error\": 0.20599999570846558}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[5] train metrics:{\"loss\": 0.608828143119812, \"error\": 0.19600000047683716}\n",
      "Learning Rate : 0.01\n",
      "[5] test metrics:{\"loss\": 0.8208724799156188, \"error\": 0.27800000429153443}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.62 sec\n",
      "[6] train metrics:{\"loss\": 0.5463282823562622, \"error\": 0.1560000033378601}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.37 sec\n",
      "[7] train metrics:{\"loss\": 0.48658841800689695, \"error\": 0.12199999904632568}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.31 sec\n",
      "[8] train metrics:{\"loss\": 0.439767835855484, \"error\": 0.08000000381469727}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.37 sec\n",
      "[9] train metrics:{\"loss\": 0.3977358751296997, \"error\": 0.06199999952316284}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.37 sec\n",
      "[10] train metrics:{\"loss\": 0.36597972917556765, \"error\": 0.040000006675720214}\n",
      "Learning Rate : 0.01\n",
      "[10] test metrics:{\"loss\": 0.8630177636146545, \"error\": 0.2619999952316284}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.54 sec\n",
      "[11] train metrics:{\"loss\": 0.3339305880069733, \"error\": 0.027999997615814207}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.37 sec\n",
      "[12] train metrics:{\"loss\": 0.31664701652526855, \"error\": 0.018000006675720215}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.31 sec\n",
      "[13] train metrics:{\"loss\": 0.2961531929969788, \"error\": 0.011999997615814209}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.32 sec\n",
      "[14] train metrics:{\"loss\": 0.2847169075012207, \"error\": 0.01}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.41 sec\n",
      "[15] train metrics:{\"loss\": 0.27683760213851927, \"error\": 0.008}\n",
      "Learning Rate : 0.01\n",
      "[15] test metrics:{\"loss\": 0.934027232170105, \"error\": 0.24800000429153443}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.51 sec\n",
      "[16] train metrics:{\"loss\": 0.26372310280799866, \"error\": 0.003999997615814209}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.38 sec\n",
      "[17] train metrics:{\"loss\": 0.2609496955871582, \"error\": 0.002}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[18] train metrics:{\"loss\": 0.2545489358901978, \"error\": 0.001999997615814209}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.42 sec\n",
      "[19] train metrics:{\"loss\": 0.2516889967918396, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[20] train metrics:{\"loss\": 0.24724558281898498, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "[20] test metrics:{\"loss\": 0.9537713403701782, \"error\": 0.2519999952316284}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.62 sec\n",
      "[21] train metrics:{\"loss\": 0.24384495186805724, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.38 sec\n",
      "[22] train metrics:{\"loss\": 0.24074391829967498, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.42 sec\n",
      "[23] train metrics:{\"loss\": 0.2382001476287842, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.41 sec\n",
      "[24] train metrics:{\"loss\": 0.2357859002351761, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.43 sec\n",
      "[25] train metrics:{\"loss\": 0.23409945845603944, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "[25] test metrics:{\"loss\": 0.9573998489379882, \"error\": 0.2560000066757202}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.62 sec\n",
      "[26] train metrics:{\"loss\": 0.23147874915599823, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.35 sec\n",
      "[27] train metrics:{\"loss\": 0.22997407710552215, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.4 sec\n",
      "[28] train metrics:{\"loss\": 0.2282076551914215, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.41 sec\n",
      "[29] train metrics:{\"loss\": 0.2269881145954132, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.41 sec\n",
      "[30] train metrics:{\"loss\": 0.22583238649368287, \"error\": 0.0}\n",
      "Learning Rate : 0.01\n",
      "[30] test metrics:{\"loss\": 0.9633373365402221, \"error\": 0.2640000019073486}\n",
      "Learning Rate : 0.01\n",
      "Epoch Time: 0.51 sec\n"
     ]
    }
   ],
   "source": [
    "%run main.py --dataset small_cifar5 --model resnet --filters 0.4 --lr 0.01\\\n",
    "--resume checkpoints/cifar100_resnet_0_4_forget_None_lr_0_1_bs_128_ls_ce_wd_0_0005_seed_1_25.pt --disable-bn\\\n",
    "--weight-decay 0.1 --batch-size 128 --epochs 31\\\n",
    "--forget-class 0 --num-to-forget 25 --seed 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dict={}\n",
    "training_epochs=25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dict['epoch']=training_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Number of Parameters: 1787741\n"
     ]
    }
   ],
   "source": [
    "parameter_count(copy.deepcopy(model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loads checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "model0 = copy.deepcopy(model)\n",
    "\n",
    "arch = args.model \n",
    "filters=args.filters\n",
    "arch_filters = arch +'_'+ str(filters).replace('.','_')\n",
    "augment = False\n",
    "dataset = args.dataset\n",
    "class_to_forget = args.forget_class\n",
    "init_checkpoint = f\"checkpoints/{args.name}_init.pt\"\n",
    "num_classes=args.num_classes\n",
    "num_to_forget = args.num_to_forget\n",
    "num_total = len(train_loader.dataset)\n",
    "num_to_retain = num_total - num_to_forget\n",
    "seed = args.seed\n",
    "unfreeze_start = None\n",
    "\n",
    "learningrate=f\"lr_{str(args.lr).replace('.','_')}\"\n",
    "batch_size=f\"_bs_{str(args.batch_size)}\"\n",
    "lossfn=f\"_ls_{args.lossfn}\"\n",
    "wd=f\"_wd_{str(args.weight_decay).replace('.','_')}\"\n",
    "seed_name=f\"_seed_{args.seed}_\"\n",
    "\n",
    "num_tag = '' if num_to_forget is None else f'_num_{num_to_forget}'\n",
    "unfreeze_tag = '_' if unfreeze_start is None else f'_unfreeze_from_{unfreeze_start}_'\n",
    "augment_tag = '' if not augment else f'augment_'\n",
    "\n",
    "m_name = f'checkpoints/{dataset}_{arch_filters}_forget_None{unfreeze_tag}{augment_tag}{learningrate}{batch_size}{lossfn}{wd}{seed_name}{training_epochs}.pt'\n",
    "m0_name = f'checkpoints/{dataset}_{arch_filters}_forget_{class_to_forget}{num_tag}{unfreeze_tag}{augment_tag}{learningrate}{batch_size}{lossfn}{wd}{seed_name}{training_epochs}.pt'\n",
    "\n",
    "model.load_state_dict(torch.load(m_name))\n",
    "model0.load_state_dict(torch.load(m0_name))\n",
    "\n",
    "model.cuda()\n",
    "model0.cuda()\n",
    "\n",
    "for p in model.parameters():\n",
    "    p.data0 = p.data.clone()\n",
    "for p in model0.parameters():\n",
    "    p.data0 = p.data.clone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dict['args']=args"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distance between w(D) and w(D_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "def distance(model,model0):\n",
    "    distance=0\n",
    "    normalization=0\n",
    "    for (k, p), (k0, p0) in zip(model.named_parameters(), model0.named_parameters()):\n",
    "        space='  ' if 'bias' in k else ''\n",
    "        current_dist=(p.data0-p0.data0).pow(2).sum().item()\n",
    "        current_norm=p.data0.pow(2).sum().item()\n",
    "        distance+=current_dist\n",
    "        normalization+=current_norm\n",
    "    print(f'Distance: {np.sqrt(distance)}')\n",
    "    print(f'Normalized Distance: {1.0*np.sqrt(distance/normalization)}')\n",
    "    return 1.0*np.sqrt(distance/normalization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distance: 0.4293469691461159\n",
      "Normalized Distance: 0.0048218586950619195\n"
     ]
    }
   ],
   "source": [
    "log_dict['dist_Original_Retrain']=distance(model,model0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Distance of w(D) from initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ntk_init(resume,seed=1):\n",
    "    manual_seed(seed)\n",
    "    model_init = models.get_model(arch, num_classes=num_classes, filters_percentage=filters).to(args.device)\n",
    "    model_init.load_state_dict(torch.load(resume))\n",
    "    return model_init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init = ntk_init(init_checkpoint,args.seed)\n",
    "for p in model_init.parameters():\n",
    "    p.data0 = p.data.clone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Distance: 1.9694975157670922\n",
      "Normalized Distance: 0.022144503782409307\n"
     ]
    }
   ],
   "source": [
    "log_dict['dist_Original_Original_init']=distance(model_init,model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Replacing indexes [26 86  2 55 75 93 16 73 54 95 53 92 78 13  7 30 22 24 33  8 43 62  3 71\n",
      " 45]\n"
     ]
    }
   ],
   "source": [
    "train_loader_full, valid_loader_full, test_loader_full = datasets.get_loaders(dataset, batch_size=args.batch_size, seed=seed, augment=False, shuffle=True)\n",
    "marked_loader, _, _ = datasets.get_loaders(dataset, class_to_replace=class_to_forget, num_indexes_to_replace=num_to_forget, only_mark=True, batch_size=1, seed=seed, augment=False, shuffle=True)\n",
    "\n",
    "def replace_loader_dataset(data_loader, dataset, batch_size=args.batch_size, seed=1, shuffle=True):\n",
    "    manual_seed(seed)\n",
    "    loader_args = {'num_workers': 0, 'pin_memory': False}\n",
    "    def _init_fn(worker_id):\n",
    "        np.random.seed(int(seed))\n",
    "    return torch.utils.data.DataLoader(dataset, batch_size=batch_size,num_workers=0,pin_memory=True,shuffle=shuffle)\n",
    "    \n",
    "forget_dataset = copy.deepcopy(marked_loader.dataset)\n",
    "marked = forget_dataset.targets < 0\n",
    "forget_dataset.data = forget_dataset.data[marked]\n",
    "forget_dataset.targets = - forget_dataset.targets[marked] - 1\n",
    "forget_loader = replace_loader_dataset(train_loader_full, forget_dataset, seed=seed, shuffle=True)\n",
    "\n",
    "retain_dataset = copy.deepcopy(marked_loader.dataset)\n",
    "marked = retain_dataset.targets >= 0\n",
    "retain_dataset.data = retain_dataset.data[marked]\n",
    "retain_dataset.targets = retain_dataset.targets[marked]\n",
    "retain_loader = replace_loader_dataset(train_loader_full, retain_dataset, seed=seed, shuffle=True)\n",
    "\n",
    "assert(len(forget_dataset) + len(retain_dataset) == len(train_loader_full.dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NTK based Forgetting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### NTK Update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delta_w_utils(model_init,dataloader,name='complete'):\n",
    "    model_init.eval()\n",
    "    dataloader = torch.utils.data.DataLoader(dataloader.dataset, batch_size=1, shuffle=False)\n",
    "    G_list = []\n",
    "    f0_minus_y = []\n",
    "    for idx, batch in enumerate(dataloader):#(tqdm(dataloader,leave=False)):\n",
    "        batch = [tensor.to(next(model_init.parameters()).device) for tensor in batch]\n",
    "        input, target = batch\n",
    "        if 'mnist' in args.dataset:\n",
    "            input = input.view(input.shape[0],-1)\n",
    "        target = target.cpu().detach().numpy()\n",
    "        output = model_init(input)\n",
    "        G_sample=[]\n",
    "        for cls in range(num_classes):\n",
    "            grads = torch.autograd.grad(output[0,cls],model_init.parameters(),retain_graph=True)\n",
    "            grads = np.concatenate([g.view(-1).cpu().numpy() for g in grads])\n",
    "            G_sample.append(grads)\n",
    "            G_list.append(grads)\n",
    "        if args.lossfn=='mse':\n",
    "            p = output.cpu().detach().numpy().transpose()\n",
    "            #loss_hess = np.eye(len(p))\n",
    "            target = 2*target-1\n",
    "            f0_y_update = p-target\n",
    "        elif args.lossfn=='ce':\n",
    "            p = torch.nn.functional.softmax(output,dim=1).cpu().detach().numpy().transpose()\n",
    "            p[target]-=1\n",
    "            f0_y_update = copy.deepcopy(p)\n",
    "        f0_minus_y.append(f0_y_update)\n",
    "    return np.stack(G_list).transpose(),np.vstack(f0_minus_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Jacobians and Hessians"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_init = ntk_init(init_checkpoint,args.seed)\n",
    "G_r,f0_minus_y_r = delta_w_utils(copy.deepcopy(model),retain_loader,'complete') \n",
    "\n",
    "np.save('NTK_data/G_r.npy',G_r)\n",
    "np.save('NTK_data/f0_minus_y_r.npy',f0_minus_y_r)\n",
    "del G_r, f0_minus_y_r\n",
    "\n",
    "model_init = ntk_init(init_checkpoint,args.seed)\n",
    "G_f,f0_minus_y_f = delta_w_utils(copy.deepcopy(model),forget_loader,'retain') \n",
    "\n",
    "np.save('NTK_data/G_f.npy',G_f)\n",
    "np.save('NTK_data/f0_minus_y_f.npy',f0_minus_y_f)\n",
    "del G_f, f0_minus_y_f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "G_r = np.load('NTK_data/G_r.npy')\n",
    "G_f = np.load('NTK_data/G_f.npy')\n",
    "G = np.concatenate([G_r,G_f],axis=1)\n",
    "\n",
    "np.save('NTK_data/G.npy',G)\n",
    "del G, G_f, G_r\n",
    "\n",
    "f0_minus_y_r = np.load('NTK_data/f0_minus_y_r.npy')\n",
    "f0_minus_y_f = np.load('NTK_data/f0_minus_y_f.npy')\n",
    "f0_minus_y = np.concatenate([f0_minus_y_r,f0_minus_y_f])\n",
    "\n",
    "np.save('NTK_data/f0_minus_y.npy',f0_minus_y)\n",
    "del f0_minus_y, f0_minus_y_r, f0_minus_y_f"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This only requires access to the gradients and the initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### w_lin(D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = np.load('NTK_data/G.npy')\n",
    "theta = G.transpose().dot(G) + num_total*args.weight_decay*np.eye(G.shape[1])\n",
    "del G\n",
    "\n",
    "theta_inv = np.linalg.inv(theta)\n",
    "\n",
    "np.save('NTK_data/theta.npy',theta)\n",
    "del theta\n",
    "\n",
    "G = np.load('NTK_data/G.npy')\n",
    "f0_minus_y = np.load('NTK_data/f0_minus_y.npy')\n",
    "w_complete = -G.dot(theta_inv.dot(f0_minus_y))\n",
    "\n",
    "np.save('NTK_data/theta_inv.npy',theta_inv)\n",
    "np.save('NTK_data/w_complete.npy',w_complete)\n",
    "del G, f0_minus_y, theta_inv, w_complete "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### w_lin(D_r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "G_r = np.load('NTK_data/G_r.npy')\n",
    "theta_r = G_r.transpose().dot(G_r) + num_to_retain*args.weight_decay*np.eye(G_r.shape[1])\n",
    "del G_r\n",
    "\n",
    "theta_r_inv = np.linalg.inv(theta_r)\n",
    "np.save('NTK_data/theta_r.npy',theta_r)\n",
    "del theta_r\n",
    "\n",
    "G_r = np.load('NTK_data/G_r.npy')\n",
    "f0_minus_y_r = np.load('NTK_data/f0_minus_y_r.npy')\n",
    "w_retain = -G_r.dot(theta_r_inv.dot(f0_minus_y_r))\n",
    "\n",
    "np.save('NTK_data/theta_r_inv.npy',theta_r_inv)\n",
    "np.save('NTK_data/w_retain.npy',w_retain)\n",
    "del G_r, f0_minus_y_r, theta_r_inv, w_retain "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scrubbing Direction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#### Scrubbing Direction\n",
    "w_complete = np.load('NTK_data/w_complete.npy')\n",
    "w_retain = np.load('NTK_data/w_retain.npy')\n",
    "delta_w = (w_retain-w_complete).squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta_w_copy = copy.deepcopy(delta_w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Actual Change in Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual Norm-: 0.4293469786643982\n",
      "Predtn Norm-: 0.01818803805860664\n",
      "Actual Scale: 23.606008371047462\n"
     ]
    }
   ],
   "source": [
    "delta_w_actual = vectorize_params(model0)-vectorize_params(model)\n",
    "\n",
    "print(f'Actual Norm-: {np.linalg.norm(delta_w_actual)}')\n",
    "print(f'Predtn Norm-: {np.linalg.norm(delta_w)}')\n",
    "scale_ratio = np.linalg.norm(delta_w_actual)/np.linalg.norm(delta_w)\n",
    "print('Actual Scale: {}'.format(scale_ratio))\n",
    "log_dict['actual_scale_ratio']=scale_ratio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trapezium Trick"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delta w -------: 0.01818803805860664\n",
      "Inner Product--: -0.10559447380659286\n",
      "Angle----------:  0.1057916981335798\n",
      "Pred Act Norm--:  0.4281945332803079\n",
      "Predicted Scale:  23.542645550913875\n"
     ]
    }
   ],
   "source": [
    "m_pred_error = vectorize_params(model)-vectorize_params(model_init)-w_retain.squeeze()\n",
    "print(f\"Delta w -------: {np.linalg.norm(delta_w)}\")\n",
    "\n",
    "inner = np.inner(delta_w/np.linalg.norm(delta_w),m_pred_error/np.linalg.norm(m_pred_error))\n",
    "print(f\"Inner Product--: {inner}\")\n",
    "\n",
    "if inner<0:\n",
    "    angle = np.arccos(inner)-np.pi/2\n",
    "    print(f\"Angle----------:  {angle}\")\n",
    "\n",
    "    predicted_norm=np.linalg.norm(delta_w) + 2*np.sin(angle)*np.linalg.norm(m_pred_error)\n",
    "    print(f\"Pred Act Norm--:  {predicted_norm}\")\n",
    "else:\n",
    "    angle = np.arccos(inner) \n",
    "    print(f\"Angle----------:  {angle}\")\n",
    "\n",
    "    predicted_norm=np.linalg.norm(delta_w) + 2*np.cos(angle)*np.linalg.norm(m_pred_error)\n",
    "    print(f\"Pred Act Norm--:  {predicted_norm}\")\n",
    "\n",
    "predicted_scale=predicted_norm/np.linalg.norm(delta_w)\n",
    "predicted_scale\n",
    "print(f\"Predicted Scale:  {predicted_scale}\")\n",
    "log_dict['predicted_scale_ratio']=predicted_scale"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Normalized Inner Product between Prediction and Actual Scrubbing Update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def NIP(v1,v2):\n",
    "    nip = (np.inner(v1/np.linalg.norm(v1),v2/np.linalg.norm(v2)))\n",
    "    print(nip)\n",
    "    return nip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49541777628094164\n"
     ]
    }
   ],
   "source": [
    "nip=NIP(delta_w_actual,delta_w)\n",
    "log_dict['nip']=nip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reshape delta_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_delta_w_dict(delta_w,model):\n",
    "    # Give normalized delta_w\n",
    "    delta_w_dict = OrderedDict()\n",
    "    params_visited = 0\n",
    "    for k,p in model.named_parameters():\n",
    "        num_params = np.prod(list(p.shape))\n",
    "        update_params = delta_w[params_visited:params_visited+num_params]\n",
    "        delta_w_dict[k] = torch.Tensor(update_params).view_as(p)\n",
    "        params_visited+=num_params\n",
    "    return delta_w_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *\n",
    "def get_metrics(model,dataloader,criterion,samples_correctness=False,use_bn=False,delta_w=None,scrub_act=False):\n",
    "    activations=[]\n",
    "    predictions=[]\n",
    "    if use_bn:\n",
    "        model.train()\n",
    "        dataloader = torch.utils.data.DataLoader(retain_loader.dataset, batch_size=128, shuffle=True)\n",
    "        for i in range(10):\n",
    "            for batch_idx, (data, target) in enumerate(dataloader):\n",
    "                data, target = data.to(args.device), target.to(args.device)            \n",
    "                output = model(data)\n",
    "    dataloader = torch.utils.data.DataLoader(dataloader.dataset, batch_size=1, shuffle=False)\n",
    "    model.eval()\n",
    "    metrics = AverageMeter()\n",
    "    mult = 0.5 if args.lossfn=='mse' else 1\n",
    "    for batch_idx, (data, target) in enumerate(dataloader):\n",
    "        data, target = data.to(args.device), target.to(args.device)            \n",
    "        if args.lossfn=='mse':\n",
    "            target=(2*target-1)\n",
    "            target = target.type(torch.cuda.FloatTensor).unsqueeze(1)\n",
    "        if 'mnist' in args.dataset:\n",
    "            data=data.view(data.shape[0],-1)\n",
    "        output = model(data)\n",
    "        loss = mult*criterion(output, target)\n",
    "        if samples_correctness:\n",
    "            activations.append(torch.nn.functional.softmax(output,dim=1).cpu().detach().numpy().squeeze())\n",
    "            predictions.append(get_error(output,target))\n",
    "        metrics.update(n=data.size(0), loss=loss.item(), error=get_error(output, target))\n",
    "    if samples_correctness:\n",
    "        return metrics.avg,np.stack(activations),np.array(predictions)\n",
    "    else:\n",
    "        return metrics.avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activations_predictions(model,dataloader,name):\n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "    metrics,activations,predictions=get_metrics(model,dataloader,criterion,True)\n",
    "    print(f\"{name} -> Loss:{np.round(metrics['loss'],3)}, Error:{metrics['error']}\")\n",
    "    log_dict[f\"{name}_loss\"]=metrics['loss']\n",
    "    log_dict[f\"{name}_error\"]=metrics['error']\n",
    "    return activations,predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictions_distance(l1,l2,name):\n",
    "    dist = np.sum(np.abs(l1-l2))\n",
    "    print(f\"Predictions Distance {name} -> {dist}\")\n",
    "    log_dict[f\"{name}_predictions\"]=dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "def activations_distance(a1,a2,name):\n",
    "    dist = np.linalg.norm(a1-a2,ord=1,axis=1).mean()\n",
    "    print(f\"Activations Distance {name} -> {dist}\")\n",
    "    log_dict[f\"{name}_activations\"]=dist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Original Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original_Model_D_r -> Loss:0.046, Error:0.0\n",
      "Original_Model_D_f -> Loss:0.072, Error:0.0\n",
      "Original_Model_D_t -> Loss:0.749, Error:0.24\n"
     ]
    }
   ],
   "source": [
    "m_D_r_activations,m_D_r_predictions=activations_predictions(model,retain_loader,'Original_Model_D_r')\n",
    "m_D_f_activations,m_D_f_predictions=activations_predictions(model,forget_loader,'Original_Model_D_f')\n",
    "m_D_t_activations,m_D_t_predictions=activations_predictions(model,test_loader_full,'Original_Model_D_t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrain Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrain_Model_D_r -> Loss:0.047, Error:0.0\n",
      "Retrain_Model_D_f -> Loss:1.195, Error:0.36\n",
      "Retrain_Model_D_t -> Loss:0.771, Error:0.256\n"
     ]
    }
   ],
   "source": [
    "m0_D_r_activations,m0_D_r_predictions=activations_predictions(model0,retain_loader,'Retrain_Model_D_r')\n",
    "m0_D_f_activations,m0_D_f_predictions=activations_predictions(model0,forget_loader,'Retrain_Model_D_f')\n",
    "m0_D_t_activations,m0_D_t_predictions=activations_predictions(model0,test_loader_full,'Retrain_Model_D_t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrub using NTK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "scale=predicted_scale\n",
    "direction = get_delta_w_dict(delta_w,model)\n",
    "\n",
    "model_scrub = copy.deepcopy(model)\n",
    "for k,p in model_scrub.named_parameters():\n",
    "    p.data += (direction[k]*scale).to(args.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NTK_D_r -> Loss:0.046, Error:0.0\n",
      "NTK_D_f -> Loss:0.928, Error:0.28\n",
      "NTK_D_t -> Loss:0.754, Error:0.254\n"
     ]
    }
   ],
   "source": [
    "ntk_D_r_activations,ntk_D_r_predictions=activations_predictions(model_scrub,retain_loader,'NTK_D_r')\n",
    "ntk_D_f_activations,ntk_D_f_predictions=activations_predictions(model_scrub,forget_loader,'NTK_D_f')\n",
    "ntk_D_t_activations,ntk_D_t_predictions=activations_predictions(model_scrub,test_loader_full,'NTK_D_t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions Distance Retrain_Original_D_f -> 9.0\n",
      "Predictions Distance Retrain_NTK_D_f -> 2.0\n"
     ]
    }
   ],
   "source": [
    "predictions_distance(m0_D_f_predictions,m_D_f_predictions,'Retrain_Original_D_f')\n",
    "predictions_distance(m0_D_f_predictions,ntk_D_f_predictions,'Retrain_NTK_D_f')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activations Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Activations Distance Retrain_Original_D_f -> 0.7483457326889038\n",
      "Activations Distance Retrain_Original_D_r -> 0.011976897716522217\n",
      "Activations Distance Retrain_Original_D_t -> 0.08279383927583694\n",
      "Activations Distance Retrain_NTK_D_f -> 0.233624666929245\n",
      "Activations Distance Retrain_NTK_D_r -> 0.011692313477396965\n",
      "Activations Distance Retrain_NTK_D_t -> 0.06863094866275787\n"
     ]
    }
   ],
   "source": [
    "activations_distance(m0_D_f_activations,m_D_f_activations,'Retrain_Original_D_f')\n",
    "activations_distance(m0_D_r_activations,m_D_r_activations,'Retrain_Original_D_r')\n",
    "activations_distance(m0_D_t_activations,m_D_t_activations,'Retrain_Original_D_t')\n",
    "activations_distance(m0_D_f_activations,ntk_D_f_activations,'Retrain_NTK_D_f')\n",
    "activations_distance(m0_D_r_activations,ntk_D_r_activations,'Retrain_NTK_D_r')\n",
    "activations_distance(m0_D_t_activations,ntk_D_t_activations,'Retrain_NTK_D_t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fisher Forgetting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finetune and Fisher Helper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *\n",
    "def get_metrics(model,dataloader,criterion,samples_correctness=False,use_bn=False,delta_w=None,scrub_act=False):\n",
    "    activations=[]\n",
    "    predictions=[]\n",
    "    if use_bn:\n",
    "        model.train()\n",
    "        dataloader = torch.utils.data.DataLoader(retain_loader.dataset, batch_size=128, shuffle=True)\n",
    "        for i in range(10):\n",
    "            for batch_idx, (data, target) in enumerate(dataloader):\n",
    "                data, target = data.to(args.device), target.to(args.device)            \n",
    "                output = model(data)\n",
    "    dataloader = torch.utils.data.DataLoader(dataloader.dataset, batch_size=1, shuffle=False)\n",
    "    model.eval()\n",
    "    metrics = AverageMeter()\n",
    "    mult = 0.5 if args.lossfn=='mse' else 1\n",
    "    for batch_idx, (data, target) in enumerate(dataloader):\n",
    "        data, target = data.to(args.device), target.to(args.device)            \n",
    "        if args.lossfn=='mse':\n",
    "            target=(2*target-1)\n",
    "            target = target.type(torch.cuda.FloatTensor).unsqueeze(1)\n",
    "        if 'mnist' in args.dataset:\n",
    "            data=data.view(data.shape[0],-1)\n",
    "        output = model(data)\n",
    "        if scrub_act:\n",
    "            G = []\n",
    "            for cls in range(num_classes):\n",
    "                grads = torch.autograd.grad(output[0,cls],model.parameters(),retain_graph=True)\n",
    "                grads = torch.cat([g.view(-1) for g in grads])\n",
    "                G.append(grads)\n",
    "            grads = torch.autograd.grad(output_sf[0,cls],model_scrubf.parameters(),retain_graph=False)\n",
    "            G = torch.stack(G).pow(2)\n",
    "            delta_f = torch.matmul(G,delta_w)\n",
    "            output += delta_f.sqrt()*torch.empty_like(delta_f).normal_()\n",
    "\n",
    "        loss = mult*criterion(output, target)\n",
    "        if samples_correctness:\n",
    "            activations.append(torch.nn.functional.softmax(output,dim=1).cpu().detach().numpy().squeeze())\n",
    "            predictions.append(get_error(output,target))\n",
    "        metrics.update(n=data.size(0), loss=loss.item(), error=get_error(output, target))\n",
    "    if samples_correctness:\n",
    "        return metrics.avg,np.stack(activations),np.array(predictions)\n",
    "    else:\n",
    "        return metrics.avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "def l2_penalty(model,model_init,weight_decay):\n",
    "    l2_loss = 0\n",
    "    for (k,p),(k_init,p_init) in zip(model.named_parameters(),model_init.named_parameters()):\n",
    "        if p.requires_grad:\n",
    "            l2_loss += (p-p_init).pow(2).sum()\n",
    "    l2_loss *= (weight_decay/2.)\n",
    "    return l2_loss\n",
    "\n",
    "def run_train_epoch(model: nn.Module, model_init, data_loader: torch.utils.data.DataLoader, \n",
    "                    loss_fn: nn.Module,\n",
    "                    optimizer: torch.optim.SGD, split: str, epoch: int, ignore_index=None,\n",
    "                    negative_gradient=False, negative_multiplier=-1, random_labels=False,\n",
    "                    quiet=False,delta_w=None,scrub_act=False):\n",
    "    model.eval()\n",
    "    metrics = AverageMeter()    \n",
    "    num_labels = data_loader.dataset.targets.max().item() + 1\n",
    "    \n",
    "    with torch.set_grad_enabled(split != 'test'):\n",
    "        for idx, batch in enumerate(tqdm(data_loader, leave=False)):\n",
    "            batch = [tensor.to(next(model.parameters()).device) for tensor in batch]\n",
    "            input, target = batch\n",
    "            output = model(input)\n",
    "            if split=='test' and scrub_act:\n",
    "                G = []\n",
    "                for cls in range(num_classes):\n",
    "                    grads = torch.autograd.grad(output[0,cls],model.parameters(),retain_graph=True)\n",
    "                    grads = torch.cat([g.view(-1) for g in grads])\n",
    "                    G.append(grads)\n",
    "                grads = torch.autograd.grad(output_sf[0,cls],model_scrubf.parameters(),retain_graph=False)\n",
    "                G = torch.stack(G).pow(2)\n",
    "                delta_f = torch.matmul(G,delta_w)\n",
    "                output += delta_f.sqrt()*torch.empty_like(delta_f).normal_()\n",
    "            loss = loss_fn(output, target) + l2_penalty(model,model_init,args.weight_decay)\n",
    "            metrics.update(n=input.size(0), loss=loss_fn(output,target).item(), error=get_error(output, target))\n",
    "            \n",
    "            if split != 'test':\n",
    "                model.zero_grad()\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "    if not quiet:\n",
    "        log_metrics(split, metrics, epoch)\n",
    "    return metrics.avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "def finetune(model: nn.Module, data_loader: torch.utils.data.DataLoader, lr=0.01, epochs=10, quiet=False):\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=lr, weight_decay=0.0)\n",
    "    model_init=copy.deepcopy(model)\n",
    "    for epoch in range(epochs):\n",
    "        run_train_epoch(model, model_init, data_loader, loss_fn, optimizer, split='train', epoch=epoch, ignore_index=None, quiet=quiet)\n",
    "\n",
    "def test(model, data_loader):\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    model_init=copy.deepcopy(model)\n",
    "    return run_train_epoch(model, model_init, data_loader, loss_fn, optimizer=None, split='test', epoch=epoch, ignore_index=None, quiet=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readout_retrain(model, data_loader, test_loader, lr=0.01, epochs=100, threshold=0.01, quiet=True):\n",
    "    torch.manual_seed(seed)\n",
    "    model = copy.deepcopy(model)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=lr, weight_decay=0.0)\n",
    "    sampler = torch.utils.data.RandomSampler(data_loader.dataset, replacement=True, num_samples=500)\n",
    "    data_loader_small = torch.utils.data.DataLoader(data_loader.dataset, batch_size=data_loader.batch_size, sampler=sampler, num_workers=data_loader.num_workers)\n",
    "    metrics = []\n",
    "    model_init=copy.deepcopy(model)\n",
    "    for epoch in range(epochs):\n",
    "        metrics.append(run_train_epoch(model, model_init, test_loader, loss_fn, optimizer, split='test', epoch=epoch, ignore_index=None, quiet=quiet))\n",
    "        if metrics[-1]['loss'] <= threshold:\n",
    "            break\n",
    "        run_train_epoch(model, model_init, data_loader_small, loss_fn, optimizer, split='train', epoch=epoch, ignore_index=None, quiet=quiet)\n",
    "    return epoch, metrics\n",
    "\n",
    "def extract_retrain_time(metrics, threshold=0.1):\n",
    "    losses = np.array([m['loss'] for m in metrics])\n",
    "    return np.argmax(losses < threshold)\n",
    "\n",
    "def all_readouts(model,thresh=0.1,name='method'):\n",
    "    train_loader = torch.utils.data.DataLoader(train_loader_full.dataset, batch_size=args.batch_size, shuffle=True)\n",
    "    retrain_time, _ = readout_retrain(model, train_loader, forget_loader, epochs=100, lr=0.01, threshold=thresh)\n",
    "    test_error = test(model, test_loader_full)['error']\n",
    "    forget_error = test(model, forget_loader)['error']\n",
    "    retain_error = test(model, retain_loader)['error']\n",
    "    print(f\"{name} ->\"\n",
    "          f\"\\tFull test error: {test_error:.2%}\"\n",
    "          f\"\\tForget error: {forget_error:.2%}\\tRetain error: {retain_error:.2%}\"\n",
    "          f\"\\tFine-tune time: {retrain_time+1} steps\")\n",
    "    log_dict[f\"{name}_retrain_time\"]=retrain_time+1\n",
    "    return(dict(test_error=test_error, forget_error=forget_error, retain_error=retain_error, retrain_time=retrain_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fisher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_scrubf = copy.deepcopy(model_scrub)\n",
    "modelf = copy.deepcopy(model)\n",
    "modelf0 = copy.deepcopy(model0)\n",
    "\n",
    "for p in itertools.chain(modelf.parameters(), modelf0.parameters(), model_scrubf.parameters()):\n",
    "    p.data0 = copy.deepcopy(p.data.clone())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hessian(dataset, model):\n",
    "    model.eval()\n",
    "    train_loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False)\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "    for p in model.parameters():\n",
    "        p.grad_acc = 0\n",
    "        p.grad2_acc = 0\n",
    "    \n",
    "    for data, orig_target in tqdm(train_loader):\n",
    "        data, orig_target = data.to(args.device), orig_target.to(args.device)\n",
    "        output = model(data)\n",
    "        prob = F.softmax(output, dim=-1).data\n",
    "\n",
    "        for y in range(output.shape[1]):\n",
    "            target = torch.empty_like(orig_target).fill_(y)\n",
    "            loss = loss_fn(output, target)\n",
    "            model.zero_grad()\n",
    "            loss.backward(retain_graph=True)\n",
    "            for p in model.parameters():\n",
    "                if p.requires_grad:\n",
    "                    p.grad_acc += (orig_target == target).float() * p.grad.data\n",
    "                    p.grad2_acc += prob[:, y] * p.grad.data.pow(2)\n",
    "    for p in model.parameters():\n",
    "        p.grad_acc /= len(train_loader)\n",
    "        p.grad2_acc /= len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "240f31524bf34f4d814b6f56fe97574c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbd3409d534f428ca95d1b658caa68a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3449db9af4b74067a16e9e9258229097",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "hessian(retain_loader.dataset, model_scrubf)\n",
    "hessian(retain_loader.dataset, modelf)\n",
    "hessian(retain_loader.dataset, modelf0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mean_var(p, is_base_dist=False, alpha=3e-6):\n",
    "    var = copy.deepcopy(1./(p.grad2_acc+1e-8))\n",
    "    var = var.clamp(max=1e3)\n",
    "    if p.size(0) == num_classes:\n",
    "        var = var.clamp(max=1e2)\n",
    "    var = alpha * var\n",
    "    \n",
    "    if p.ndim > 1:\n",
    "        var = var.mean(dim=1, keepdim=True).expand_as(p).clone()\n",
    "    if not is_base_dist:\n",
    "        mu = copy.deepcopy(p.data0.clone())\n",
    "    else:\n",
    "        mu = copy.deepcopy(p.data0.clone())\n",
    "    if p.size(0) == num_classes and num_to_forget is None:\n",
    "        mu[class_to_forget] = 0\n",
    "        var[class_to_forget] = 0.0001\n",
    "    if p.size(0) == num_classes:\n",
    "        # Last layer\n",
    "        var *= 10\n",
    "    elif p.ndim == 1:\n",
    "        # BatchNorm\n",
    "        var *= 10\n",
    "#         var*=1\n",
    "    return mu, var\n",
    "\n",
    "def kl_divergence_fisher(mu0, var0, mu1, var1):\n",
    "    return ((mu1 - mu0).pow(2)/var0 + var1/var0 - torch.log(var1/var0) - 1).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fisher Noise in Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1.weight 4.6\n",
      "conv1.bias 0.0\n",
      "bn1.weight 0.5\n",
      "bn1.bias 0.2\n",
      "layer1.0.bn1.weight 0.1\n",
      "layer1.0.bn1.bias 0.0\n",
      "layer1.0.conv1.weight 1.6\n",
      "layer1.0.conv1.bias 0.0\n",
      "layer1.0.bn2.weight 0.1\n",
      "layer1.0.bn2.bias 0.0\n",
      "layer1.0.conv2.weight 1.6\n",
      "layer1.0.conv2.bias 0.0\n",
      "layer1.0.shortcut.0.weight 0.7\n",
      "layer1.1.bn1.weight 0.0\n",
      "layer1.1.bn1.bias 0.0\n",
      "layer1.1.conv1.weight 0.9\n",
      "layer1.1.conv1.bias 0.0\n",
      "layer1.1.bn2.weight 0.0\n",
      "layer1.1.bn2.bias 0.0\n",
      "layer1.1.conv2.weight 0.9\n",
      "layer1.1.conv2.bias 0.0\n",
      "layer2.0.bn1.weight 0.1\n",
      "layer2.0.bn1.bias 0.1\n",
      "layer2.0.conv1.weight 3.5\n",
      "layer2.0.conv1.bias 0.0\n",
      "layer2.0.bn2.weight 0.2\n",
      "layer2.0.bn2.bias 0.2\n",
      "layer2.0.conv2.weight 4.8\n",
      "layer2.0.conv2.bias 0.1\n",
      "layer2.0.shortcut.0.weight 1.5\n",
      "layer2.1.bn1.weight 0.0\n",
      "layer2.1.bn1.bias 0.0\n",
      "layer2.1.conv1.weight 3.5\n",
      "layer2.1.conv1.bias 0.0\n",
      "layer2.1.bn2.weight 0.1\n",
      "layer2.1.bn2.bias 0.1\n",
      "layer2.1.conv2.weight 2.8\n",
      "layer2.1.conv2.bias 0.1\n",
      "layer3.0.bn1.weight 0.3\n",
      "layer3.0.bn1.bias 0.2\n",
      "layer3.0.conv1.weight 9.9\n",
      "layer3.0.conv1.bias 0.0\n",
      "layer3.0.bn2.weight 0.4\n",
      "layer3.0.bn2.bias 0.3\n",
      "layer3.0.conv2.weight 16.0\n",
      "layer3.0.conv2.bias 0.1\n",
      "layer3.0.shortcut.0.weight 1.5\n",
      "layer3.1.bn1.weight 0.1\n",
      "layer3.1.bn1.bias 0.0\n",
      "layer3.1.conv1.weight 8.3\n",
      "layer3.1.conv1.bias 0.0\n",
      "layer3.1.bn2.weight 0.2\n",
      "layer3.1.bn2.bias 0.1\n",
      "layer3.1.conv2.weight 7.0\n",
      "layer3.1.conv2.bias 0.1\n",
      "layer4.0.bn1.weight 1.4\n",
      "layer4.0.bn1.bias 0.6\n",
      "layer4.0.conv1.weight 25.9\n",
      "layer4.0.conv1.bias 0.2\n",
      "layer4.0.bn2.weight 2.0\n",
      "layer4.0.bn2.bias 1.0\n",
      "layer4.0.conv2.weight 30.0\n",
      "layer4.0.conv2.bias 1.0\n",
      "layer4.0.shortcut.0.weight 5.3\n",
      "layer4.1.bn1.weight 2.1\n",
      "layer4.1.bn1.bias 1.0\n",
      "layer4.1.conv1.weight 23.1\n",
      "layer4.1.conv1.bias 0.4\n",
      "layer4.1.bn2.weight 2.9\n",
      "layer4.1.bn2.bias 0.5\n",
      "layer4.1.conv2.weight 4.8\n",
      "layer4.1.conv2.bias 0.0\n",
      "linear.weight 75.9\n",
      "linear.bias 0.1\n",
      "Total: 251.3311948776245\n"
     ]
    }
   ],
   "source": [
    "# Computes the amount of information not forgotten at all layers using the given alpha\n",
    "alpha = 1e-6\n",
    "total_kl = 0\n",
    "torch.manual_seed(seed)\n",
    "for (k, p), (k0, p0) in zip(modelf.named_parameters(), modelf0.named_parameters()):\n",
    "    mu0, var0 = get_mean_var(p, False, alpha=alpha)\n",
    "    mu1, var1 = get_mean_var(p0, True, alpha=alpha)\n",
    "    kl = kl_divergence_fisher(mu0, var0, mu1, var1).item()\n",
    "    total_kl += kl\n",
    "    print(k, f'{kl:.1f}')\n",
    "print(\"Total:\", total_kl)\n",
    "log_dict['fisher_info']=total_kl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "fisher_dir = []\n",
    "alpha = 1e-6\n",
    "torch.manual_seed(seed)\n",
    "for i, p in enumerate(modelf.parameters()):\n",
    "    mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "    p.data = mu + var.sqrt() * torch.empty_like(p.data0).normal_()\n",
    "    fisher_dir.append(var.sqrt().view(-1).cpu().detach().numpy())\n",
    "\n",
    "for i, p in enumerate(modelf0.parameters()):\n",
    "    mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "    p.data = mu + var.sqrt() * torch.empty_like(p.data0).normal_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 0.4283576956548189, 'error': 0.15157894536068564})\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 0.2621564567089081, 'error': 0.12000000476837158})\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 1.2169767618179321, 'error': 0.3359999656677246})\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 1.0217268447875976, 'error': 0.30999999976158144})\n"
     ]
    }
   ],
   "source": [
    "print(test(modelf, retain_loader))\n",
    "print(test(modelf, forget_loader))\n",
    "print(test(modelf, valid_loader_full))\n",
    "print(test(modelf, test_loader_full))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrain Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrain_Model_D_r -> Loss:0.047, Error:0.0\n",
      "Retrain_Model_D_f -> Loss:1.195, Error:0.36\n",
      "Retrain_Model_D_t -> Loss:0.771, Error:0.256\n"
     ]
    }
   ],
   "source": [
    "m0_D_r_activations,m0_D_r_predictions=activations_predictions(model0,retain_loader,'Retrain_Model_D_r')\n",
    "m0_D_f_activations,m0_D_f_predictions=activations_predictions(model0,forget_loader,'Retrain_Model_D_f')\n",
    "m0_D_t_activations,m0_D_t_predictions=activations_predictions(model0,test_loader_full,'Retrain_Model_D_t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fisher_D_r -> Loss:0.428, Error:0.15157894736842106\n",
      "Fisher_D_f -> Loss:0.262, Error:0.12\n",
      "Fisher_D_t -> Loss:1.022, Error:0.31\n"
     ]
    }
   ],
   "source": [
    "fisher_D_r_activations,fisher_D_r_predictions=activations_predictions(modelf,retain_loader,'Fisher_D_r')\n",
    "fisher_D_f_activations,fisher_D_f_predictions=activations_predictions(modelf,forget_loader,'Fisher_D_f')\n",
    "fisher_D_t_activations,fisher_D_t_predictions=activations_predictions(modelf,test_loader_full,'Fisher_D_t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions Distance Retrain_Fisher_D_f -> 6.0\n",
      "Activations Distance Retrain_Fisher_D_f -> 0.5536348819732666\n",
      "Activations Distance Retrain_Fisher_D_r -> 0.35991254448890686\n",
      "Activations Distance Retrain_Fisher_D_t -> 0.4132028818130493\n"
     ]
    }
   ],
   "source": [
    "predictions_distance(m0_D_f_predictions,fisher_D_f_predictions,'Retrain_Fisher_D_f')\n",
    "activations_distance(m0_D_f_activations,fisher_D_f_activations,'Retrain_Fisher_D_f')\n",
    "activations_distance(m0_D_r_activations,fisher_D_r_activations,'Retrain_Fisher_D_r')\n",
    "activations_distance(m0_D_t_activations,fisher_D_t_activations,'Retrain_Fisher_D_t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NTK + Fisher Noise in the Weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in itertools.chain(modelf0.parameters(), model_scrubf.parameters()):\n",
    "    p.data = p.data0.clone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv1.weight 3669.6\n",
      "conv1.bias 7.4\n",
      "bn1.weight 485.7\n",
      "bn1.bias 211.7\n",
      "layer1.0.bn1.weight 31.8\n",
      "layer1.0.bn1.bias 14.2\n",
      "layer1.0.conv1.weight 1211.8\n",
      "layer1.0.conv1.bias 0.2\n",
      "layer1.0.bn2.weight 9.6\n",
      "layer1.0.bn2.bias 5.4\n",
      "layer1.0.conv2.weight 1337.5\n",
      "layer1.0.conv2.bias 1.0\n",
      "layer1.0.shortcut.0.weight 635.2\n",
      "layer1.1.bn1.weight 1.7\n",
      "layer1.1.bn1.bias 5.4\n",
      "layer1.1.conv1.weight 835.7\n",
      "layer1.1.conv1.bias 0.5\n",
      "layer1.1.bn2.weight 5.6\n",
      "layer1.1.bn2.bias 1.5\n",
      "layer1.1.conv2.weight 903.2\n",
      "layer1.1.conv2.bias 0.9\n",
      "layer2.0.bn1.weight 39.6\n",
      "layer2.0.bn1.bias 60.2\n",
      "layer2.0.conv1.weight 3216.6\n",
      "layer2.0.conv1.bias 0.8\n",
      "layer2.0.bn2.weight 55.0\n",
      "layer2.0.bn2.bias 59.9\n",
      "layer2.0.conv2.weight 4686.4\n",
      "layer2.0.conv2.bias 3.4\n",
      "layer2.0.shortcut.0.weight 1206.0\n",
      "layer2.1.bn1.weight 4.9\n",
      "layer2.1.bn1.bias 6.5\n",
      "layer2.1.conv1.weight 3386.0\n",
      "layer2.1.conv1.bias 1.1\n",
      "layer2.1.bn2.weight 21.3\n",
      "layer2.1.bn2.bias 7.3\n",
      "layer2.1.conv2.weight 2668.5\n",
      "layer2.1.conv2.bias 2.5\n",
      "layer3.0.bn1.weight 151.0\n",
      "layer3.0.bn1.bias 68.1\n",
      "layer3.0.conv1.weight 9609.3\n",
      "layer3.0.conv1.bias 1.9\n",
      "layer3.0.bn2.weight 113.7\n",
      "layer3.0.bn2.bias 40.7\n",
      "layer3.0.conv2.weight 16113.8\n",
      "layer3.0.conv2.bias 2.5\n",
      "layer3.0.shortcut.0.weight 1508.7\n",
      "layer3.1.bn1.weight 8.7\n",
      "layer3.1.bn1.bias 5.2\n",
      "layer3.1.conv1.weight 7819.0\n",
      "layer3.1.conv1.bias 1.5\n",
      "layer3.1.bn2.weight 16.8\n",
      "layer3.1.bn2.bias 5.6\n",
      "layer3.1.conv2.weight 7383.2\n",
      "layer3.1.conv2.bias 2.2\n",
      "layer4.0.bn1.weight 629.6\n",
      "layer4.0.bn1.bias 145.3\n",
      "layer4.0.conv1.weight 29363.2\n",
      "layer4.0.conv1.bias 9.2\n",
      "layer4.0.bn2.weight 628.2\n",
      "layer4.0.bn2.bias 74.9\n",
      "layer4.0.conv2.weight 35040.9\n",
      "layer4.0.conv2.bias 38.5\n",
      "layer4.0.shortcut.0.weight 5425.2\n",
      "layer4.1.bn1.weight 371.6\n",
      "layer4.1.bn1.bias 33.5\n",
      "layer4.1.conv1.weight 27250.9\n",
      "layer4.1.conv1.bias 7.4\n",
      "layer4.1.bn2.weight 155.2\n",
      "layer4.1.bn2.bias 9.1\n",
      "layer4.1.conv2.weight 4343.5\n",
      "layer4.1.conv2.bias 1.7\n",
      "linear.weight 49426.9\n",
      "linear.bias 64.5\n",
      "Total: 220672.87370443344\n"
     ]
    }
   ],
   "source": [
    "# Computes the amount of information not forgotten at all layers using the given alpha\n",
    "alpha = 1e-9\n",
    "total_kl = 0\n",
    "torch.manual_seed(seed)\n",
    "for (k, p), (k0, p0) in zip(model_scrubf.named_parameters(), modelf0.named_parameters()):\n",
    "    mu0, var0 = get_mean_var(p, False, alpha=alpha)\n",
    "    mu1, var1 = get_mean_var(p0, True, alpha=alpha)\n",
    "    kl = kl_divergence_fisher(mu0, var0, mu1, var1).item()\n",
    "    total_kl += kl\n",
    "    print(k, f'{kl:.1f}')\n",
    "print(\"Total:\", total_kl)\n",
    "log_dict['ntk_fisher_info']=total_kl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(seed)\n",
    "for i, p in enumerate(model_scrubf.parameters()):\n",
    "    mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "    p.data = mu + var.sqrt() * torch.empty_like(p.data0).normal_()\n",
    "\n",
    "for i, p in enumerate(modelf0.parameters()):\n",
    "    mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "    p.data = mu + var.sqrt() * torch.empty_like(p.data0).normal_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 0.046477304014720416, 'error': 0.0})\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 0.9240099191665649, 'error': 0.2800000309944153})\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 0.823030412197113, 'error': 0.21599996089935303})\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<class 'float'>, {'loss': 0.7542795805931092, 'error': 0.2520000019073486})\n"
     ]
    }
   ],
   "source": [
    "print(test(model_scrubf, retain_loader))\n",
    "print(test(model_scrubf, forget_loader))\n",
    "print(test(model_scrubf, valid_loader_full))\n",
    "print(test(model_scrubf, test_loader_full))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NTK_Fisher_D_r -> Loss:0.046, Error:0.0\n",
      "NTK_Fisher_D_f -> Loss:0.924, Error:0.28\n",
      "NTK_Fisher_D_t -> Loss:0.754, Error:0.252\n"
     ]
    }
   ],
   "source": [
    "ntk_fisher_D_r_activations,ntk_fisher_D_r_predictions=activations_predictions(model_scrubf,retain_loader,'NTK_Fisher_D_r')\n",
    "ntk_fisher_D_f_activations,ntk_fisher_D_f_predictions=activations_predictions(model_scrubf,forget_loader,'NTK_Fisher_D_f')\n",
    "ntk_fisher_D_t_activations,ntk_fisher_D_t_predictions=activations_predictions(model_scrubf,test_loader_full,'NTK_Fisher_D_t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions Distance Retrain_NTK_Fisher_D_f -> 2.0\n",
      "Activations Distance Retrain_NTK_Fisher_D_f -> 0.23201343417167664\n",
      "Activations Distance Retrain_NTK_Fisher_D_r -> 0.012660030275583267\n",
      "Activations Distance Retrain_NTK_Fisher_D_t -> 0.06889393925666809\n"
     ]
    }
   ],
   "source": [
    "predictions_distance(m0_D_f_predictions,ntk_fisher_D_f_predictions,'Retrain_NTK_Fisher_D_f')\n",
    "activations_distance(m0_D_f_activations,ntk_fisher_D_f_activations,'Retrain_NTK_Fisher_D_f')\n",
    "activations_distance(m0_D_r_activations,ntk_fisher_D_r_activations,'Retrain_NTK_Fisher_D_r')\n",
    "activations_distance(m0_D_t_activations,ntk_fisher_D_t_activations,'Retrain_NTK_Fisher_D_t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test-error vs Remaining Information in the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_scrubf = copy.deepcopy(model_scrub)\n",
    "modelf = copy.deepcopy(model)\n",
    "modelf0 = copy.deepcopy(model0)\n",
    "\n",
    "for p in itertools.chain(modelf.parameters(), modelf0.parameters(), model_scrubf.parameters()):\n",
    "    p.data0 = copy.deepcopy(p.data.clone())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00ff55e86aad49ccb1d84032e1e46d42",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "885025189e1d4eb5b77200b2c0ff18c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d86de9a91da4512827e4d2e83bfacdb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "hessian(retain_loader.dataset, model_scrubf)\n",
    "hessian(retain_loader.dataset, modelf)\n",
    "hessian(retain_loader.dataset, modelf0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "alpha_list = [1e-8,1e-7,1e-6,1e-5]\n",
    "test_error_list = []\n",
    "information_list = []\n",
    "\n",
    "runs = 3\n",
    "for s in range(runs):\n",
    "    torch.manual_seed(s)\n",
    "    test_error_list.append([])\n",
    "    information_list.append([])\n",
    "    for alpha in alpha_list:        \n",
    "        for i, p in enumerate(model_scrubf.parameters()):\n",
    "            mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "            p.data = mu + var.sqrt() * torch.empty_like(p.data0).normal_()\n",
    "\n",
    "        for i, p in enumerate(modelf0.parameters()):\n",
    "            mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "            p.data = mu + var.sqrt() * torch.empty_like(p.data0).normal_()\n",
    "        metrics = test(model_scrubf, test_loader_full)\n",
    "\n",
    "        total_kl = 0\n",
    "        for (k, p), (k0, p0) in zip(model_scrubf.named_parameters(), modelf0.named_parameters()):\n",
    "            mu0, var0 = get_mean_var(p, False, alpha=alpha)\n",
    "            mu1, var1 = get_mean_var(p0, True, alpha=alpha)\n",
    "            kl = kl_divergence_fisher(mu0, var0, mu1, var1).item()\n",
    "            total_kl += kl\n",
    "            \n",
    "        test_error_list[s].append(metrics['error'])\n",
    "        information_list[s].append(total_kl)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha_list = [1e-8,1e-7,1e-6,1e-5]\n",
    "alpha_list = np.ndarray.flatten(np.array([alpha_list for i in range(runs)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_error_list = np.ndarray.flatten(np.array(test_error_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "information_list = np.ndarray.flatten(np.array(information_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "info_dict = {}\n",
    "info_dict['alpha'] = alpha_list\n",
    "info_dict['error'] = test_error_list*100\n",
    "info_dict['info'] = information_list\n",
    "df = pd.DataFrame(info_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Information in the Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_scrubf = copy.deepcopy(model_scrub)\n",
    "modelf = copy.deepcopy(model)\n",
    "modelf0 = copy.deepcopy(model0)\n",
    "\n",
    "for p in itertools.chain(modelf.parameters(), modelf0.parameters(), model_scrubf.parameters()):\n",
    "    p.data0 = copy.deepcopy(p.data.clone())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f20cbc7939284ace83ca88886640de1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8f2e85c2e9d4b5e88fb39a8eed1d220",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b3026fcb4964fb6846dce47cc055192",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "hessian(retain_loader.dataset, model_scrubf)\n",
    "hessian(retain_loader.dataset, modelf)\n",
    "hessian(retain_loader.dataset, modelf0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_activations(model_scrubf, modelf0, delta_w_s, delta_w_m0, data_loader, \\\n",
    "                    loss_fn=nn.CrossEntropyLoss(),\\\n",
    "                    optimizer=torch.optim.SGD, \\\n",
    "                    seed=1,quiet=False):\n",
    "\n",
    "    model_scrubf.eval()\n",
    "    modelf0.eval()\n",
    "    \n",
    "    data_loader = torch.utils.data.DataLoader(data_loader.dataset, batch_size=1, shuffle=False)\n",
    "\n",
    "    \n",
    "    metrics = AverageMeter()    \n",
    "    num_classes = data_loader.dataset.targets.max().item() + 1\n",
    "    \n",
    "    for idx, batch in enumerate(tqdm(data_loader, leave=False)):\n",
    "        batch = [tensor.to(next(model_scrubf.parameters()).device) for tensor in batch]\n",
    "        input, target = batch\n",
    "        \n",
    "        output_sf = model_scrubf(input)\n",
    "        G_sf = []\n",
    "\n",
    "        for cls in range(num_classes):\n",
    "            grads = torch.autograd.grad(output_sf[0,cls],model_scrubf.parameters(),retain_graph=True)\n",
    "            grads = torch.cat([g.view(-1) for g in grads])\n",
    "            G_sf.append(grads)\n",
    "\n",
    "        grads = torch.autograd.grad(output_sf[0,cls],model_scrubf.parameters(),retain_graph=False)\n",
    "            \n",
    "        G_sf = torch.stack(G_sf)#.pow(2)\n",
    "        delta_f_sf_update = torch.matmul(G_sf,delta_w_s.sqrt()*torch.empty_like(delta_w_s).normal_())\n",
    "        G_sf = G_sf.pow(2)\n",
    "        delta_f_sf = torch.matmul(G_sf,delta_w_s)\n",
    "\n",
    "        output_m0 = modelf0(input)\n",
    "        G_m0 = []\n",
    "\n",
    "        for cls in range(num_classes):\n",
    "            grads = torch.autograd.grad(output_m0[0,cls],modelf0.parameters(),retain_graph=True)\n",
    "            grad_m0 = torch.cat([g.view(-1) for g in grads])\n",
    "            G_m0.append(grad_m0)\n",
    "\n",
    "        grads = torch.autograd.grad(output_m0[0,cls],modelf0.parameters(),retain_graph=False)\n",
    "            \n",
    "        G_m0 = torch.stack(G_m0).pow(2)\n",
    "        delta_f_m0 = torch.matmul(G_m0,delta_w_m0)\n",
    "        \n",
    "        kl = ((output_m0 - output_sf).pow(2)/delta_f_m0 + delta_f_sf/delta_f_m0 - torch.log(delta_f_sf/delta_f_m0) - 1).sum()\n",
    "        \n",
    "        torch.manual_seed(seed)\n",
    "        output_sf += delta_f_sf_update#delta_f_sf.sqrt()*torch.empty_like(delta_f_sf).normal_()\n",
    "        \n",
    "        loss = loss_fn(output_sf, target)\n",
    "        metrics.update(n=input.size(0), loss=loss.item(), error=get_error(output_sf, target), kl=kl.item())\n",
    "    \n",
    "    return metrics.avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_variance(model1,model2,alpha,seed=1):\n",
    "    \n",
    "    delta_w_s = []\n",
    "    delta_w_m0 = []\n",
    "    \n",
    "    for i, (k,p) in enumerate(model1.named_parameters()):\n",
    "        mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "        delta_w_s.append(var.view(-1))\n",
    "\n",
    "    for i, (k,p) in enumerate(model2.named_parameters()):\n",
    "        mu, var = get_mean_var(p, False, alpha=alpha)\n",
    "        delta_w_m0.append(var.view(-1))\n",
    "\n",
    "    return torch.cat(delta_w_s), torch.cat(delta_w_m0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "alpha_list = [1e-8,1e-7,1e-6,1e-5]\n",
    "test_error_list = []\n",
    "information_list = []\n",
    "\n",
    "runs=7\n",
    "for s in range(runs):\n",
    "    test_error_list.append([])\n",
    "    information_list.append([])\n",
    "    for alpha in alpha_list:     \n",
    "        delta_w_s, delta_w_m0 = get_variance(model_scrubf, modelf0, alpha)\n",
    "        metrics = test_activations(model_scrubf, modelf0, delta_w_s, delta_w_m0, test_loader_full,seed=s)\n",
    "        test_error_list[s].append(metrics['error'])\n",
    "        information_list[s].append(metrics['kl'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha_list = [1e-8,1e-7,1e-6,1e-5]\n",
    "alpha_list = np.ndarray.flatten(np.array([alpha_list for i in range(runs)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_error_list = np.ndarray.flatten(np.array(test_error_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [],
   "source": [
    "information_list = np.ndarray.flatten(np.array(information_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "info_dict_act = {}\n",
    "info_dict_act['alpha'] = alpha_list\n",
    "info_dict_act['error'] = test_error_list*100\n",
    "info_dict_act['info'] = information_list\n",
    "df_act = pd.DataFrame(info_dict_act)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Information in Activations for Different Subsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_scrubf = copy.deepcopy(model_scrub)\n",
    "modelf = copy.deepcopy(model)\n",
    "modelf0 = copy.deepcopy(model0)\n",
    "\n",
    "for p in itertools.chain(modelf.parameters(), modelf0.parameters(), model_scrubf.parameters()):\n",
    "    p.data0 = copy.deepcopy(p.data.clone())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0903ea60d24445f884558edbc5f53a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "719bda8db32f4be3b7dc2e2ecb33b7a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9cfa9fe85ac04fc5a7dda020d19401e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "hessian(retain_loader.dataset, model_scrubf)\n",
    "hessian(retain_loader.dataset, modelf)\n",
    "hessian(retain_loader.dataset, modelf0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets={}\n",
    "datasets['Forget_Set']=forget_loader\n",
    "datasets['Retain_Set']=retain_loader\n",
    "datasets['Test_Set']=test_loader_full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "information_list = []\n",
    "\n",
    "runs=1\n",
    "for s in range(runs):\n",
    "    for k in datasets.keys():     \n",
    "        delta_w_s, delta_w_m0 = get_variance(model_scrubf, modelf0, 1e-8)\n",
    "        metrics = test_activations(model_scrubf, modelf0, delta_w_s, delta_w_m0, datasets[k],seed=s)\n",
    "        information_list.append(metrics['kl'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels=['Forget Set' ,'Retain Set', 'Test Set']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "def plot_info(ax,df,information_list,title,no_barplot):\n",
    "    \n",
    "    if no_barplot:\n",
    "        sns.lineplot(x=\"info\", y=\"error\",data=df,ci='sd',ax=ax)\n",
    "        ax.set(xscale=\"log\")#,yscale='log')\n",
    "        ax.set_xlabel('Remaining Information (NATs)',size=16)\n",
    "        ax.set_ylabel('Test Error (%)',size=16)\n",
    "        ax.set_title(title,size=16)\n",
    "        ax.tick_params(axis=\"y\", labelsize=16)\n",
    "        ax.tick_params(axis=\"x\", labelsize=16) \n",
    "    else:\n",
    "        y_pos = range(len(information_list))\n",
    "        ax.grid(zorder=0)\n",
    "        ax.xaxis.grid(False)\n",
    "        ax.yaxis.grid(True)\n",
    "        ax.set_axisbelow(True)\n",
    "        ax.bar(y_pos, information_list, align='center', color=matplotlib.cm.get_cmap('tab10')(0.95), width=0.5,capsize=5)\n",
    "        ax.set_title('Information in Activations',size=18)\n",
    "        ax.set_facecolor('whitesmoke')\n",
    "        ax.tick_params(axis=\"y\", labelsize=18)\n",
    "        ax.set_xticks(y_pos)\n",
    "        ax.set_xticklabels(labels=labels, size=18, rotation=45)\n",
    "        ylabel='NATs'\n",
    "        ax.set_ylabel(ylabel,size=18)\n",
    "        ax.set_ylim(bottom=-0.001)\n",
    "        \n",
    "    ax.set_facecolor(np.array([231,231,240])/256)#'whitesmoke')\n",
    "    ax.grid(color='white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABAwAAAEYCAYAAADYnGRcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd5xcVf34/9d76vbNluymZxMSAgkQAiK9FwGRIvIVERBRUCmiH/3ZEVH82AuiqIAfkaqodBEFaaFJCQFCCell07Ykm+31/fvjnIHJZHZ3ZrOzs+X9fDzmMTv3nnvvuWXP3HnfU0RVMcYYY4wxxhhjjIkXyHYGjDHGGGOMMcYYM/xYwMAYY4wxxhhjjDE7sYCBMcYYY4wxxhhjdmIBA2OMMcYYY4wxxuzEAgbGGGOMMcYYY4zZiQUMjDHGGGOMMcYYsxMLGBhjjDHGGGMyQkQ+IiKvikiriKiIHJXtPGWKiBzl9/GCLOdDReTmbOZhsIjId/z+VGU5HzeLiGYzD9liAQNjjDHGGGPMDuJ+/H55F9axO3An0ABcBpwHvDVIWcwKEdnX/4itynZesklEfuSvj2WDsK7TReQ7g5CtXc3HBSLyhWznY7ixgMEw4y9UFZFZA1z+QhFZJiIdIrJtsPOXSb7wPSbJ9JtFZHUW8qODUXiJSEhEmkTk/5LM+6ffzskJ04tEpEtEfprmtmLXT9UA8nmziKxPId1R/lxZ+WGGLStLR19ZmmS9h/p1bxGR0ADXkbQ8E5GqTD4ljPvBUZpkXkaOlzFZchQQAr6gqn9Q1dtUdXOW87Sr9gWuAqqSzHsKyAVuHcoMJZELXJSplfsy93xgBTBLRI7cxVWejjumyVyD2581u7iNVFwA9BYwuMjnY8yxG/5RREQmATcAzwLHAMdlN0dpuwqX70TfA84Y4rwAHAzctKsrUdUu4DngiPjpIhIEDgFaEucBhwJB3BdPOv6By/fGAWU2NUfhzpWVH2ZUsrJ00A1KWZrEJ/z7eOCkAa7jKJKXZxtx+f7HANfbn9gPjp0CBmTueBmTDRP8e/1grlREwiKSM5jrHAyq2qOqbaraneV8tKlqZwY38UHcub0Y2AJcmKkNqWqX35+sNgdQ1U5VbctmHrLFbvhHl9m4H5l/UtWnVfWlXVmZL4xlcLI2cKq6QlVfycJ2n1fVfp+4p+hJYDf/QyRmX6AId2OYGDA4AlBgYTobUdUan+/2XcmsMWOclaWDu93BLEsB8D8U/h/wBC7o+ok+F0iTqrb7fNcM5npT3PagHy9jBktc7ZvviMgpIvKiiLSJyEYR+Ul8bR/f3vtq/3GVX251wrpuFZHNItIuIitE5H9FJC9hm7E27PNE5Oe+NmQbcFBsO74G1TEi8pyItIjIehH5qp9fIiJ/8LWRWkTkwYT7MURkkoj8TEQWi8hWv09vishX/QOed/MC/NF/fNxv+93+AqSXPgxEJF9EfuD3sV1ENonILSIyPSHdu8uLyCdF5A2ffo2IfCWN87RTHwZxx+lgEXlSRJpFpE5EbhKRglTX7X0KWAk8DtwOfEREinrJS5GIfF9E3vLHtU5EnhaRs/38J/BleNzxfPcYSkIfBiLyOf/51CTbCvhzvzhu2gki8hcRWSmuH41tIvLvxFoR/to8EpiekI+j/PykfRiIyD4ico/fr9h185X46yZ+eREpFpHf+uuxTUSeEZEDk+zHF0TkNRFpFJHtIrLUX8fhXs9KpqiqvYbRC1cVRoFZcdOeAJ7GPeVahLs5WgKcEZfmZr9c/OtmPy+Mq86zGujw79cA4bjlq/wylwA/BjYAPUBJXJ4OAe4CGoHNwNf9sicCrwDNwIvA/gn7dALwEO6JTSzvXwKCcWkS867Ad+L2bXXCOicCtwC1QDvwGnBuL8fyIFxhtt3v16+AnBTOxbt58J+/46fNxj11asJVj/o2EOhnXbEAwNlx074IvIl7wtUB5MbNewZ4NWEd44HfAdV+n98GLu5ln6vipuUBvwXqfJ7v8edSgQsSrqH1wAJcoKIFWAZ8Nskx2OHl54VwTzBX4L7Ia3HX7WHZ/r+y19h7YWXpqCxL49Zxtl/HWcBtPu8lSdLlAz/05VI7sAn4O1BJ3+VZ7Dxe4D//f/6clyXZxpvAfXGfr8ZdX9v9cX0MOCjJ8Ux8VSU7XnHXxnNAK64t+L3AnIQ0T9DP9e3T7Y77HtiCK6vXAn8FQtn+v7XX8Hrh7k8U+HLctNj/xgv+Gvou8DngYT/9G3FpzwXu9tO/4D+f7udN98u3A7/AlZl3+bRPxl+Pcf+ri/3/wRf9+ub4+Qq8irvP+YHPz+N++hX+/+Fuv42fA13Aown7eiKwys+/1K8/tk+/j0u3D/B7P/37fp/OBQ5OOGYXxC0T9v+b6v/XLvH73IYrk6YkOebP+/x8C9f3w/N++jkpnrt3v7sSpi32x+mnwGdw/UsocEMa18UEoBO4Ku6YKAn3pH7eOFw5FNv3z/vzdwtwm09zPK5GrcYdz3OBmQnnv8p/LvHH7m9Jtne8T/vFuGl3AI/49XwaV7trnb8ODo9Ldzquf42ahHxU+vk3478j4pZ5H+47e5u/Hi4H/u3zcHtC2pvjzu0D/rxehSvTa4HCuLRX+rT3++vlM7jvsjeBgiEvC7JdGNkr4YT0fpO7EXjDX7gn+gu/K5YO2M1fpLEb1YOA3fy8O3za7+JuOL/j/9HviNtGlV+2GncjcgpwGq6tTixPy/wFfBzvFZY/Al7H3byd4i/kdUAkbt2fxd3UngQcjbvxagR+GJfmIL++P/q/D8IXoCTc5OJuAN/x/9AX+/XeTkJhlZDv7/p8Xwl0A1encC56u8mN3aQfB1zrp32yn3VFcYXb9XHT7vHHMRd3I3qMn56L+wK9Li5tEbAUd2N3kd/2T/y+XJ5kn6vipsVupr+OK0h/iPsSSvxCuxl3g/sWrmA63l87Chzt00zB1YhQXLOJg/A3wsA3cTf+V+AitB/C3Tifmu3/K3uNvRdWlo7KsjRuHf8EtuLK1hP8sp9LSBPBNStp9vk9HvgIcCOwB32XZ7HzeIH/PNnv7yUJ29jfpzszbtpNuI7djvbn8s+4Mn5vP388LriqPj+x8xTt5Xid6Lf9CHAqcA6w3J+3yelc3z7dMtyPvTNxZfU5uO+JSCrH3l5j50XfAYNmdrzXEP8/vTFhHbH/96qE6bGy5uSE6T/x0z+VZB1PkCSw5ef1AAfGTYv4/4ce4FcJ6X/ul5kTNy0XkCTrvtX//02Mm3aBX/6oPo7ZBXHTLvLTfpyQ9oN++q1Jlt8AFMdNz/P/88+leO6U5AGDHY6Tn/4P3HdZSj9Ega/69cyIm/YK8N8kaa+n92BCIO7vm0n4Md7XNYQLPrSRECj256sTqIiblp9knZW4H+kPJUx/goTAel95xD3g6wL2SfhfiAW/jk1cnrjfAn76WX76Z+KmLQLeTOV8DMUr6xmwV8IJ6f0mtxOYHTetwhdg8ZHc4xILMGAvkj+t+Jafvo//XOU/LyKhwIzL07fjpoVwkeHOhALjVJ/2yF72T/yy38Td7MUXFgpck2SZm9nxJveyxP300x/1eQom5PvqhHQPAu+kcC56u8n9ZEK614F/p7C+J4ElccehBv8kD3dT+x3/99F+O2fFLXslrmCcnbDOG3EFXihhn6v85zm4Qv0rCcv9iuQBA8UHB/y0KC4SfUOS4xBKWOeDwN3Z/h+yl71UrSxldJelE3E3aL/3nwO42lHPJ6S70G+n16BlH+VZ7DxeEDftERJu1oFf+uMf7WX9QX+elgLX9nV99nG8XsL9yI9/4jrDXzM/T+f6Bsr7Oyb2slfsRd8Bg9uTpL/OzyuImxb7H6uKmxbABTsXJVlHqb9mH0yyjtN7yacCzySZfl+y/zNcXy4KnNLL+iI+H+W44JsCH4qbH/v/PaqPY3ZB3LSH/D4lqwX1Cu5hTSBh+e8nSfsAUJviuVOSBwySHacv+Xl7pbjupcCTCdOu8OuYl3Ce60nhhy/pBwxO8dPia8EW4B5cPdDHdgqAMn9u/wHUJcx/ghQDBrjyVUly7wvM9/N+nbg8O9/Ll/npP42b9jiu5sGwqKVrfRiMHMtU9d1hS1R1C+6Gblo/y8Xaxt+WMD32+ciE6feqv1KT+Gfc9rtwTzjeUdVVcWne9u9TYxNEZKKI/F5E1uCesnTiqvGOw/2zpesIoFpVn0iYfhvuyc3chOmJnVa9Tv/HrS+J61uS4vqeAuaKSBkuj+W810fB07x3ro6ISx9zIvBfXBvAUOwF/AtX0CTuc8yBuB8Wf02Y/rde0reo6uOxD+r6QniH1PbvReBk307tMBGJpLCMMUPNytL3jNSy9FzcD/FbAFS1B5fnA0VkTly6E4BNqnr/LuQx3i3AQeJH3vBl8MeAuzSu3xgROU5EHheROlxgoxPXDGBOknX2SUTygf2Av/hrBQB/rTzDztddf9d3Ha7d8Q9F5CIRmZ1unozxViaZVuffy/pZdjzuh9sbiTNUtR5XM2BmkuXeSTM/W/37ql6mv5tPf1/1LRF5B/eApg73YCc22kFJH9vuzwxgg6puTTLvDaAQd08Yr7fj29+x7c+unDdE5HBcefaoiMyKvXD3qD24vg1iynHHbfHOa9plD+PKtvPjpp2Jqzl3S0KedxORP4vIVlygqhZ3bk9m188rJLmOcbV1e0h+He9wDlQ12fH/Bu46XCgi1SJyu4ick617awsYjBzJepdtB/rrITbWA3Nir/mbEubTS7p4iQVdRy/TiOVL3FBV9+Migdfgeu4+ANfO5910aSrtJZ+97VPisWvHPTkfqGTrS2U/nsL9eD8cd6O+TlVjQ8QsxN2Ihv28pbrjsEMVfnpnwisWCOitkJ/o37ckTO9tSKNkX2ap7t//4tpinYrbnzoR+aOIJH4JGpNNVpa+Z6SWpZ/ANc96Q0TGicg43JNE2PHmsQzXNGSw3I2rhn2e/3wCrmx+9+ZURPbDPU1swt04H4Q7T68ysHNUgvve6O089XeOIO64+iDW8bhaCz8A3vEdgX1uAHkzY1tfowBkqpPXlj7m9Zof7X3Egvh8/hzXVGgR8Encj8njcdXvYeh/M2VqlIVdPW+xgMB3cTWfYq/ncMfo3KHolM8HUO8ADpb3hk8+H/dd+m6Q2Hfm+BTuwdu1uGZgH8Cd28fI3LXaq1SuR1V9DtdE8iO4Jsz74prxLJYkw/Fm2oDGLTYjSuzmYQKu0yfiPsfPj+ntidhA7YbrEOQ8VX33yZyIfGgX1llP8ic1GRm6ZxA9i3vadATuh3z8CAjP4G7oDuG9jsXi1eF+9F/Ry7qX9jI9dpNZwY4R9sqUc50idcP3/Aj4kYhMwP2w+Tmu3d1HB3t7xgwxK0uHARHZH5jnPyYLcJ4nIlf6Wge1uKYkg0JVm0XkHuDjuODoucBKVX0mLtmZuHL+wxo3pJmIlOA6xUrXVty1NCHJvAkM4Byp6krgfBERXLXZy4DrRWS1qv6z76WNGRQ1uCe98xJn+P+ViWTmqXRfzgOeUtWzE/IzK0nadMv3lcCJIjJOVRPLgbm810HqsCYihbgfsI/ghh5OtA+uCe2puM5la3Fl2PwUVj+Q78w/4TqnPF9EbsQ157hBdxwp7FhgEnChqv4xfmERuWYX8xG7r97pOsb1kxMgeY2OlKhqE+44/h1ARC4BfoML2vxkoOsdCKthMPrFqrWfnTD94/79iQxvPzY0TvyNUzhu+/E6cJ3O9OdJYIqIHJow/Rzcj+o3B5DPjFPVZuBlXMDgcOICBr4K3pvA/+CO2ZMJiz+MK3zWqupLSV6NvWz2BXx/CAnTEz+nI1YQ93quVHWTqt6Eaws9aDfsxmSRlaXDwyfwnQzi+nuJf/0Q14TjaJ/238CEfoIq/ZZnCW7BDZH7AVyP2olNVPJwT/DevekUkWPYualFStuN+944S3Yc2m06LsD8RIr5TrZuVdXFuO8dsLLaDBEf0HsAWCAiJybM/hru98k9Q5ytbhKeNvsmQV9MkrbJv6f6pPde3D59LWH9J+FGprrfH5Ph7mxclf/fqerfEl+4MrgF139M7DzfiWuO+6nElfmgZUyTn5by03Nffr2GC96ehzvGf0pIFnuan3huT8A1203UBJQk5K237W/BPQz8kIi8W376Zb/uPw7oOu6ldu4i/241DMzgUtUlInIn8B3f3vJZ4GBcBPBOVX09w1l4Czdc1vdFpBt3s5us8AV3c/pBEXkYF5HcoKobkqS7Gfek/W4R+Saus6uP46oXfaaPqj7DwVPAl3EF18KEeQtxoxPE0sX7Be4p/UIR+QWuRkE+LohwuKqelmxjqvq2iNwBfM9XaX4ZV5U5dgM9kC+o2I+IL4nIP4FuVX1JRO7DVbtdhDt/C3BVwH4/gG0YM6xYWZp9PkDyMVxnW3cnmb8Y/7QJ+A/ux/xFwJ0i8gNcG9tCXHXUX6rq2/RSnvWRjf/gei//A+7H/q0J8x/2ebhZRP6Ia+t7JTs3jYht91IR+RPufL6mqh3s7Epcfw8Pisj1uLbfV+M6xPpZH3ndiYjsg6uW+xdc3xlBXAduXbjqucYMlW/gypp7/XW9HPdA5aO4e6DEH36Z9jfgMyLyF9zDjkrcD9+6JGlfxN0/fdPXiGgGVqnqf3tZ9824YOdXRaQKt3+zcCPxbMYdi5HgU7iAwMPJZqpqiy9HTxeRyapajesY+BjgJv8j/WncPfAC3O/QWBOv53mvtlNs1Ib/JvTtk8yfcOXgV3F9AT2fMP9pXPOtn/ljvx5Xvf88XD88eyekfx5XQ/bXIvIsLuDwmA8OJHMFLvi+UER+47d1Cu575g5V/U8/+e/NWyLyPO57awOu1s3FuAcCfx7gOgfMahiMDRfgqopfiGtb+Sn/+ROZ3rC/+Tkd9w90C64qzVO4KGSiy3CF7gO4wvjiXtbZjOvo6d9+Pffhqjudp6rJqkgNJ0/iCspYjYJ4C/281aq6Ln6GqjbgniY9hCsU/wX8H264tsfp28U+7Vdwkc55uDGGwd1wputB3DA5l+DarL3opz+Fa9P7B9yXyedw49B/ZQDbMGY4ugArS7Ppg7hOtP4v2Uxf1fdu4EwRKfBNAk4Afos7Bg/hyq5y3qvK31t5lpR/YnYHbpjF51R1ecL8f+HGGT/Ur/tCXAAjMd2ruJ6/P4S7oX0RV2022TYf9vs+DjdU1+9wAaTDegkE9WUTrv+H/8G1873Tb/cUVX05zXUZM2C+D6cDcX0xnYsbceT9uL41Torv5HOI/A/wU1yz0Otw5foNJNQKAFDVtbj/7Vxc+XIn7p4nKV8WfQBXzr4ft6/n4vb9wMR7vuFIRObhztfDqtpXXxJ/xwUiPwHgO3o8GFeFfn/cj/urcAGTB+KWu9PPOwwXYLmTnTt1TeZ2XMCziITODv32t+GO/X9xQyb/DNcM5GTee2If7xe475iP+PXdSe8di+MDzIfg7u8v8eufjrtXP7+35VLwM6AY933yW9ywyi8AB/vvjyElvXfibIwZrUTky7gf81X+i88YY4wxxhhjdmA1DIwZ5UTkFBH5moicKCIniMj3cD0B32XBAmOMMaZ/IrK7iHxXRJ4XkRoRaRSRxSLyTd/WPTH9HBG5V0S2ikiziCz0/VkkW3exiFznh09rE5E3RORzqbSjNsaYTLMaBsaMciJyJK7a9B64fg+qce1Xr1LVtmzmzRhjjBkJROSHuOZ89+PaOXfiOtj8f7iO1w5S1Vafdjdc9eEuXPXzBlx/Gnvhqts/GrfeCK5ZygJcVfi3gJOAM4CrVfU7Q7B7xhjTKwsYGGOMMcYY0wcReR+wzPcpFD/9GuCbwOWq+ms/7S7cSB77+57cY+PBvwG0AXuovwGPGyrt86p6Xdx6/47r42K2b+9vjDFZYU0SjDHGGGOM6YMfwjhZR8F/8e97wbtD8Z0KPBELFvjlm4CbcCNnHBC3/Dm4nudvTFjvL4EwbtQAY4zJmlE5rGJpaZlOnZo45PHIFwoF6OoaCcO0jlx2jDNvJB3j115bXKuq47Odj5FitJa9/RlJ1/RIY8c2c4bzsR1hZe8U/77Zv+8DRHGjbiSKDfl2APCCH+54P2BRkiaCLwDKjsGFpIZr2Tucr7Gxxs7F8DJcz0dvZe+oDBhMnTqNhx/ub6S5kaesrIC6uqZsZ2NUs2OceSPpGE+aVGLVQNMwWsve/oyka3qksWObOcP52I6UsldEgsCVuL4K7vCTY8NjVidZJDZtsn8vwQ3Nt1NaVW0Xkdq4tL0armXvcL7Gxho7F8PLcD0fvZW9ozJgYIwxxhhjTIb9EjfG/DdUdamfluff25Okb0tI01faWPq8ZDNE5GLgYnABg7KygjSyPTRCoeCwzNdYZOdieBlp58MCBsYYY4wxxqTBD1F8GXCDqv4gblaLf48mWSwnIU1faWPpW5LNUNUbgBsA5s9foMPxaeVwfYo6Ftm5GF5G2vmwTg+NMcYYY4xJkYh8B/gW8EfgswmzN/j3ZE0JYtNiTRC2Aq3J0opIFCgnedMGY4wZMhYwMMYYY4wxJgU+WHAV8Cfg07rz+OSv45oYHJxk8YP8+0sAqtoDLAIW+ABBvPcDEktrjDHZYgEDY4wxxhhj+iEi38YFC24FLvQ/+Hfgh098ADhKRObHLVsAfBpYhhsBIeZOXD8FFyes6gu4zhT/gjHGZJH1YeC1dXWTEwpmOxvGGDOmdHT1EAlZ7NoYM7yJyKXA1cBa4FHgHBGJT7JZVR/xf38dOBb4t4j8AtgOXIRrevDBhFoJNwKfBH4uIlXAW8DJwBnANaq6OkO7ZIwxKbGAAdDVoyzd3MScigJywhY0MMaYodDZ3cNbW5qYMz7fyl5jzHB3gH+fhmuOkOhJ4BEAVV0uIocCPwS+BkRwTQ9OVNVH4xdS1Q4ROQ64BvgYUAasAC4HfpOB/QBgwdurM7XqIfXKHlXZzoIxo54FDLymjm7Wbm1l94qRM8SFMcaMZKrQ2NbJitpm9pxQSGDHp3XGGDNsqOoFwAVppH8LOC3FtNtwIy5cNpC8GWNMJlk9UKBHla4epaa5g4bWzmxnxxhjxoxQIEBDWxebtvc2DLkxxhhjjMmWMR8w6OpRLvv7Eh5YspmCSJCVdS109yR2eGuMMSZTinPCrK5vobmjK9tZMcYYY4wxccZ8wCAUEPaoKODZ1VvZuL2d9q4etjTZky5jjBkqwYCQEwqyvMYCtsYYY4wxw8mYDxgAXHjgVAqiIW59qZr8SJA19a20dXVnO1vGGDNm5EWCtHR2s6GhNdtZMcYYY4wxngUMgIJoiFPnVbKyvoVnVm0lGIB12+ym1RhjhlJxToi129pobLemCcYYY4wxw4EFDLz9phSxR0U+f3ttI6pQ09jB9ja7aTXGmKESECE/HGR5TTNd1jTBGGOMMSbrLGAACO5G9ZwFk2nr7OZvr20iLxxkZV0zPWo3rcYYM1RywkHau3tYb7W8jDHGGGOyzgIGuA63ppfmUhANcsKc8SxcWU91QxstHd3UWAeIxhgzpIqjIaobWm2YW2OMMcaYLLOAgVdRECU3EuT4OeWU5Ia55eVqCiIhVte10t7Vk+3sGWPMmCEiFEbCLKtpprPbyl9jjDHGmGyxgIEXDAi7leejCmcvmMjara08tbIeEaxqrDHGDLFIKEC3Kqu3tmQ7K8YYY4wxY5YFDOIURkNMKIyy+/h85lUWcPfrG+lRZXNju/XabYwxQ6woGqKmsYO6ZmsaZowxxhiTDRYwSDBlXC4BEc5eMInObuWvr24iNxxkVV2LdYBojDFDSEQojIZYUdtiTcOMMcYYY7LAAgYJIqEAM8vyyI0EOXGP8Ty7eitrt7bS1N5FTVNHtrNnjDFjSjgYQERYVdeCWtDWGGOMMWZIWcAgibL8CONyQhw7q5yyvDC3vlxNXjjImvoWOuwplzHGDKnCaIj6lg5qmi1oa4wxxhgzlCxgkISIUFWWR4/AOQsmUd3QxpMr6lGguqEt29kzxpgxpygnzMraFto6u7OdFWOMMcaYMcMCBr3Ij4SYWhxlRnku8ycVcu+STXR3Kxu2t9FkHSAaY8yQCgWEUEBYUdts/ckYY4wxxgwRCxj0YWJxLuFgkLP2mUS3Kn95dSO5oSCr6q0trTFm5BCRo0REk7y2JaQrEZGbRKRWRJpF5FER2Ttb+U5UEA3R0NbF5kYbNcEYY4wxZihYwKAPoYAwqzyfnEiAD+5ZwQtrt7G6voXGti7qrC2tMWbk+TxwcNzruNgMERHgAeBE4HLgTCAMPC4iU4Y+q8kV54RZVddCc4fV9DLGGGOMybSUAgYiEhSR00XkdyKyWES2iEijiKwSkYdF5OsislumM5sN43LDjM+PcPjMUioKItz6cjW54QCr6lvp7LYOEI0xmZOBsvctVX0+7vVS3LxTgUOB81T1TlV92E8LAF8ZxN3aJcGAkBMKsrymhe4eq+lljDHGGJNJfQYMRCQqIl8F1gF3A8cC7wB/BX4PPA5Ega8DS0XkXyLyvsxmeehNK81FRDhnwWQ2Nbbz2LI6unuUDdYBojEmA7JU9p4KbFDVx2MTVLUBV+vgtF1cd1IPv72FNfUtaS+XFwnS0tnNhobWDOTKGGOMMcbEhPqZvxxoBH4C/FlVNyZLJCIB4GjgXOApEblcVf8wqDnNopxQkKrSXFBlvynF3P/GFt4/bRzV29spL4iQH+nvMBpjTFoyVfbeLiLlwDbgX8DXVHWtnzcPWJJkmTeA80WkQFWbBrY7O2vr7ObmF9fT1N7NVSfkUpYfSWv54pwQa7e1MS4vQmHUymBjjDHGmEzo7y7ry8Bd2k8Pf6raA/wH+I+IXAlMH6T8DRsVBVE2N7bz4b0rWbKxkT8v3sin3j+V1XUtzJ1QiGv+a4wxg2Kwy94G4GfAk8B2YAHwDeA5EVmgqluAUmB1kmXr/XsJsFPAQEQuBi4GmDp1GmVlBX3vWZwbz38fH73heW56cT03X3AAeZFgyssC73Y+mxtOb7nBFgoF09pvkzo7tpljx9YYY0wq+gwYqOpf0l2hqq4H1g84R8NUMCDsVp5PS0c3H5pXwd9f28SRM0uZMi6H+pYOyvKj2c6iMWaUGOyyV1VfAV6Jm/SkiDwFvIDrCPFbA2Wvwx0AACAASURBVMmnX/cNwA0A8+cv0Lq61CshFGoP5+w3iZv+u45Lb3uZSw6dTiDN4Ou2tk4mFEapKs1La7nBVFZWQDr7bVJnxzZz7NgaY4xJxS6NkiAiM0RkpoyRx+uF0RATCqMcXFXCxKIoty+qJicUYFVdK13WAaIxZogMRtmrqotw/SIc4CdtxdUiSFQaN3/Qza0s5KP7TuLl9Q3c8/qmtJcvjoaobmilobUzA7kzxhhjjBnbBhQwEJE9ReQtYAWwDHh7OI3VnUlTxuUSDggfWzCJLU0dPPJOLV2qbNhuHSAaYzIrQ2VvrNnDG7h+DBLNBdYOZv8FiU7YvZwjZpby4JtbeG51enEJEaEwEmZZTbONXGOMMcYYM8gGWsPgt7ieuyuAWcBS4HeDlanhLBIKMLMsj0nFORw4bRz/eGsL7Z3dVDe009LRne3sGWNGt0Ere/2oCnNwzRIA7gcmi8iRcWmKgA/5eRkjIpy3/2TmjM/n/15Yx4ra5rSWj4QCdKuyemv6Iy4YY4wxxpje9Tes4qW9VHmdD3xXVWtVdRVwLTAmahgAlOVHGJcT4tR5lQRFuPOVDYQDsLq+hX76KDPGmH4NdtkrIreLyDUi8mEROUZEvgQ8DFQDv/LJ7geeA24TkbNF5AN+mgA/HoTd6lMoGOCyw6ooyQ3zq6dXU9fckdbyRdEQNY0d1DW3ZyiHxhhjjDFjT381DC4FXkgyvvdbwGUiEvJPoC7EPekaE0SEqrI8ciJBTt2rklc3NPJOTQvbWjvZ2mLtaI0xu2ywy94lwKnAH3HDKX4BV1PhQFWthXdHXDgFeAS4HrgH6AaOVtV1u75L/SuIhrjiiBl0dvdw7cJVtHWmXmtLRCiMhlhR20J7lzVNMMYYY4wZDP0FDObjbiofE5HfiEixn/553A1nG64jrKOAyzKVyeEoPxJianGU908rZnJxDncsqiYSCLCyvoWuHqtlYIzZJYNa9qrqD1R1H1UtVtWwqk5V1YtVdWNCunpVvVBVS1U1T1WPVdVXB3XP+jG5OIfPHTKd9Q1t3Pj8WnrSqLUVDgYQEVbVWW0vY4wxxpjB0GfAQFU7VfUHwF7AZGCpiJynqi/h2s8eBBwI7Kaq/814boeZicW5RENBPrbvJOpaOvnXOzV09fSwyTpANMbsgrFe9u49sYiz953Eourt3P1aeiMnFEZD1Ld0UJNmkwZjjDHGGLOzlDo9VNW1qno68CngKhF5Apihqi/515j8hRwKCLPK85lQHOWQqhL++XYNLe09rNvWSmsaVWmNMSaZsVz2Hr97OUfuVso/3trCs6vq01q2KCfMytqWtJo0GGOMMcaYnaUUMBCRoIjkqOo/cMNuPYVrX/tDEcnNaA6HuXG5YcbnRzh5zwoiQeGORdWERFhT35rtrBljRrixXPaKCOfuP4U9KvL544vrWZ7GyAmhgBAKCCtqm9Nq0mCMMcYYY3bU3ygJk0TkIaAVaBKRV4B9VPXbwH64drZvichpmc/q8DWtNJe8SJAz9prAG5ubeHtLM/UtHWxrtQ4QjTHps7LXCQWESw+tojQvzHULV1ObRjODgmiIhrYuNjfaqAnGGGOMMQPVXw2DPwCFwHHAwbietu8XkaCqLlfVk4AvA9eJyAOZzerwlRMKUlWay35TiphWksudr2wgKMKKWusA0RgzIFb2egXREF84YgadPT38Ks2RE4pzwqyqa6G5oyuDOTTGGGOMGb36CxgcClytqk+p6ovA5UAlMDOWQFX/BuwJvJmxXI4AFQVR8qMhPjp/IltbO3l4aQ0d3T32dMsYMxBW9saZWJTDJYdUUd3Qxu/TGDkhGBByQkGW17TQbcFbY4wxxpi09RcweAc4R0TyRSQEfBpoB9bHJ1LVZlX9aobyOCIEA8Ju5flMKIpyxMxSHllaQ2NbF+u2tlrHW8aYdFnZm2CviYV8bMFkFldv5++vbex/AS8vEqSls5sNDdavjDHGGGNMuvoLGFwCHAtsx437fSVwkaranVcShdEQEwqjnDCnnJxwkDsWVRMQWLvVDpcxJi1W9iZx7Owyjp5VxkNv1fB0GiMnFOeEWLutjcZ2a5pgjDHGGJOOPgMGqvoCMBs4ADgMmKqqtw1FxkaqKeNyKYyGOGPvCSytaeaNTU3UNHfQYB0gGmNSZGVvciLCOftNZs+KAv704nqW1aQ2ckJAhPxwkOU1zdavjDHGGGNMGvodVlFVO1R1kao+r6rbB2OjInKyiDwlIk0isl1EXhKRY+Lml4jITSJSKyLNIvKoiOw9GNvOtEgowMyyPPaZVMjM0jz+vHgDQYGVddaG1hiTukyUvaNBKCBccuh0yvLDXPf0amqbUhs5ISccpL27h/XbxnQlDWOMMcaYtPQ3rOLJ6a5QRMpF5H19zP8McB/wMnAGcBbwVyDPzxfgAeBEXEdfZwJh4HERmZJufrKhLD9CaW6Ys+ZPpLGti3+8VUNbVw9bmqwDRGNM/zJR9o4mBdEQVxw+g+4e5ZcLV9GaYj8xxdEQ1Q2tVuPLGGOMMSZF/dUwuFlE/isiF4hIYV8JReQAEfkVsAp4fy9pqoBfAv+fqn5RVR9R1X+p6o9U9UGf7FRcD+HnqeqdqvqwnxYAvpLynmWRiFBVlkdFUZSjZpXxn2W1bGvpZE19K21d1gGiMaZfg1r2jkYTi3K45NDpbNzexu+fW0tPCjW4RITCSJhlNc10dvcMQS6NMcYYY0a2/gIGuwH/AX4B1IjICyJyo4j8r4hcJSK/FJH7RWQT8Dyuve1pqnp9L+u7EOgBftfHNk8FNqjq47EJqtqAq3VwWmq7lX35kRBTi6Mct3sZBZEQty+qRgTWWXVYY0z/BrvsHZXmTSjk4/tN5tUN2/lbiiMnREIBulVZvbUlw7kzxhhjjBn5+uv0sFFVvwFMAj4DLMf13P0/wLeB84Ay4I/A/qp6sKo+1scqDwPeBs4WkRUi0iUiy0Xk0rg084AlSZZ9A5gmIgUp7lvWTSzOpSgnzJl7T2BFXQuvVm9nS2M729usp25jTO8yUPaOWsfMLueY2WX88+0aFq5MbeSEomiImsYO6pqtmZgxxhhjTF9CqSTyQ3n9yb8AEJGAqqZbp3OSf/0E+AawAteHwa9FJKSq1wKlwOoky8buBEuApsSZInIxcDHA1KnTKCsbHnGF48oKOHjPCby6uYl739jMJ4/cjeLcMLnhYNrrCoWCw2a/Ris7xplnxzh1g1j2DkuBgBAMCK2d3QMqE2POWTCZTdvb+dNL66ksiLB7Rd/Xl4hQGA2xoraFgmiYaKjf/n+NMcYYY8aklAIGyQzwhjUAFAIXqOrdftpjvm+Dr/t2uAPNzw3ADQDz5y/QurqdYgpZ886WJk6YXcZP1m/jqnuXcMrcCnYrz6OyMCet9ZSVFTCc9ms0smOceXaMd81oCRaAG/Fg7oQC3tzUREtHN3mRgQUNgn7khGseWc51z6zm28fPZnxBtM9lwsEAbV09rKprYU5FPq6/XWOMMcYYE2+oH6vU+fdHEqb/G6gEJgJbcbUIEpX6962ZyVrmTCvNpbIwyjGzynlieR11LZ2srmulvWvU3PcbY8yA5EdC7DXR9evY1D7w5lr5kRBXHDEDVbh24eqURk4ojIaob+mgpjm1oRmNMcYYY8aaoQ4YvNHP/B6fZl6SeXOBtao64h5N5oSCVJXmcsysUopzQtyxqBpFbTxwY4wBcsNB5k4sIBgUmjoGHjSYUBh9d+SE3z27JqWRE4pywqysbaEtxaEZjTHGGGPGkqEOGNzj3z+QMP1EYL2qbgLuByaLyJGxmSJSBHzIzxuRKgqilORHOGOfCayub2XR+u1sbmyncReeqBljzGiREwoyr7KQSDDA9rbOAa9nbmUh5+4/mdc2NnLXq/2PnBAKCKGAsKK2mR7tP8BgjDHGGDOWDHXA4CHgceD3IvJZETlBRG4ETgCu9GnuB54DbhORs0XkA36aAD8e4vwOmmBAmFmWz7zKAvaoyOfu1zfR1aOsrG2xm1RjjMENebhnZQH50SDbWgceNDh6VjnHzS7nX0treGpFXb/pC6IhGtq62NxooyYYY4wxxsRLKWAgIhER+YyIzN2VjamqAqcDfwauBh4EDgQ+rqo3+zQ9wCm4fg6ux9VK6AaOVtV1u7L9bCvKCTGxKIfT95pAW2c3D76xheaOLmqarP2sMWZng1X2jiThYIA5FYWMyw2ztXXgZePZCyYxb0IBt7xczdtb+m/JVpwTZlVdC8270CTCGGOMMWa0SSlgoKodwC+B8bu6QVXdrqqXqmqlqkZUdR9VvSMhTb2qXqiqpaqap6rHquqru7rt4WDKuFwmFkU5bvdyFq6qZ0tjO2vqW+iwDhCNMQkGs+wdSUIBYfb4fMryI2xt6UAHUAsrGBAuOaSK8fkRfvP0arY09V17IBgQckJBlte00J1C3wfGGGOMMWNBOk0SlgLTMpWRsSISCjCzLI8jdiujJDfMHa9soKtHqW5oy3bWjDHD05gse4MBYVZ5PpWFUba2dg4oaJAXCfKFI2agwLVPraKlo++ODfMiQVo6u9nQYB3SGmOMMcZAegGDq4Fvi8jumcrMWFGWH6GyIMKZ+0xg3bY2XlrXwIbtbbs0pJgxZtQas2VvQIQZZXlMKs5ha2vngPp7qSyMcumhVWxubOd3z63pt/ZAcU6ItdvarENaY0YpcY4VkbNEpDjb+THGmOEulEbaS4AC4A0RWQpsBOLvvFRVE0c/MEmICFVleWxr62KvCQXcu2QTe08oZFV9C3tNKEREsp1FY8zwMabLXhFhekkuwYCwtr6VktwwgUB6ZeSelQWcu/8U/vTSeu5avIGP7Te517QBEfLDQZbXNLP3pCJCaW7LGDN8iMhVwFGqenTc5AeAk/zfG0Xk4JHeR5YxxmRSOjUMCoHVwEtAI+4GtjDuVTTYmRvN8iMhpo3L4dS9KunsVh54cwuNbV3UNVsHiMaYHYz5sldEmDoulxlleWxt6xxQHwNHzSrj+N3L+fc7tTzZz8gJOeEg7d09rN9mTROMGeFOB16LffAjb50M3ABciitPv5qdrBljzMiQcg0DVT0okxkZiyYW57K5qYMTdi/nobdrOGTGOMLBAMW5YcLBoR7x0hgzHFnZ+55JxTkEA8Ly2maKoyFCaZaTH913Epsa27n1pfVUFkTZo7Kg17TF0RDVDa5GQ3FueFezbozJjmnAO3GfTwfWqernAERkOnBWNjJmjDEjhf0qzaJQQJhZmsdhM0spz4/w51c20tHVwwbrANEYY5KqLIyyR0U+De1ddHanN7pMMCB89uDpVBZG+fUzq9nc2PvICSJCYSTMsprmtLdjjBk2coH4m6qjgUfjPi8DJg1pjowxZoRJK2AgIuUico2ILBSRN/z7d0WkPFMZHO1K8yNMLs7hjL0nUN3Qxn/XbmN9Q5uNBW6MeZeVvTsqy48yt7KAxvautIekzYsEueLwGQBcu7DvkRMioQDdqqze2rJL+TXGZM164H0AIjIb2B14Mm5+GWD/4MYY04eUAwYiMhN4FfgarinDO/7968BiEZmRkRyOAdNKc9mzsoD5kwq5/43NtHX0sLquZUDDiBljRhcre5MryYswb2IhLZ3dtHf1PVxioorCKJcdVsWWxnZ++2zfIycURUPUNHZQ19x7bQRjzLB1N3CRiNwG3IsLDjwUN38fYFU2MmaMMSNFOjUMfgx0AHNV9WBVPUNVDwbmAu3AjzKRwbEgJxSkqjSXU/asoEeV+97YRENbF/Ut1gGiMcbK3t4U54TZa2IhbV09tHamFzTYo6KA8983hSWbGvnz4g29phMRCqMhVtS20J5mbQZjTNZdA9wHfASYAHxGVWsBRKQA16fBY9nLnjHGDH/pBAyOAb6pqvGdx6Cqy4BvA8cNZsbGmoqCKFNKcjlxznheXNfAum2trKxrpcvazhoz1lnZ24eCaIi9JhbS1aN9Ni9I5ojdyjhhTjmPvlPL48tre00XDgYQEVZZzS9jRhRVbVLVM4F8VS1T1dvjZncAewHfzU7ujDFmZEgnYJADNPQyb6ufbwYoGBBmluVz6IwSKgpcB4jtnd1s2G4dIBozxlnZ24/8iAsaADS1p9f/y0fnT2KfiYXc9nI1b25u7DVdYTREfUsHNTb0rTHDmoh0iMjZ8dNUdadooqp2qOoaVW0eutwZY8zIk07A4HXgs73M+wxx49yagSnKCTFlXC5n7D2BTY3tPLdmG9UN7Wk/NTPGjCpW9qYgNxxk7sQCgkGhKY1OYwMB4bOHTGdCYZTrn1nDpj5GTijKCbOytoW2NJs/GGOGVAgbBcwYYwZNOgXqNcDJIvKaiHxDRD4pIl8XkVeADwLfy0wWx5ap43KZW1nAfpOLePDNzTS2dbK6vtmqwRozdlnZm6KcUJB5lYVEggG2t3WmvFxuOMgVR8xAgGufWtXrKDWhgBAKCCtqm+mxMtkYY4wxY0DKAQNVfQD4sF/mGuAPwPeBIPBhVf1HRnI4xkRCAWaU5XHy3ApEhHuXbGZbq3WAaMxYlamyV0QeFhEVkWsSppeIyE0iUisizSLyqIjsvYu7MWQioQB7VhaQHwmyrTX1oEFFgRs5oaa5g+uf6X3khIJoiIa2Ljb3URPBGGOMMWa0CKWTWFXvA+4TkRKgFKhX1a0ZydkYVp4foaokl5P2GM+9SzZzaFUJkVCA3adlO2fGmGwY7LJXRD4GzE8yXYAHgCrgclwfCV8HHheRfVV1/UC3OZTCwQBzKgtZXtPM1tYOSnIjKS03p6KA8983mT++sJ47X6nm3P2nJE1XnBNmVV0LRTkh8iNpfY0aY4bGbBE5JNXEqvpsJjNjjDEjWUo1DEQkIiIbROQUAFXdqqorLFiQGSJCVVkeB88oZWJRlL8s3khLZxedNmKCMWNKJspeH3T4BfA/SWafChwKnKeqd6rqw35aAPjKQLeZDaGAMHt8PmX5EepbOlJu1nXEzDJO3GM8/1lWx2PLko+cEAwIOaEgy2taeq2JYIzJqm8DC9N49cs3BfuriKz0tbNW95P+QF9Dq1FEtvtaXfv2knaSiNwiIjUi0ioiL4nIWanurDHGZFJKj0ZUtUNEIoB12T9E8iMhZpTkcMbelVz/zFqeWbGVE+ZPpbWzm9xwMNvZM8YMgQyVvT8ClqjqnSJyR8K8U4ENqvp4XB4aROQB4DTg84OYj4wLBoRZ5fkERdjc2E5JbhhXiaJvZ+0zkY3b27l9UTWVhVHmTSjcKU1eJMjW1k42NLRSMX7n+caYrLofWDLI6/xfoB5YBIzrK6GIHAQ8AVTjghcAlwELReQQVX09Lm0p8DRQAfwcWA+cA9wlIheq6h8HeT+MMSYt6dSljLWjfTRDeTEJJhbnsmdlIQdMLeaht2u4qL6F2vpW5lTkp3TTa4wZFQat7BWRw4DzSdIcwZtH8pvsN4DzRaRAVZt2NR9DKSDCzLI8ggGhelsbJXlhAv2Un4GA8JmDp/H9R5dz/TNruPL42Uwoiu6UrjgnxNptbexjHSAaM9z8VVUTA6K7ajdVXQkgIkuAgj7S/groAI5Q1Wq/zF3AW8DPgBPi0n4NmAGc6vusQUT+ADwH/FRE/jrSyl1jzOiSzigJfwc+JCK3ichHRORQETkk/pWpTI5VoYAwszSPk/YcTygg/OjfS6lrbqehLb1xxo0xI9qglL2+psLvgZ+q6tJekpXi+i1IVO/fS9LO/TAgIkwvyWVaSQ5bWzrpSaEZQW44yBcOn0EwIPxy4cqkIycERMgPB2nv6qHLmiYYM6rFggX9EZFZwAG4oEV13PLVwF+B40RkQtwi5wArYsECn7YbuA5XJp88CNk3xpgBS6eGwf3+/Rz/ir87Ev/Z6soPstL8CDPL8jlpzwrueX0T88bnkxMOMX9yEaGA1TIwZgwYrLL3K0AuboSFQSMiFwMXA0ydOo2ysr4eumVXeXkh87qV9u4eQiLu6PVjfHk+n77lZW59ZSPXn7OAcHDnOHteNESwNJ9Iknlm14RCwWF9TY1kdmwz5gD//lySec8DFwL7A/8QkYnAZOD2XtLG1nfXYGfSGGNSlU7A4KSM5cL0aXpJLodUlbBkcxN/fXUjVaV5bG5sZ3JxTrazZozJvF0ue0VkGvBN4NNAVETi69dHRWQc0IirXZCsFkGpf9+p9oGq3gDcADB//gKtqxv+NWc3N7azvLaZ4miIUAo/8j/xvin84YV1fPkvizn/fTuPnLD/7hU8sng98yYUUpwbzkSWx6yysgJGwjU1EtmxzZhJ/r06ybzYtMkDSLuDkRSszaSxut/psgDh8DLSzkdKAQMRCQKrgS02MsLQywkH2a08j6+fuAcX3vISC1fUkRMKUJYXJsc6QDRm1BrEsncmkAPclmTel/1rAa6vghOSpJkLrB0t7WgrC6OEAvD2lmaKoqGktQbiHTazlA3b2/jn2zVMLs7h2NnlO6UpiIRYVtPM/MlF/a7PGJNRs4FNqST0T/g/oao/HMTt5/n39iTz2hLSpJN2ByMxWJsJY3W/02UBwuFlpJ2PVO9qFNcR1vszmBfTh4qCKPtPH8dB08fxr3dqqW3uYO3W1mxnyxiTWYNV9i4Gjk7yAhdEOBpYjmv+MFlEjowtKCJFwId4r2nEqFCWH2VuZQGN7V10dPU/ZO1H9pnIvpOLuGNRNUs2Nu40PxIK0K3KGiuXjckqP/Rsc2/zRSQoImf40V/WMMjNtIAW/75zT6kucBufJp20xhiTFSkFDFS1B1c1yurAZ0kwIERDAU7cYzzRYIB7Xt/ElqZ2Glo7s501Y0yGDFbZq6rbVPWJxJefvcZ/bsIFBZ4DbhORs0XkA36aAD/elTwMRyV5EeZNLKSls5v2ru4+0wYCwsUHTWNSUQ7XP7uajdt3HumyKBpiS2M7dc3JHhYaY7JJRPYUkZ/iytS/4QKl9wEfH+RNbfDvyZoSxKZVDyCtMcZkRTr1Jm8CLvdVZE0WBMSNKX7ynuN5c3MTS7c0s7KuhW7rnduY0WzIyl4foDgFeAS4HrgH6AaOVtV1md5+NhTnhNlrYiFtXT20dvYdNMgNB7niiBmEAwF++dQqmtp3HDlBRCiMhlhR20J7CrUWjDGZJSIFIvJpEXkOV1vrC8B44HvAeFU9S1X/PMibfdG/H5xk3kG4mmMvA6jqRlxA4KBe0gK8NMj5M8aYtKTT6aEAewDLROQhYCM79tatqvqDwcyc2dnUcbkcOqOEF9c18LfXNjKzzHWAOMk6QDRmtMpY2auqO40ToKr1uF68LxzIOkeigmiIvSYW8uamJlo6usmL9B6bKc+PcNnhVfz4sRX85pk1fOmomTvMDwcDtHX1sKquhTkV+YjYaDbGDDURORxXhn0EyMc1y/oi8ALwLPCaqmak/ZCqLheRl4CzRORKVd3g8zQJOAt4TFXj+1i4E/iyiHwoNrSiDxBfDmwDHspEPo0xJlXpBAyujvv7kiTzFbCAQYZFQgFmludz2l6VXLdwNY8vryM3HKQ0P0xOyCp/GDMKWdk7BPIjLmjw1qYmmtq7KIj2/vU4uzyfTx4whRv/u47bX67m/XMqdphfGA1R39JBTXOYioJkTZONMZkiIkuBWUANrlPAm1X1dT9vt11Y73nAdP9xPBARkW/5z2tU9da45FcAjwMLReQ6P+1yXM3eLyWs+oe4QMIdIvJzXI2Dj+GGU/y0qu7caYoxxgyhdAIGuRnLhUnL+PwI+0ws5JCqEv6zrJb9pxazblsrs8tHzvAcxpiUWdk7RHLDQeZOLOCtzU00dXRREOn9K/KQGaVUb2/nobe2cOeL65gzbsdaXkU5YVbWtlAUDdloNsYMrdm4TlwvjuurZTB8CjgyYdr3/PuTwLsBA1V9VkSOAq7xL8XVbDhLVV+NX4Gq1onIobjAwaVAAfAmcLaq/mUQ82+MMQOSch8Gqtre3yuTGTXvERGqyvL4wJ7jyQsHufu1TWzZ3s72tq7+FzbGjChW9g6tnFCQeZWFRIIBtrf13ansmftMYL/JRfz4X0t5feP2HeaFAkIoIKyoa6ZHrZ8ZY4bQT3A/uv8jIstE5FsiMr2/hfqjqkepqvTyOipJ+udU9VhVLVDVQlX9gKou6mXd1ap6nqqWq2qOqu5nwQJjzHDRZ8BARE4QkcKEaTuNBysiFSKSrKqsyZD8SIg9xufzwbnjeaemmSUbG1lpN6bGjApW9mZXJBRgz8oC8iNBtvUxEk1AhIsOmsbsigJ+++waNjTsOHJCQTREQ2sXmxstpmPMUFHVrwJTgQ8DbwHfBlaIyGPA2ezYB4wxxph+9FfD4J/AnrEPvhOWRhHZLyHdDOA6zJCaWJzLITNKmVGay92vb6a+uYMtdmNqzGhgZW+WhYMB5lQWMi43zNbWjl7T5YSD/OrsfQkHA1y7cOeRE4pzwqyub6G5w2qAGTNUVLVbVe9T1VNxwYNvApNwTQgEuFhEThMR6zHaGGP60V/AIFn3ztbl8zARCgizyvI5fa9KGju6eHRZHWvqW204L2NGPit7h4FQQJg9Pp+y/Aj1LR1oLzW4JhbncvlhVdS3dPLrp1fT1f1eGRwMCNFgkBW1NgSuMdmgqptV9UequgeuD4JbgEOBu4EaEbkrqxk0xphhLuU+DMzwVJofYf7kYg6bUcoTK+pY39DK+m0ZGSnIGGPGnGBAmFWeT2VhlK2tnb0GDWaV53Ph+6eytKaZ216u3iFdXiRIc0c3GxqsbDYmm1R1oapeAEwEPgu8AZyZ1UwZY8wwZwGDUWB6SS4n7jGegkiIe17fzMbtbTS2W/VXY4wZDAERZpblMak4h/qWzl77ijm4qoRT5lbw5Mp6Hnmndod5xTkh1m2zstmY4UBVm1T1RlU9CNg72/kxxpjhLJ1hFc0wlRMOsmdlAafMHc+dr2xkcfV2CqNh9p5USECsFrMxxuwqEWF6SS5BgbVb2yjJDRMI7Fy+nrH3ZTjKNgAAIABJREFUBDZsb+fPizcwoTDKPpOKABd0yAsHWV7TzN6TigglWdYYs+tE5P40F1HgtEzkxRhjRoNUAgafEJHj/N8BXMF6oYicGJdmyqDnzKSloiDK4TNLeXFdA/ct2cycinxqmqJUFkaznTVjzMBY2TvMiAhTS/IIBgKsqm9hXE6YYMIP/4AIFx80lf99tJ3fPruGbx0/m8nFrl+1nHCQbW2drN/WSlXpToNeGGMGxylpprfORYwxpg+pBAw+l2RasmG8rMDNomBA2K28gNP2quQXT67i0XfqKM4JU5IbJhKylifGjEBW9g5Tk4pzCAaE5bXNFOeEd5ofDQX5/OEz+O4jy7j2qVVcecJsCqPu67Y4GmJDg6uhUJy787LGmF2jqv3e9IjIkcCPgQOAjRnPlDHGjGD9Faq5abzscUmWFeWEWDC5mCNmlrJwZT2rt7ZSnTAuuDFmRLCyd5irLIyyR0U+29uSd4RYlh/h84dXsbW1k9/EjZwgIhRE/n/27jtMsqpM/Pj3vbdyde6Znu7Jw8DkISwgGJCgrmEVUNTVn2KWdV3BLLsqyq5rDpjWvMoaVsVFBHVFEUSJAkuaYYCBYSKTZzqHiu/vj3Oru7qme6Z7uqqru+f9PE893XVTnTpVffre957znhBP7Oslk7MZbYyZTCKyRkR+C9wCLAeuAE6obqmMMWZqO2zAQFVT43lMVqHN6BY0xHnxitnUxUL88uHd7OjsP2RecGPM1GZt7/TQnIyyoqWGvEJ6hOlslzYneesZbuaEH943NHNCJOSRU2Vru82aYMxkEJEFInI18ADwPOCrwFJV/aSq2h+iMcYchvVVn2EiIY8VrbW8dHULW9v7uX97J5sP9o2a1dsYY8zRa0pGiIU9+jI5UtncIevPXNTI+avncNvmg/zh8aGZE+qiIfZ2pzjQa/EeYypFRBpF5AvA48DFwM+BFar6XlU9UN3SGWPM9GABgxlodjLCOcc1sWx2khse2cvOzgEO9qarXSxjjJmRfBHWtNUykM3Tnzk0aHDBmjmcNr+enz+4k4d2dgFuaEJtNMSm/X2kRuidYIw5eiISFZHLgU3A+4DbgFNV9fWquqWqhTPGmGnGAgYzkIiwZFaSC9e2ksrl+cPj+9l8sN/GyxpjTIXUREOsaaslm1f60sODBp4IbztzAQsb43zrzq3s6HA9oMO+h4iw+UDfiHkQjDHjJyJvBZ4EPoULGLxAVV+oqg9Wt2TGGDM9jWWWBDMNJSMhTptfxzlLm7j5iQOctqCelpoIi2wqL2OMqYhkxAUNHt3dQ286SzIy9C/WzZywmE/84Qm+ctsWrnjBCdTFQtRGQxzsS7OvN0xLjU2Da0wZfBc3e8x9wDXASSJy0mG2V1W9alJKZowx05AFDGawtvo4L17Zwv07uvjlw7uZVx9jVjJCMmofuzHGVEI87LOqrYZHd/fQk85SUxQ0aEpEuPSsJXzmlif5+u1b+OC5xxH2PepiYZ7a30ddNEQs7Fex9MbMGIKbMvH0MWyrgAUMjDFmFGMekiAiG0Rk7SjrVonIhvIVy5RDyBNWttRw/uoWdnQOcM+2DrYctK6vxkwn1vZOP7GQz+rWWiK+R9dAZti645oTvPWMBTyxv5cf3rcDVSXkCSFP2HSg1xLUGjNx547zcV51immMMdPDeG41r8DN+T2SBG4+23ETkRuBFwKfVNWPFi1vBD4PXBi87l3Ae1V13dG8zrGqKRnhnOObuWd7J7/dsJc1rTW01kVpTlrXV2OmiYq0vaayIiGPlXNq2Li3h47+DA3x8OC6MxY2sqszxfWP7GFufYwXr2ihJhiasKc7RVtdrIolN2Z6U9U/V7sMxhgzk4w36eFotz5OBDrH++Ii8lrgkHFlIiLAr4EXAZcCFwFh4E8iMn+8r3OsW9yU4MI1rWTzyu8e28dTB/rJWgJEY6aTsra9ZnKEfY/lc2qpj4dp7x8+U835a+Zw+oJ6fvHgLh582n2E9bEwWw720ZvOVqO4xhhjjDGHOGzAQEQuFZGNIrIRd8L6P4XnRY/tuAQzN43nhYMeBFfhprspdT7wbOBiVf2pqt4YLPOAD43ndQzEwj6nLajnvOObuWdbJ4/t6WFn10C1i2WMGUUl214zuUKesGx2kuZEhIN96cEhYZ4Ibz1jIYua4nz7rm1s7+jH94So77Npfx+5vA1NMMYYY0z1HamHwU7g/4KHAI8XPS88bgb+BXjbOF/7s8B6Vf3pCOvOB3aq6p8KC1S1E9fr4IJxvo4B5tRG+bvVLTQnwly3fjdb2wcOmfrLGDNlVLLtNZPM94TjZyeZUxulvT8zGDSIhjwue84SYmGPr/xlM10DGRIRn950jp2d/VUutTHGGGPMEXIYqOq1wLUAbpQAH1HVpyb6oiLyHOANjDAcIbAaWD/C8keAN4hIjar2TLQcxxLfE1a01HLBmjl8/54d3LH5IM2JECvn1BY+W2PMFFGpttdUjyfCcc0JfE94umOAxkQYT4TGRJjLzlrCZ25+kq/dvoUPnbuU+liI7R0DNCQi1NqsNsYYY4ypojGfiajqa0uXiUgtsBB4TFXHdLtaRCLAt4EvqOrjo2zWBGwZYfnB4GcjMCxgICKXAJcALFiwkObmmrEUZ1oJhfwJva9m4L1z6tjUMcDvH9/HO849gcYmdwJrnInWsTkyq+PxKVfba6pPRFjUGMcX2NY+QGM8jOcJS5oSvO2MhXzjzq1cfe8O3nbGAhJhnyf39bJ2bh0ha6ONMcYYUyVjDhiIyIeAWlW9Inj+LOC3QB2wVUSeP8Y7YB/CZfz+5FGUd1Sq+h3gOwAnnXSKHjgw8zogNDfXMNH3lc7mOXtRA3dtOsBHrnuYNz9jASfPq7cT0kA56tgcntXx+JSx7TVTgIiwoDGB73lsPthHQyyM7wmnL2zgwq4BfrV+D/PqY7xkZQsdAxl2dPSzuClR7WIbY4wx5hg1nlkS3gRsL3r+Ody42tfgsnT/25EOICILgY8AVwBREWkQkYZgdeG5D7TjehGUagp+to+j3KZIJORx6sIGzjthFvfv6GL97i52d1oCRGOmsDcxwbbXTD1z62McPytJx0CGbJDg8PzVczhjYQP/89Au7t/RSX00xM7OATr7M1UurTHGGGOOVeMJGCwAngAQkWbgTODDqvoLXG+Bc8ZwjOOAGPBj3EV/4QHwgeD3tbhcBatH2H8VsM3yF0zM7GSEC1a30FIT4VcP72HzwV76M9ar2Zgpqhxtr5mC5tRGWdGSpGsgQyaXR0R4yzMWsLgpznfu3sb2jgFqIiGe2NdLxqbCNcYYY0wVjCdgkGdoCMNzgTRwe/B8L26I/JE8CJw7wgNcEOFc4EngBmCeiJxd2FFE6oCXBevMBIgIJ7TUcOHaVvb0pPnzpna2HuwfzNxtjJlSytH2mimqORllRUsN3aks6WyeSMjjsrOWkAj7fOW2zfRnc+RU2dpusyYYY4wxZvKNJ2CwAXiNiIRxXWT/oqrpYN18YN+RDqCqHap6a+kjWL01eN6DCwrcBfxYRF4jIi8MlgmuO66ZoGQkxPNOaOakubX8YeM+ntjXQ+dAttrFMsYcasJtr5nampIRVrfV0pfJkcrmaIiHueysxfSksnztti3EQx57u1Mc6E1Vu6jGGGOMOcaMJ2DwSeD1QD/wYuDzReteBDxQrkKpah54KXAT8A3gOiAHnKuq2w+3rxm7uXUxLjqxDQ/hNxv2sml/3+BYWmPMlDFpba+pnvpYmDVttQxk8/RncixuSvD2Mxey6UAfV9/7NDURn037+0hlbWiCMcYYYybPeKZV/I2InAicBtyvqo8Wrb6XCZy0quohKfpV9SDwluBhKiDke5w6v54XLJ/Fbzbs5YGnO2mtizKvPlbtohljApVse83UUhMNsaatlg27e+hL5zhtQQMvX5viunW7mVsf5eylzWw+0MfyliQiNrONMcYYYypvzAEDAFV9HJedu3T518pWIjOpmpIRLlzbyn3bO7l+/R5OmJ2gOREmFvarXTRjTMDa3mNHMuKCBo/u7qE3neVlq1rY2TnAtQ/vpq0uxpKmODu7fBrjEeJhzwIHxhhjjKmo8QxJQETmiMinROR2EdkgIquC5e8UkdMqU0RTaUubE7zixFb296b50xMH2WbJtYyZUqztPbbEwz6r2mrwROjN5HjLMxZwXHOC79y1jY7+LNs7+nno6U7u3dbBxn097OtJ0ZvOkrfEtcYYY4wpszEHDERkBbAO+EegD1iOmyKR4Pf3lL10ZlLEwj7PO2EWfzOvjpuf2M+GPT0277cxU4S1vcemWMhndWstEd9jIJvjsucspibq8/Xbt3CgJ0N9PExNJETPQI5N+/t4eGc3927r4PE9PeztTtGbsgCCMcYYYyZuPEMSvgBsBl4I9OCm9iq4A/h0GctlJtmc2iivPrmNDXt6+PUje1jYEOOkefX4nnV3NabKrO09RkVCHivn1LBxbw/dqRzvPmsJn775Sf79j09SE/VZ01rrHm21NMTC5FXpy+RoPzAU8K2Ph2hKhElGQsTDvrXpxhhjjBmX8QQMzgZer6odIlI6wH030Fa+YpnJ5nvCKfMbeOGK2Vy/fg/3bu+ktS7GXEuAaEy1TbjtDaamvRxYBTTipmK8E7hSVTcUbbcAuAp4AW4a2z8C71HVbeV4I2b8wr7H8jm1PLGvF0X5/MtWsn53N+t2dbN+Vzd3b+0AYGFjnLWttaxtq2XprCQhT8irMpDJs/lAPyqKqFAXCwIIURdACFkAwRhjjDGHMa6kh7ipDUfSjJvyy0xjdbEQr1jbyr3bOrjhkT2saKmhKRkmFrIEiMZU2UTb3ibg/3DT1O4DFgL/DNwtImtVdauIJIBbgBTwRkCBfwf+JCInqmrvBN+DOUohT1g2O8mm/bC/N80ZCxs4c1EjeVW2d/QPBg9ufGwvv310L7GQx6rWWtYGvQ9mJSMAqCqpbJ6t7f3kVRGE2qhPYyJCTTREImIBBGOMMcYMN56AwX3AxcBvRlh3EXB3WUpkqmpxU4JXn9TGl2/bwh837mduQ5QTZtVUu1jGHMsm3Paq6k+BnxYvE5F7gMeAVwJfBN4OHAcsV9Ung20eBp4A/gH40tG/BTNRviccPzuJ7wl7ulOEfY9E2GdRY4JFjQleumoOfekcj+7tYd2uLtbv6ub+HZ0AzK2LsqatlrWtdSybnaQ+FgZcACGdy7O9vR/FRYhqoj5NiQg1UZ9k2Cfkjys3sjHGGGNmmPEEDD4J3CgivwZ+gju3eK6I/APwauDcCpTPTLJIyOPcZbO4Y0s7f3pyP38zv46WmujgCaYxZtJVqu09EPzMBj/PB+4uBAsAVHWziNwBXIAFDKrOE+G45gSzaiK092bY15smm8/ji5AI+yQiPqfOr+fU+fWoKru6UoPDF2554gB/eHw/YV9Y0VLDmmD4QmttlGjQi8wFEJQdHf2oui9aMuLTFA9TG3M9EMIWQDDGGGOOKWMOGKjqH0Xk1cCXgb8LFn8J2Am8WlXvqED5TBXMTkb4f6fMZf3ubn61fg+LmuKcPK8ez+b7NmbSlbPtDXIg+MAi4DO4HAiFngergetH2O0R4FVHV3pTbiJCfSxMfSzMwqY4fekcHX0Z9vSk6U6l8TwhHvKJhDzm1rs8NH+7fDapbJ6N+3pYt8sFEH76wE5++gDMSkZY2+aSJ66cU0M87BMNDQUF0tk8O7sGyHdCHiUR9mlKhKmLhUmE3esYY4wxZuYaVw4DVf2liFyHO7Fswd2hWqeq+UoUzlSHiLB2Xh0vWdnCtQ/v5q7N7cyti9FaZwkQjamGMra9fwVODX5/EjhPVfcGz5uA9hH2OYhLlDgiEbkEuARgwYKFNDcfe0OYQiG/qu97UfAzr0ouD9l8nnwwo6Inrk0HeFbRPjva+7lz037u2HSAv24+yJ+ePEDIE05Z0MCzj2/m2UtnsWxOzeC+Bao62PugcHzfE3wRPE8od1i52nU7k1ndGmOMGYvDBgxE5Cng5ar6UGGZqiqwvtIFM9WVjIR45Ymt3LOtg19v2Mua1loaE5Fhd56MMZVRwbb3YqAOl6vgA8BNIvIcVd1ytAdU1e8A3wE46aRT9MCBngkWcfppbq5hqr3vgUyOzoEs+3tSdKWygBALecRC3mAQYGldlKWnzOX/ndjKk/v7WLfb5T748s1P8uWbn6Q+FnK9D9pqWT2nlprooacMmVyegWyebD4/+BqN8TD18TDxiDfhpLlTsW5nCqtbY4wxY3GkHgaLgegklMNMQfMb4vz9yXP54q1PcePj+5jfGGfprGS1i2XMsWAxFWh7VfXR4Ne/isjvgC242RLegetdMFJPgtF6HpgpLBb2iYV95tRGSWfzdA24nAcd/VkUJep7xMM+nggh32PFnBpWzKnhVSdBR3+G9bvdzAsPPN3F7ZvbEYElTYnBqRuXNCXwPCHse8PyGmRzefb3ptndnUKBiO/RmAjTEA+RCIY7lPZaMMYYY8zUNd5pFc0xJOR7nHt8M3dsPshfnjrIaQvqaYiHqYmGrKeBMdOcqnaIyJPA8cGiR3BDHkqtAjZMWsFM2UVCHrNqosyqiZLN5elO5TjQm2J/XwZVCHtCPOzjB1MqNsTDPGdJE89Z0kQ+r2xu7xucuvGGDXu4/pE9JCM+q4umbmyIu8S4Id+jpjiAkFcO9qbZ051CAN8XmuJhGuNh4hF/WI8HY4wxxkw9YwkY6JE3MTNVUzLC606dx8O7uvnVuj201kXxxcP3hbpoiIZ4iHg4RDzsWfZsY8qrom2viMwBVuBmXgC4AfiCiBynqk8F2ywGno3rhWBmgJDv0Zhwd/2X5JWeVJYDfRkOlMy4UJhO0fOEpc1JljYnuXBNKz2pLI/sccGDdbu6uWdbBwALGmKsDaZuPH5WYnD/kCfDhjJk80pHf4a9PalgvUdjPERDPEwi4v6XWADBGGOMmTrGEjD4VxHZP4btVFXfONECmalndWstf7eqhWse3MWHf7uRhY0xFjbEmd8Qo7U2yuyaKL4H0ZBPfSxEXSxEPOgOG/LsxM+Yo1S2tjdImHg/8DDQBSwD3oubUvGLwWbfBd4FXC8iH8UFLD4BbAe+fVTvwExpvifUB/kGFjfF6U3n6OhLs7cnQ3c6gyeQCA+fSrEmGuKMhY2csbARVWVH58DgzAu/f3w///voPmIhj5VzhqZunF0zNLom5AmhSIjC4LZcXukayLGvN+3KJEJDwvVASER8miezQowxxhhziLEEDE4GUmPYznoizFCxsM+rT2ojHvJ4uivF1oN93L65nXTOJWgP+8KChjgLG2LMq4/RVhdjTm0E3/OIh71gCrAQ0bBLgOVbEMGYsShn23s38Grg/UAEFwS4Ffh0IeGhqvaKyHnAVcCPAAFuBt6jqpYZbYbzRKiNhqiNhpjfoPRlcnT0Z9nbnaInlUZEiIc9okVJDEVc27+gIc5LVrbQn8nx2N6hqRsfeLoLgNbaKGva3PCF5S01w4a0+Z6QiPgkcMfN55WegRwHetMIQnNzLY/v6aExESYZ8YlHfJvi1xhjjJlEYwkYXKiq91S8JGZKa62LccbiRnI5Nw83wO7uFFva+9na3sfWg/3cvbWDgawLIoQ8YX6D64kwty5KW32UttoYYd8jGXU9EWqjYWJhz8awGjOysrW9qvpZ4LNj2G4bcFE5XtNMXyJCMhIiGQkxrz5GfyZH10CGvT1pOvozKEosdGj+gXjY55R59Zwyrx5VZU93mnW7u1i3q5u/bDrAHzfuJ+QJy1uSrG2rY21rLW110WHH8EoCCL4IfZkc7Qcyg9vUx0M0JcIkI6FhuReMMcYYU36W9NCMie8Jy2Yl2dbRT89AlqxCIuKzak4NJ8+tI+K7Obj3dqfZ2t7HlvZ+thzs597tHfRn8oPHmFcXZUFjnLl1rjdCa12EWMinLhqiLh6mJuITDXtEfQsiGGPMVBAP+8TDPnNqY6SyeboLwYOBLKgSDfnEwt6wO/8iQmtdlNa62bxg2WzS2Twb9/Wyfnc363Z18bMHdvIzoDkRdr0P2upY2VJDIlIyDaMMvT5AXpWBTJ7NB/pRUUSFulgQQIi6mRgsgGCMMcaUjwUMzJgloyFWzqlFVUnnlFQ2z0AmS3cqR9dAloF0jmhYWNaSZHVrrZtuy4P9vRm2tvezJeiJ8ODTXdyx2c3S5gnMrYuxoCHG3PoY8+qjzKuLE4/41MVC1MdCg4mwLKmiMcZUVzTkEQ1mXMjk8nSnsuzvTXOwaMaFRNjHK7loj4Q81rS5GRVec8pcDvSmg+CBS5z4500H8QSOn5UcnHlhYWP8kNf3RIYFEFTd/6Kt7f3kVRGE2qhPYyJCTTREImK5dIwxxpiJsICBGTcRIRoSoiGPuliIllq3PJtXUpmcuwOVytKdytKVyhEOCcfPTrCsJUnE9wh7Qkd/1gUQ2vvZ2t7P+t093LW1Izg+tNVGmd8QdwGEetcboTbq5vKuDU4CLamiMcZUT9j3aEpEaEpEyOaV3mDGhf29aXL5PL54o16wNycjnL20mbOXNpPNK0/t72VdEEC4dt1url23m7poiOcu38fcRJjVrbXUxQ49ZRERYkGSXSAIaOfZ3t6P4hJ81ER9mhIRaqI+yaIZIIwxxhhzZIcNGKiq/Vc1YxbyhFA0RDLqpmME1300nc0zkM3Tm87RPZChO5Uj5AtLZyU5fpYLIoQ86Enn2HKwPwgi9PH43p7BKbsEmFMbZX5DjHl1MeY1RJlXH6cpEaY+FqYuFgryIVh3VDP9WdtrpptQyYwLPakc7f1p9vakyeYUPxhaMFJPsZAnLGupYVlLDRed2EbnQIZHdnWzbnc3dzy5n/a+DAIsboqztq2ONW21HNeUGLGtdwFtfzA5Y6FH3I6OflRBFZJRn6ZEmNpY6JBZIIwxxhgznPUwMBXlFd39aYiHoT4GMBhEGMjkBnsjiAjHNSdY0hwn5M0i4nv0prNsax8YzIuwaX8f923vHDx+S02E+fVuOMOChhjz6+PMqonQEA9REw2md7SkisYYM2k8cXkF6mIhFjbE6Uvn6BgIZlxIu9kP4mF/2GwJxepjYZ61pIlnLWnilBNm88u/bmXdrm7W7+ri1xv2cMMje0iEfVa11gwOX2hKREY8VnGPuIJ0Ns/OzgHyHZDHJfJtDpIo+p4Q8mTYT/v/YYwx5lhmAQNTFZGQR2RwSIObozubV1LZHKlMnp50lq6BLL4nLGqKsbApxjkiRHyP/kyOpzsH2HKw383ScLCP+4PpuwBmJcPMr48zrz7G/IYoCxritNXFqLekisYYM6lEhGQ0RDLqZlzoSw/NuNDenwGUeMgFD0Zqkz0RljQlWNKU4PzVc+hNZ9mwpycIIHQPBpDn1cdYG0zdeMLs5GF7DRT+/xRkcnl2daXJ6cDwsiOAEvZ9wr4LOsRCLp9ONOQNBhWKAwz2f8UYY8xMYwEDM2WEPCEUCZGMjDykoXCimVVlfkOMBQ1xzgLCvpDK5tnVlWLLQZcXYcvBfh7cORREaEqEmR/kQpjfEGNRQ5z5DXHq4y6pYqzkBNIYY0z5JSI+iYhPa12MgWyO7oEs+4IZFwSI+N4hMy4US0ZCnL6ggdMXNKCqPN05wLrdLnjwx437ufGxfUR8j5VzagYDCIWg9GjC/uhJdVWVvEIur/SlXXlzqm54AzpsWwF83wWkI35haIT731IcVCj8Ptp7NMYYY6YSCxiYKa10SMPcoiENqVye/nSO7nSW7oHsYHLEZy1xwQd31yjFtiCx4pb2Ph7e1T147Ia4u+M1vz7O/IYoS5qSXHB6gr3dKZdUMeRZcixjjKmQWMgnVuMzuyZKOpunJ+WCB+39GVTVXcTr6PuLCPOD4O+LV7QwkMnx2N5e1u3uYv2ubh4KgsYtNREXPGirY0VLcjC/wViICL4w5tw4ubySV6U/k6cnlSMfBBwKFDeTg6KEPI9ISIj6HuGQy8ET8YWQ5xHyBV+GgguWm8cYY0y1WMDATEuFLqW10RAtuLtHubwykHWzNPSkXBDB94S5dVHOXNyAh5BTZXdXiu2dA4O9ETbs7hk8J/3ybZuZnXR5EebXxzh+doJFjQka4mFLqmiMMRUSCXk0hSI0Jd2MCz2pLAd60+RVae/PEAqmazxc+xsL+5w8r46T59UBsKc7NTh1421PtXPzEwdcgsXZSdYEvQ/m1cfKOozA9wQfITyGmEQ+r+RUGci4pMC5fHpYcAEgGBWBLxAOBb0Xwh5RL+i5UAgwlORdMMYYY8rFAgZmxvA9IVkY0hAkwFJVUrk8qeCErDuVJRzyaKuP8oyF9QhCXvPs7k6zo2OAnpxy/9Z2/rh3PxqcuNVEfOY3FHoixDhhdpKlzXFqY+HBxF3WtdQYY8oj5AkN8TAN8TCJiM/q1loO9qbZ15smm8/jiYxpdoM5tVHm1EZ53gmzyOTyPLFvaOrGax7cxTXsojEeZk1bLStbaobNthMLeYNJc70KXYB7nuCNNbigSi6vpLJ5+jN5skFPBsXFFIqJQDTIteByLshgjzkLKhhjzOGd8tiWahehLB5Ysbhsx7KAgZnRRCQ4+fOpj4cHl2dyQ7M0dKWyJKMh5tbFOGftXP60bie5nLKvN8XTnQPBNI/9/OnJ/eSCIEIiXAgixFjQGGN5Sw0nzEpSE3UnnJZU0RhjyqMw48Kipji96RwdfRn29qbpSafxRIiH/CPmoAn7Hqtaa1nVWsvfnwwH+9KsD6ZuvG97B7c9dXDUfQvJDmNhj3jId0GF8PCgwrDnxdsVLZtIDzVPBM8fe3Ahn1eyuTydmTy5INhQGA5R0NBYw91b212vhVAh74JHxHcP3xdCIkM/LamjMRUzEy5Sy3mBaqYWCxiYY1IhyVVtNMTsmqEhDfGwz/KWJL2pLI1JlzPhtCC5Vi6v7OtN83Rnih0dboaGPz91kFzQhzQe8pgXBBEWNcZZ3pIVT0KNAAAgAElEQVRkeUsNyWhoTCe0xhhjRici1ETdlLnzGmL0Z/J0DmTY050aNuNCbAxX1U2JCM9d2sxzlzaTyyu7ugboy+QYyOTpz7qfw3/PMZDN0x/8PNibpj+TH1yWPWQswcgifhDEDoIO8bEGGwrbFO0bGiX4UAguHOkEz/eE+miIXF7J5pRUJk+nZl0ehsJAPXGzRai6ngthzxucpjISzBYR9kdO6mjBBWOMmRksYGBMwGWtdieSIw1p6M/kmF0bZWFjglSuDoGgJ0KaXV0ptne4ngi3b27n1k3ublU05AWJFWMsbkqwck6S5bNrqImFiFtSRWOMOSoiMjjjQltdzPUWG8iyrydFe38acN3wY6NM11jM91zyxInI5tUFFYqDDMHP/pJgQ/GygUyO9v7M4L4D2Rzp3NiCD+FC8CHkES8EIIqGVMQKAYjBXhDFwQifOe399KRzxEPuwv9IVJVcMGNETypHLu9mjMjroeUVGJyOMhHxWNyUHDXAYYwxZmqzgIExh1E6pKHV5dIik8sHY0lzzEvlOG5Whv50PeBOHPf1pNnVNTA4pOHure38JejyGvGFucHsDMc1x1k5p4ZVc2qJhr1h3UWLz3EPGaM6ypMx7T/a9sPe92j7jm97u8NkjJkMhdl0WmrdjAvdqQz7ezIc7M+gKFHfXThXKt9MyCv0fpj4sQoJfPtH6NnQPxhYKFkXBBs6+rMMZFPBvnnSufzoL/T7jcPKXzp8orSXQ3GwIT5CD4lCwKIQGChMR7m3J8OCBrWAgTHGTFMWMDDmKBSGNNREQ8yuccsKCakGsjn6Ujk6U25oQ05dNux9PSl2daV4umuAbe0D3Lu9g9s3uyCCLxDyPTwJupMO/nS/S9EyEfBHWF547gfbeCJ4HniULgfP8/AoOp7nQgeeV3Q8QLzicgwvg1fy2l7x8UvL6w29t9OPn82DT+3HC95HYX3htfzg4RVNZyYCvi+D70VgxMCHyNDi4mCF4E6IlzQnjpgozRgzvUVCHs2hKM3JKNlcnu5UjgO9KQ70ZcgrhD0hfoQZF6qpOIHvRA0lSswdEmRoba7h0R3tg8GG/pJhGN2pLPt63D79WRckH2v5C7074mGfmqjPqa86ceJvxhhjTFVYwMCYMvG9oS6yTQmYT9GQhmye/nSC7lSWroEc6VyevOaDnggp9nanB8eN5vO4n0GgIa9BEisd4feS9bm8ktY8eWUw78LQ+sPvW7xdRf1584R2F4aCCCMFV4b9HAyAQMjz+MFrTxpT0jBjzMwQ8j0aEx6NiTBL8kpPOkt7b2ZwxgU/mHFhpg4PK/6/VOrUZS3MT4z9NDCfVxd0yA4felE81KIQXBgoClBU/H+KMcaYirKAgTEVNGxIQyxMa7A8G8zSkMrm6RrI0p3KksrkyAOKu9iHINnU4NGGZ7hWhrYBUFFQRt1GgyXFzyV4MHhnXga3yauiCqqua6kGQQxVDX6X4Hd3Iqmq5JFgv6Ft8xSOoeTycNzcOjbu6BgeyMiPEhAJxsxqENzIjTFwMtKxxzgs2BgzQ/meUB8LUx8Ls3BwxoU0e3sydKfSiBC0i0PtI+LaQWF4b4Th7aZ7IsOeF3o8HTpMTIY9l5LnRccIelMV96gSqjfUyztM8OFwOvozFSqRMcaYyWABA2OqIOR71PgeNVFoHqXfaeFiXDW4hC95rrhfhpYVPR9p2YjbDF3c5wvLg54OhUBBPtghXxQEUCUIBOhgMKE4uJB3O4wYAHnuCbPJDbiM5pMRACmcxKcON5bXGHNM8USojYaojYaY36D0ZXJkc4UWZnh7CaVt5lCbV2g3GWwXdaiNLmo/BwOrw44xdMzBdrPw2oMB16BMwb75wfKVtJeD7yxoV4div8F0ikPBkGBHcnmlvT99SDDkkLa4cFxhMHgyGFDh0DYXhg8Py42QFNEYY8z0YQEDY6aoQ8fqT83xtmNRHPxIRnxOX1A/6cEPESx/gTHmECIuZ8B0UggYDAsquxWDywbXF7WVMNR+xsMeJ7bVDWtn4dC2tnTZ0OvqUC8zHSqHFgIjQdvrhoRN3/9fxhhzrJte/yGNMdNSafBjpo4XNsaYyVAYllCaAHY8gWVPhGTUTgONMcYcnp21G2OMMcYYUyUi4onIe0XkMREZEJHtIvJFEUlWu2zGGGMBA2OMMcYYY6rnKuBLwAbgUuAXwGXAr0XEztWNMVVlfdGMMcYYY4ypAhFZjQsS/FJVLypavhn4KvAa4L+rVDxjjLEeBsYYY4wxxlTJa3HJJ75csvy7QB/w+kkvkTHGFLGAgTHGGGOMMdVxOm7GzHuKF6rqAPBgsN4YY6pGdAbOjysi+4Cto6yuBzrHcbgjbX+49aOtG2n5WJbNAvYfpizlNN56Otp9x7Kt1fHE9j1W63gi9VvYv0FVZ5epPDNe0PZ2MPF6H8v+1fxeT8fvdLXbjJncXkxW3R5LbfGiY6XtFZF1QIuqzhlh3TXAq4CoqqZL1l0CXBI8XQ48XumyHoXJPKcyh2efxdQyVT+PkdvewXl0j5EH8J1ybn+49aOtG2n5WJYB903VejrafceyrdWx1fFk12859j9WH5NV79X8Xk/H73S124yZ3F5MVt0eq23xTH8Am4Bto6z7IaC44HXVy3oU723SzqnsYZ/FdHpMt8/jWByS8Osyb3+49aOtG2n5WJdNlom89nj2Hcu2VscT2/dYreOJvm41vxvT2WTVezW/19PxO13tNmMmtxeTVbfHals80/UB0VHWxYq2McaYqpiRQxJmKhG5T1VPq3Y5ZjKr48qzOjYzjX2nK8fqtnKsbqcGEfk98HwgoaqpknV3AMt0mg7PsO/Y1GGfxdQy3T6PY7GHwXT2nWoX4BhgdVx5VsdmprHvdOVY3VaO1e3UcC/ufPwZxQtFJAacDNxXjUKViX3Hpg77LKaWafV5WA8DY4wxxhhjqkBE1gIPAdep6kVFyy8FvgpcrKo/rlb5jDHGAgbGGGOMMcZUiYh8DXgXcB3wv8BK4DLgDuA8Vc1XsXjGmGOcBQyMMcYYY4ypEhHxgffgpklcjJtu7efAx1S1p4pFM8YYy2EwU4hIo4j8RkQ2ishDIvIHETm+2uWaSUTk5yLysIg8ICL3iMjzql2mmUpE3iwiKiIXVrssxpSbiFwRtNV5+44fPRFZKiK3B3X5gIhMmwRSU519RyeXquZU9YuqulxVo6o6T1XfZ8ECY8xUYAGDmUOBL6vqMlU9CfgN8L0ql2mm+QdVPVFVTwH+AfiFiNjfUJmJyGLg7cDd1S2JMRVzE/Ai4C/VLsg09y3gv1R1GfAh4CciIlUu00xh31FjjDGABQyqSkTmi8jXROQuEekL7qguHmXbBSLyPyLSKSJdIvJLEVlYWK+qHar6x6Jd7sR1aztmlbN+wdVx0dP6ChZ92ih3HQcBmO8BlwKpkY5jTDmV+zs8Fqp6t6o+NdGyTzflrGsRmQ2cCVwNoKo3AQKcWvE3MgVV4P/dMfkdNdNTMKTDTAHi2OcxyQp1XqkbmRYwqK7jgVcD7cBto20kIgngFmAF8EbgYuAE4E8ikhxlt/cA15e1tNNP2etXRK4SkaeAa4GLLBFR2ev4fcAdqvp/FSuxMcNVsh02w5WzrhcCu1Q1U7TrlmD5sci+x+aYUnxRqqq5YNna6pXo2CYiDQDq5ERkhYg8q9rlmslEpCn4KUGdLwPeXom2PFTuA5px+YuqzgEQkbcBfzvKdm8HjgOWq+qTwfYPA0/gusZ/qXhjEfl4sP0lFSr3dFH2+lXV9wLvFZEXAZ8TkWerarqC72GqK1sdi8ga4CLguRUvtTFDytpOiMgfcXOnj+QCVb2jjGWfbiryP88AVrfmGCIiYeC1IlKnql8Plt0LxEXkLFVtr24Jjy0iMhf4RxEZUNVPishxwAbg5yLygKr2V7mIM46INAMfFpGMqv6ziKwC1gM/AK4Besv5etbDoIrGcXf6fODuwj/3YN/NuOl2LijeUEQ+CrwEeLGq9pWrrNNRJeq3aP2NQCNwTEezy1zHZ+GG0TwhIltw3Y2/IyLvKluBjSlR7nZCVZ+vqrNGeRzLwYJy1/U2oC24cChYHCw/5lTy/50xU1AUd657lYh8TERuAlqBfwMsUeTk6weeDVwpIt8EHgRuAD5nwYKKEdzfwYdE5IfAXcDvgK9UImBmAYPpYTUualTqEWBV4UnQs+BlwN+qaucklW0mOGL9ikhcRJYUVojIM4FmwMZ4js0R61hVv6mqbaq6WFUX45IeXlK4e2BMlY2pHTZlMZb2Yh9wD/AmABF5Ae4EyoYzHZ59j820F8we8U+4C9N/xt1geKeqXlMyTMlUWNAdvh14MfA48GZgD/AvqvpAVQs3g6nqfuBjwK+A1wMHgHer6sPgPpdyvp4FDKaHJty4xFIHcXe5EZHVwJW4i9g/i8iDInLfpJVwejti/QJx4L9FZL2IPAh8AZfDwLq9jc1Y6tiYqaxs32ERuVJEdgDPBL4nIjtEZH4ZyjhTjLWu3wG8WUQ2Ap8HXqeqOgnlm87GVLf2HTVTmYiEVPUAEAsegvuuFtZb0r1JoqoqIhEggws6ZnC9PV5b2MY+j4rpx+Wg2YnrYVexHrmWw2CGUNVHcA2mqQBVPUjRPyNTeap6TrXLYEwlqOqVuACvmQBVfQKwpFoVYN9RM5Wpajb49du4acXfArxLRHKqekWQAC5UtJ2pkKCHQTr4/SJgAPgE8J5gnX0eFaKq/SLyEaAPF6C5LKjzdweBHL8oIahMJKBuAYPpoZ2R72CNdqfAjI/Vb+VZHZvpzr7Dk8fqunKsbs20JCJeaa6OooSHv8KNmX93cF30scLFqYjUA7WqumPSCz2DFT6P4otQVb0uWPcIcB0ln0cw5d8SIKqqG6pT8ulrlL+BG4J123A3ji8VEYKgQSFYsARYKSJ3lkwRP2YWMJgeHsGNOyy1CpeF1EyM1W/lWR2b6c6+w5PH6rpyrG7NtFN8dzrIIdUKRIDfAilVfVpEXoXLDv/u4ILpY8H0cu8G3ikip6vq9mq9h5mkcOc6mB3hebiZV3qBHwOdqrqt5PNQVf04sBT4FnCciKxR1bJm8p/Jiup8AfBy4CRgF3AncKOqbhSRz+N63FwqIjngcmAl8FlgHnDe0b6+5TCYHm4AzgymKQFARBbjMpLeUKUyzSRWv5VndWymO/sOTx6r68qxujXTSnChVAgW/BR3EXot7uL0Plwek9mq+hTwSmAj8H4R+S3wFeDDwH9asKA8grvcuSB32m3A54D3Ap/EZep/h4jMKvo8HgcuD/Kq/Rh3oftKCxaM3Qh1fjlwLi7x5/dxeWY8VX0U+Eyw7D24APE1uKSgbwoSJR5dGSw/UHWJyCuDX5+HS+D0TmAfsE9V/xxskwQewiW3+CguevQJoBY4McgWa0Zg9Vt5VsdmurPv8OSxuq4cq1szkwXDDs4ArsJdNHnAD3FJDy8GblPVjIgsBL4KnIZLwHeVqn41OMYhXbrN+AVd3G/HXZB+FTct6zzc7FaP4maveazojvgngWW4KS8vs+EI4ycii3Df+8dxQYE/AQ3B8ywuCHNHsO1i3DS5L8MltP2Yqj42ode3gEF1ichoH8Cfi5O+BQ3gVUBh6qibgfeo6pZKl3E6s/qtPKtjM93Zd3jyWF1XjtWtmalE5HXAl3BTKF6rql0icg7wv8B/4y6IdhaGLohIHZAEEqq6KTiGBQvKIJiu75PARcDbcYEaFZFP43oavAv4mar2FHWjj+MuaqMWlByfQrJCEfko8EbgbUUB4CuAK4BLcH8X3cX74dr3mKr2TbgcFjAwxhhjjDHGTEUi8kXgQuCk4EL0POA3uKEJl6vqzmC7elXtHGH/CWWIP9YV11+QuPDPQI+qvjhY9jlcF/h3Aj9V1d4gSJBQ1QNW/+NXPMNB8Px3uPo8O3j+WVyA5p+An6hqX1Dnz1TVW8pdHsthYIwxxhhjjKkaEYmKyGoRmV2yXIAW4EAQLHgeQ8GCDxYFCz4IXCMikdJj28Xq+InIEhF5noi0ldRfApc0f3+w3RcYfuFayE3wS+BrQc8Oq/8xEJHlIvI6EYkWBwsCWdwQMoLeHO/D1fmPi3oQfAu4JOhhU1YWMDDGGGOMMcZUhYg0A18HfoK74Bm86A8uNnfipoV7I3A97mL0Q6q6O9h/BfBcXG6O5CQXf8YRkZcCPwK+Azw/6FUAQDCk4CngeSLyTeBShoIF/cH+LwZmA08C/iQXf1oSkfOBH+ASFr5rhE22ACeKyLXA+3HDEH5WVOfnAWuBbcBAuctnAQNjjDHGGGPMpBOROcAtuEzu9wL/oarpYJ0Em12NS972A+AmXM6NXcE2bcAbgFOBH6lq+6S+gRkmyBfxQ6Ad+JSq/qiQ+6Ho8/gOLqHkPwBXAv9VuMstIn+DG54Qwc1OkZncdzD9iMjrccGyfcDrVfWLResKdf5poAs3peL3gF8UchaIyDNw+T2SwDcLfz/lFCr3AY0xxhhjjDHmcIKeBX/BXZx+AjeffK6QvLCoK/tW4D9xd7OXAc8QkXuA1cCrcXdbL1fVa4Pj2pj5oyAiL8MFA67GBW42BMtLP4/7gf8CLgP+H7BXRB4Ano/LzL8aeK6qbp3ktzDtBL05/gMXDPsPVX08WB5W1UyhzoOknu/B9cS5EOgVkeuAF+HqfTlwtqpurkg57e/JGGOMMcYYM1lEJAx8A3gG7o70rUE2+GEX+yISUdW0iNTjpgv9R2AhwXhuYDvwJZs6cWJEpBX4BS43wQcKs0uUbBMGkqraISK1wFtxvQyWB5t0AQ8A/2RTJx5e0HOgCdezIISrs8dH2K4VGFDVjuD5c3FBgzXBJh246Szfr6qPVqy8FjAwxhhjjDHGTBYRSQAPAXcBbyrp9l4HvAV4JtCI613wK1w3+DbcHPP1wBPARlV9KNjXggVHSURW4T6P96vqV4um86sBjgc+BizCfTafx42f7xKRJHAubgjC48DThYtbc3gishh4ENez4CNFy+txAYErcHkJ0rgknz9Q1UeC2RCW4fJEPAZ0qWpXJctqQxKMMcYYY4wxk2kxsBR3EVQIFvjAa3B3rs/BBQjCwNnAVcCVqroD14V7mOAC14IFR28+LkHhfnDJJkXkeFzg5i24mSq2Awtwd7gXisjHg1kRflOdIk97S3ABmExh2EeQwPPtwMVAA7AZ9zfwPuB4EXmfqj6FC+5MGkt6OAoReZOIaNEjLSKbRORTIhKrdvkOJyjvlUex3znBvueUv1RHfG0VkX8/yn1XiMgtItIVHOfCcpevUkTkQhF53wjLq/ZZBK9/g4h8fYTyZEVk2Qjb7xCRq0c51uuCfR8YYd3tJX9noz3mj6HMS0RkQETWHGlbM3VZ2zu5rO09ZLm1vdb2mslxENgEXC4i5wffn//C9SZYBXwKN/PB63Bj5j8InDHawSxnwYTtwuWS+I/g//CbgV/jkuk9BLwW97lcCGzAfR5rq1TWmeJ+3HCCdwEfFZEPAb/DTVN5Hy7B4XOAF+CCMs8DWqtRUOthcGSvAnYAtbgP7l+C3y+tZqGO4Jm4Mo/X/cG+023c0ZeA43CJbzpwXaKmiwtxyUq+VLK8ap9FMD7qb3GR/1I+8G+4OwBj9cbg58kislZV1xWtuwQXXS24EjgJ97dWbO+RXkRVN4vIT4DPAn83jvKZqcna3qnP2t4ysrbXHEtUdbeIfAP4HG64QcFduB4GT6pqFviruPHyz8B1fb816E1gAYIyUtV1IvJJXPLJ7weL24HPAB8NtskDN4jIKcCJuG7xD1ahuDNFF/BVXPDlY8Gyp4FPqOrHi7bbJyK/Bl4K/A1w56SWEgsYjMWDqvpk8PtNInIC8BYRefdU7fqkqncf5X5duEjXdLMS+Iuq3liOg4lIVFVT5TjW0aryZ/FB4Neq+vQI6/4AvFpEPl0YM3g4IjIPFxH9HfBi3AnsBwrrS5PiiMh+IHW032Hg27iTi1WWcGfas7Z36rO2t7ys7TXHBAlyDajqVSKyA3cRVI+bMeEGVe0TEa8oMNAB5IGHwXoTlFvR5/ElEXkQOAHoBzao6n3BNn7RLo24oQuPTX5pZ4bCd1tEfg7cA5yF+55vKcrJ4atqLthlES6A89dqlNeGJIzf/UACmFW8MOiS9xMR2SciKRF5UEReXrLNlUEXvxUi8nsR6RWRbUG3H0TkYhF5TER6RORPIrK0ZP/XBN0/9wXbPCAib6REabfYotc9QUR+G+y7VUQ+JiJe0XaHdMUUkVuDrovPF5H7RaRPRNaXvrdg29cG5R8QkXVBF7NbReTWcdZxcbfkM4N67RKRnSLyVQm6JRfKixsHd3GhC2XRMV4kIneJSL+IdIrIr0RkecnrFN7fy4L6TAHvLKrHfxeR9wf11RfUX0vwuCY47nYRubzkuLNF5NsisjHYb7uI/HdwElfY5mrcSdw8Ger+ueUwn4WIyHtF5HFx3bR3icjXRaSu5LUL5b5MRDaLSLeI/FlEVo+h3ufiTi7/e5RNvo7rtjbWLswX49qZjwN3AK8r+aczZiKyQER+GrzvlIg8LSLXi0sOA4Cq3gM8BbztaF7DTGnW9lrba22vtb1mBlDVfKENVNVfqOq/qOo7VfVnQbAgFFzAavC3+wrc8IWNVS34DFXyedyiqt9W1R8WBQvChQtXETkdl1/ir8C2apV5uiueEURVn1LV/1LV64uCBcV1fhru/8ODuHZ20lnAYPwWA53AgcICEVmA+8M5CTfu5Hzcye21InL+CMf4BfBbXJfI/wO+LyKfwk0V88/Am3FTlJSeOBwH/A9uPNeFuLFF3xORd4yx7NcBtwT7/gr4V4a6LB7OUuAruK6br8CdtPxCXDIUAETkBbipQR4LtvkC8GVcd6WJ+BHun8QrgG8C/4TrmgxDXUf3Af8b/P7MoDwvwtVxD/D3uLpdA9xefOIYWIbrEvQ14IXAzUXrLgbOw53IvgsXAfwhri4fBi4KXvszIvKSov2agIGgrC/C3Tk6AbhDhsZhfyLYd19R2Q+5GCjySdxncBNuntvPAW8Cflt88RF4Pa5r6Ltx36eFwPUicqReRS/AdX29bZT1/bgT1peKyJlHOBa479ejqnovrt5acV1uj8Y1uPFy7w3K+T5c3UVLtrsNV+dmZlmMtb3W9lrba22vmRFG6ikWBOe8YChCoafMW3Ht0FUlw2pMGY3Wc0/cXe5M8PszcP+/5gMftNkQJma0njLB30Chzp+Jq/NFwLtU9cBI+1ScqtpjhAfuZEBxJ48hXPebtwDZ4AMr3vY/cf88m0uW34TrVlt4fmVwzDcULWsMjnkAqCtaflmw7aJRyucF5fou8FDJOsVlki193TeXbLcO+EPR83OC7c4pWnYrLkvtCUXLWoAc8OGiZXcC6wmm6gyWnRoc79Yx1LcC/z5C/f9ryXa/wU2hU7xsB3B1ybL7cNPthIqWLQney5dK3l8eOHmUMm0sOcaXguUfLVoWwo3z/MFh3p+PyyyrwMuLll8N7Bhh+2GfBe4kODXC+3x9sN35JeV+AggXLXtlsPxZR/gcvombEme08jwfl611E3DzET6DZwT7/EvwvAF30vuzw7z+j3HdsUaqvxzwljF8l94dvG79kba1x9R7YG1vYdmtWNtrba+1vfaY5o/iv4fguTfG/c7DDXXpBP65aLmUq2z2GNPnIMHPDwJ/xvUqOLHa5ZouD0AA/yj2i+F6iN0ObKl2nVsPgyN7DHeicxB3cvptVf16yTYvwt2t6BSRUOEB/B44qbTbIm5MIQCq2o474blbh8+hWRgXtKCwQFy31p+KyNNBmTK47n/Dunoexm9Lnq/H3f04kidU9YmiMu8NyrwwKJcPnAZcq8G3PNju/3DTgUxEaZnXHanM4uaE/Rvg5xpEqYPybMZ1zTy7ZJctqjpa0pabio/B0Ofy+6LjZoEnKfqsgnL8o4g8JCI9uAuTQtetsX5exc7EzXH745LlPwuOXfqebtIgOhkoROWP9HnPxV2AjSo47pXAeSLy/MNs+kbcBcGPg/06gOuBC4q7so6Fum5Z9+OyyP6TuPmCR1Mof9t4XsNMOdb2WttrbW8Ra3vNdCEivri54gvfW0TkFcHzw+agEZE2Efk2rjfRM4EPqOpngnVecVtnJsVKEbkPuBwXvHmBqj5c5TJNeSIyR0SS6uREZK2IvEtEImPYdwVuxpAP4HoWVr3OLWBwZC8HTgdeAvwReKeIvKFkmxbgDQydSBYenw/WN5ds317yPD3KMnARJkSkBnfX7CRc19mzgnJ9n0O7BY7mYMnzVOH449yvdN9ZuDsfI2VT3jPGso31tVMc+f024iJ6u0ZYtxt3x6jYSNsVjPa5jLR8sC5F5FLgG7jvzCtwd3wK3UiPZmq4QpmHlTU4YT7Aoe9ppHoby2vHirY9nJ8AjzDKeNqgQXwNLttxt4g0iEgD7gQghsuqPl4X4OrzCuARcWOTPyQiUrJdf/AzfhSvYaYOa3ut7S1mba9jba+Z0oKg7ZuAfywEqETkZuArIrLkSPur6i7cLCWfBt6kqt8NjuEdKdhgRjfC8KnC8tK/42HUJTH9Cu4zfZOqTqfZcKpCRJpxvRWvDp7Px+UfeClQM4ZDbMT1YnwJ8PbiGwfVYrMkHNl6DTJ1i8gtuLGTnxeRa1W1N9jmAG7s3mdHOcbOMpTjmbjxK2ep6u2FhWMYFzkZ9uNO0ltGWDeHyU+K0o7rFjnSXKWtHHpCV4lo9Wtw3UbfX1gwln+Uh1EocyvuZLFwzBDuomikC4ujcQDXffiw1CXIuQL4pYhcMMImL8OdSD+bQ0/wwd0B++54CqaqO3FTgV0iLonYW3F/c7tw460LCifw+8dzfDPlWNt7ZNb2HsraXmt7TXXlcflJPgg0i8hZwPG4fCaHbZNFBpPAfUVc4sNs0XILFhylIA9BTkQW4nrmNeJmXtmiqu2jBWNkaPaEH5WuM4d1EBc4vYOJQ2gAABSkSURBVEhE/gKcguvl9W+qetj/GUXf9T9WvphjZz0MxkHddE8fxJ2cvbNo1Y24+UgfUdX7RniUY5qoRPBzsKujiDTiIv9VFXRZvA/3hzEYqRSRUxnDCVAFytOLS2j2KinKCi0ii4Bn4cbOVlqCos8q8OYRtksxtrsxd+PupJXOwf33uMDfreMs32geAxaM5WJIVa8D7sUlECuNUL8R6MWNuz235HE18GwpyUQ/Hqr6CPB+XP2tKVm9BOimPBeLZgqwtndk1vaOyNpea3tNFQUXmJfjhsR8AHex9BF1GeBTh7ujXTK0KjvScjM+wQVoLgj2/RX4Fq73xm3Ap0VkvhbNklDMgjTjVxT0eh8uWfKZQB8uaeeDhW1G23+qftctYDBOqnoD7h/1+wvjs4CPEcwfKyJvFJGzReRCEfmoiHy/TC99J9AF/IeI/J2IvBqXfGSqRPI/DqwGrhORlwRdh3+B64ZajQbnClxm7N+Im7brtbhuxZ3AFyfh9W8EXigiHxY3LdqnOPSEE1y3u6ZgzO3pIrJ2pIMFEckvAm8TkS+LyN+KyLtxDf/tHDre+Gj9Bddt9cQxbv8RXPbsuYUFItKCm/7lF6p6s6reWvxg6G5waffyUYnIPBG5U0QuDd7783HJkCK4z7XYGcCdOjR3rZkBrO0dlbW9w1nba22vmRoSuCFTCWC1iMyGqXtBNFOpqgZd5K8GHsANS1qFayvfDnxDRBaNFjQw4xPUd2H43rMJEisDHxaRQi+saVfP067AU8RHcd093wGgqttwiaceAj6F+yf6TVwypFvK8YKqug83ptfHRaw+DXyPQxMxVYWq3oSbcmwlbqzk5bi7ELtxJ4qTXZ4bcVNbNeCmhPoW8CjwnKB7ZaX9G+6k6r24+vj/7d15vJ3ztcfxzzcIYqZKr6HmBKVaQ12KapSo3rqvmqmpNV+kqJaa6YBQc2uqmioqYmz0qosQVMSQ0grqmrXUTIPIsPrH+j3pY+ecOInj7Ox9vu/Xyysnz7Bfv72ffR75ree31lqdbBvW6EKyeNZPgfvIdm2dOYJsZ7U5WbH8MLJd1hbdGAUeRT4d+q+uHFyu+8iGzTuST946nLBFxGPkJGyXj8qdq3mb/Af+fsA1wHDyH8vbRMTUZVslX3ID8jO19uN7bwPfe6fhe6/vvdZkZcJ0Cdnqdhj5+/N9ldaqHX3/PFntXtXnqWwnOw/5/7AzI+LqiHgsIrYm75WDgLMcNPj4ap9dtcptEDCQDNZsBPxG0qJlxceskNbYZXKgzz4pyiIfTwI/iYgTmj0e6xpJx5ITkJVa7UmApO+SPdKXjIj3Pup4s3bke29r8r3XWlWVI9/BdpEPJy8jV/oMAc6KiBfK/gWBFSLi/p4cbzuStG5E3NuwbSkyTeweYJ6I+FrZPmeVsifpl2Rdkv8lWxc/Vy2r79l30NpqdSKWIoO3IyLiz7X955IpYyPJ4pEvlwDDCmTA/w+z8r3TUSTrFpLmlvRLSVuVZcG7k0/73iWf5FjrOI18OrhVswcyI0rO9KHkJGmWvemadSffe9uK773WcpTFCSeXnzeQtLukbSV9ruRyTyYr7A8laxocIOnTkvqRxTTvlrRO095AG5B0HHCPpP9u2LU4WXF/ELCcpGWUhQwnVE+4I2JfsnXxQOBSSUs5WDBjasGClcnAy2AyaFDdHyFXaV0MbAhcUlJ0+gNnkr8bXeme0DQttRzCZmmTyRvT2WTl6PHkEsttIlvkWIuIiLck7cy07cJmdUuQLcfObvZAzHqQ771twvdeazVlolR1MhgGfIW8D00GJko6AriofLe/Q3ZGOZSsd/EamYJzfETc14zxt5H7yar6je1fx0g6nEwJ+SbwjYg4u+ybVII9kyJiX2UL4Y7St2w6ymqMyZL6k6mQDwODq5Stsq9PSffYv5y2GxnI+QfZseLLJf1xluWUBDMzMzMzmymShpOdUE4ia718BjidbEv7NeC26qm1pDPIwnuvAudVE1h10trPukbSfBHxTnnKvVJEXF/btz5ZJHgTYM+IuKi2r96+cvGIeKmnx97qJM1P1upYkEzrGFO29yEDAnMBEyLi1bLi4ECye8hsZMDs8eaMvOscMDAzMzMzsxkmaWsynebHwJVlNcFGZAeRq4BjIuL5el68pP8AJkXEP8rfHSyYSQ2faz8yYDOIXGU2vHbcesCxZOpBY9CgwxoU1jWSliVX9v06Io4q2zYkV3XsSXYs+j8ybWts7by+EfFBE4Y8w5ySYGZmZmZmM+Pz5JPSG0uwYCDZdeQa4Mhad5RlgKcB6h1TyoTXwYKZVK83EBHvSjqdbGV5haRvR8Swsu+eUlgV4AJJUyLi4rLPwYKPZ3ayLuAXStHhnSndnMiVB7ORBQ/vAcbWgjwTO3qxWZEDBmZmZmZm1mW1pexLAy9FxN8kbQLcQLbfPLRa3l7qFxwhaf3GJe8usNc9qlUCEfEHSROBE4DLJdFB0GAScJGkSRExS7QIbnEvAjcB3wGeAyYAvyZX3NxZ0hC2BFaFf3/nW+m774CBmZmZmZl1qp7rDlk0r/x4F7CzpB8Bh5MrCw6vBQv6k5Olh8mJqnWDxjSO+iqBiLhd0lF0HjQ4CXifbLloXVTrhjA3MDdZm+D9iHhd0qHArcCywO+BJyPinXLqpuTn3bLtQ13DwMzMzMzMPpKkE4A7qirwktYEfgssB4yMiK/Wjl0S2Lv8t39EXNWEIbed2sR1CWBzYEWyQ8K1EfFs7biNyaDB2sDU9ISyb66IeL+Hh96yap95f+AUYHVgfuAF4LiIuLqT89YGjgGWBzaLiOd6aszdyQEDMzMzMzObRr0gnqQ1gAfJGgWnRcTIsn0/4ORyyoHA3eQEaTuyH/2PImJIOVattBR7VlObuK5CXofFgb5kDv3rwLciYlTt+CposAawj1MQZlz1nS0dKEaRtThuB94mC0yuB+wBXFyt+pA0Bxko2wHoD2wcEY80Y/zdwSkJZmZmZmb2IQ3Bgr2BecgnqlsAs5el7iMj4heSJgGHABeSxdwCeBY4yK0Tu0eZuE6WtCJZdf/PwA+Ba4FvARcB10naMSJuhqnpCUcCZwOnSrq+tlTeuqAECxYGLgD+AhwcEQ8ASFoBWJf8zvcBpkgS2VZ0WzIVZ4OIGNeUwXcTrzAwMzMzM7OpGtr1jQAGAI8AjwPrABsCI4ETaisNViZXFqxWjn0qIh4t+xws6AaSFgAuBhYEDouI0WX7ycAB5FPvuYGtIuKW2nkbAM9HxDM9PeZWI2lH4OWIuLW27YtkYcNjIuK8su1EMki2N3B1RLwtaY6ImChpUfL35L6IeKXn30X38goDMzMzMzObqhYsOA4YCOxGtk4cL2kRYHvgjHJMtdJgHDAO+F39tdw6sVt9luxMMbwWLPgZ8D3gf4DxwKXAUEnbVZPeepqCdU7SrmSHg59IujMiqtaHKwMLALeU44YAg4H9gKER8Z6kPsClkk6KiLHAiJ5/B58MBwzMzMzMzOxDJM1O5r4/C9wcEeMBIuI14JzSLu404N0SFLi9Oq+ho4KXM88kSXNGxARJ8wHvAq8Cl0TEmWX/YDItYTBwUUlZ2BXYBLhF0qZVgUqbPkm7A78CTgXOrQULAJ4ha0WsIWlb/h2guTwi3ivH7AN8CVioxwbdQxwwMDMzMzPr5WoF9aoJ/2RgYWBKRLxRP6acci25HHuLsu/diBhdDxbYjJO0DLBaRNxYggVrApcB342IP0qqakKsBOxL5tb/pnZd5geuB+Yla07YR5C0ExksOIUs6Pn3sr1KzXkOeIi8Dn2BXYDfVcECSWsB2wCPAX/q+XfwyerT7AGYmZmZmVnzlBoD1YTzJEmDykRpHNBf0i4AVUCh/Pw8MBZ4gGzv931Jn2rC8NtG+WwHAhdIOrEU1bsFeAN4DaCW3tEXWIGsFfF6OX99QGSRw29GxGM9/BZajqTtyUDAfcBlHQQLqu/6BcCcZDeKN6rikZK+DvyETFs4uLoW7cQBAzMzMzOzXqzWDm4Y2Q5xgbLrbHIp/GBJm5ZjJ5VjlwNWBE4kc7m3ItvM2Uwqn+0o8mn298hOCHcD3waebDhc5c/VJK0paTPgSODTwLiIeL9nRt26JO0GXAG8AqwCbF4FvWp1PFT+fh5wODAfcJOkOyX9CTiPbJ24absGaBwwMDMzMzPrhUodgurnFYDFyEnnDQAR8TCZqz0AGCJp33Ls54FdgSXJp9+3AG8Cu0jqV02ybMaUp9pPkIGa2cj08dcj4umImCJpjurYiHgEOB7YkQwqDAVWBbasnpJb50qth4vIFoi7A6OB44A9S5cDYGpbxT7l5yFk6sEZ5GqDF4FfAl8pvyttyTUMzMzMzMx6oSoNQdIp5LxgAHBrrZAbwJXAB8C5ZLHDE8mHjv2AoyLitvIarwMTIuLdHnwLbaVWIHIl4C5gDmAHSa9GxCGlZd9sZF2JAH5MtrAcSObZD4uIp5sx9lZSAl/nAD8HToqIVyS9CvwUOLocc2HVErEEa/pExJSIGAGMkNQ3Ij5o1nvoSXLhUjMzMzOz3knSAOBR4D2yGvzqpVZBvcAhkvqT6QorAn8DxkTE1WXfN4DfAOeTy7YnuztC19Xz5Wvb5gCWAc4kAwLnRMRB9eMlzRsR/+zxAbc4SRsDOwDH1GsWAOuQ9QjWJ1dvTA0aNJwvch49paNr124cMDAzMzMz68UkrUvmci8DHAKcXi3Frj9drR0/dZJUzj0OWB1YPyKe6vl30Lpq3SnmIVcULBkRf65tX5l8Ej4Q+EVEfK+cN4BsqTgcGNHuk9buVl8hUP9+S/oSXQga9CYOGJiZmZmZ9QKNE/+GfesA1wETyaDB8BI06PAJasnrvhxYDlgK2Lyd87g/CbWgQH9gCLAGsCDZmu98MhDwuqQVyZUGmwCXkjUm9gS+DnwuIh5tyhtoIw1BMAcNalz00MzMzMyszZXJafUUdUFJ/ettECPiPrLTwVzkZOlbtaXv0xQxLK81gSz8trGDBTOmfLbVCoK7yYKTw4EjyEr8vwL2lzRXRPwV2B+4FtiFDNSsAqzhYEH3qH/PI2I0eR3uJmsa7C5psWaOr5m8wsDMzMzMrI3V6xFIOot8Ut0feBW4Eziglsu9HjkxfRM4DLiug/z6+tPYfi50OHMkLQTcSBaV/EFE3F+2nwLsS3aiuJ6sCTFF0meA1YBFgVER8VxzRt6+OlhpcBywKbnq5ozOVui0MwcMzMzMzMzaVMMEaASwFnAH8BAwCNiALHq4Z0T8sRz3n8DV5ET2RxExdHqvazOnrC64Azg6Is4t204FDiDbWV4VEW9J6kd2oJjc+atZd2n4nfky8H3gsIh4rLkjaw6nJJiZmZmZtanaxOeHQDX5+W5E/AzYEjgY+BRwvqSFyzl/BLYFliAL8XX6ujZ99XSODlI7ViQ/+5vL/pOBA8lgweUR8VY57gRg2U9+tAYf/m5HxF3A9r01WAAOGJiZmZmZtQ1J/UrF/UZfAp4HhkXEO5Jmj4g3gQuBU4FVycJ7AETE3cDSEXFpT4y7XTVMPhuDLO+UP78oaQhwELAfGSx4D0DSXsAeZPDGmiAi3m/2GJrJKQlmZmZmZm2g5Lg/SOZbD61aIwLzAPcCk4C1yZz4ybWihosAI4EpwNpVu7na63baXcE6V3LgNyU7STwDnBIR4xuOGUu2s+xHFja8uNbub12yAOVEYKeIeK3HBm9WzN7sAZiZmZmZ2ccjaXFgNPAS8ET1NLtM9N+RNBrYHlg+IsaVFQaTSjDgtTJxHUQGFz4UMHCwYMZJ2gk4nex40LdsXl/S3hHxtKS+JTBwEllYb17gWUDl/M3IdJFVyC4UDhZYUzglwczMzMyshZWVBQ+QLQ73qKrtN7gBCOAKSfNGxCTIYICkzwIDyIDDP3to2G1L0u7AZcB1wBZkuscwsjvF8QC1VRy/B84CJgPXAGNK8ObXwErAZr05f96azykJZmZmZmYtqvSHv598On1QRIyp7VsUWCAinpQ0N/BTYDDwMJkrPw5YDNiFfJq9l2sWfDxlZcFlZD2IsyLihbJ9CTItZB7gc8DLtVaX/YCVyZSEpckVHiOB30bEMz38Fsw+xAEDMzMzM7MWJGkB4P/J2gTbRcQdtX3LATcBVwInRsT7pRjiscB3gIWAf5CrDuYDjo+Ik8u5bpk4EyRtBNwO3EW2qXy8bK/SP24h0xN2jojnOnmNvtFQQ8KsmVzDwMzMzMysNS0JvExW0F9N0oOlA8LywG1kisJlJVjQJyLGSzoSuBH4BvlU+y/AnRFxE7jA4cc0HhhLpiB8VdLfI+LtEiwYQNYj+Axwk6S3yZUhvwMeiohX4EOpCmazBK8wMDMzMzNrQZIEfB64AFge+AEwhqxX8AxZz+CvteOnGwxwsODjqV2PC8nOCMcC55BpCKPJ1QX3ku0ttwSWLdueIq/bFRFxY48P3Gw6HDAwMzMzM2thkr5ABg1WIlvwPQrsCLzQUWpBrZ2iAJx+0L3K9TifDAicAuwGvAPsAzwcERMlLQQsDOwFbEAGGtaKiHFNGbRZJxwwMDMzMzNrcWWSehqwIXAC8POIeMurBpqjXI/zgLXIFQXbRcS9Zd9sVcHD8neRxSnfbMpgzabDbRXNzMzMzFrfWLLTwRjgAGAnSQs6WNAcEfEQuaJgFLAIsJakecvuKTA1UEAkBwtsluQVBmZmZmZmbaK2HH554Cjg8oh4q7mj6p1KQGAN/n09jiaLUPp6WMtwwMDMzMzMrI00BA2OIIvpeZLaJB1cj6FeUWCtwikJZmZmZmZtpCyH3wt4giy6t4ekOZo7qt6rdj0eJ7smbF2lI5jN6hwwMDMzMzNrM2WSuh/wIvDPiJjY5CH1auV6HAiMBEa5M4W1CqckmJmZmZm1KUmLRsQrzR6HJUlzRsSEZo/DrKscMDAzMzMza3OS5KfaZjajHDAwMzMzMzMzs2m4hoGZmZmZmZmZTcMBAzMzMzMzMzObhgMGZmZmZmZmZjYNBwzMzMzMzMzMbBoOGJiZmZmZmZnZNBwwMDMzMzMzM7NpOGBgZmZmZmZmZtP4F9PdJ3yiLRXaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1044x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,3,figsize=(1+3*4.5,4))\n",
    "plot_info(ax[0],df,None,'Information in Weights',True)\n",
    "plot_info(ax[1],df_act,None,'Information in Activations',True)\n",
    "plot_info(ax[2],None,information_list,'Information in Activations',False)\n",
    "fig.tight_layout()\n",
    "fig.savefig('Plots/information.pdf', format='pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finetune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    }
   ],
   "source": [
    "model_ft = copy.deepcopy(model)\n",
    "retain_loader = replace_loader_dataset(train_loader_full,retain_dataset, seed=seed, batch_size=args.batch_size, shuffle=True)    \n",
    "finetune(model_ft, retain_loader, epochs=10, quiet=True, lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finetune_D_r -> Loss:0.035, Error:0.0\n",
      "Finetune_D_f -> Loss:0.068, Error:0.0\n",
      "Finetune_D_t -> Loss:0.765, Error:0.246\n"
     ]
    }
   ],
   "source": [
    "finetune_D_r_activations,finetune_D_r_predictions=activations_predictions(model_ft,retain_loader,'Finetune_D_r')\n",
    "finetune_D_f_activations,finetune_D_f_predictions=activations_predictions(model_ft,forget_loader,'Finetune_D_f')\n",
    "finetune_D_t_activations,finetune_D_t_predictions=activations_predictions(model_ft,test_loader_full,'Finetune_D_t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predictions Distance Retrain_Finetune_D_f -> 9.0\n",
      "Activations Distance Retrain_Finetune_D_f -> 0.7553855180740356\n",
      "Activations Distance Retrain_Finetune_D_r -> 0.023555029183626175\n",
      "Activations Distance Retrain_Finetune_D_t -> 0.08928284794092178\n"
     ]
    }
   ],
   "source": [
    "predictions_distance(m0_D_f_predictions,finetune_D_f_predictions,'Retrain_Finetune_D_f')\n",
    "activations_distance(m0_D_f_activations,finetune_D_f_activations,'Retrain_Finetune_D_f')\n",
    "activations_distance(m0_D_r_activations,finetune_D_r_activations,'Retrain_Finetune_D_r')\n",
    "activations_distance(m0_D_t_activations,finetune_D_t_activations,'Retrain_Finetune_D_t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Readouts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original ->\tFull test error: 24.00%\tForget error: 0.00%\tRetain error: 0.00%\tFine-tune time: 1 steps\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finetune ->\tFull test error: 24.60%\tForget error: 0.00%\tRetain error: 0.00%\tFine-tune time: 1 steps\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fisher ->\tFull test error: 31.00%\tForget error: 12.00%\tRetain error: 15.16%\tFine-tune time: 20 steps\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NTK ->\tFull test error: 25.40%\tForget error: 28.00%\tRetain error: 0.00%\tFine-tune time: 64 steps\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NTK_Fisher ->\tFull test error: 25.40%\tForget error: 28.00%\tRetain error: 0.00%\tFine-tune time: 64 steps\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrain ->\tFull test error: 25.60%\tForget error: 36.00%\tRetain error: 0.00%\tFine-tune time: 64 steps\n"
     ]
    }
   ],
   "source": [
    "try: readouts\n",
    "except: readouts = {}\n",
    "\n",
    "thresh=log_dict['Original_Model_D_f_loss']+1e-5    \n",
    "readouts[\"e\"] = all_readouts(copy.deepcopy(model),thresh,'Original')\n",
    "readouts[\"a\"] = all_readouts(copy.deepcopy(model_ft),thresh,'Finetune')\n",
    "readouts[\"b\"] = all_readouts(copy.deepcopy(modelf),thresh,'Fisher')\n",
    "readouts[\"c\"] = all_readouts(copy.deepcopy(model_scrub),thresh,'NTK')\n",
    "readouts[\"c\"] = all_readouts(copy.deepcopy(model_scrubf),thresh,'NTK_Fisher')\n",
    "readouts[\"d\"] = all_readouts(copy.deepcopy(model0),thresh,'Retrain')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 794,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f\"logs/{m0_name.split('/')[1].split('.')[0]}.npy\",log_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activations along the path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Membership Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 796,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "def entropy(p, dim = -1, keepdim = False):\n",
    "    return -torch.where(p > 0, p * p.log(), p.new([0.0])).sum(dim=dim, keepdim=keepdim)\n",
    "\n",
    "def collect_prob(data_loader, model):\n",
    "    \n",
    "    data_loader = torch.utils.data.DataLoader(data_loader.dataset, batch_size=1, shuffle=False)\n",
    "    prob = []\n",
    "    with torch.no_grad():\n",
    "        for idx, batch in enumerate(tqdm(data_loader, leave=False)):\n",
    "            batch = [tensor.to(next(model.parameters()).device) for tensor in batch]\n",
    "            data, target = batch\n",
    "            output = model(data)\n",
    "            prob.append(F.softmax(output, dim=-1).data)\n",
    "    return torch.cat(prob)\n",
    "\n",
    "def get_membership_attack_data(retain_loader, forget_loader, test_loader, model):    \n",
    "    retain_prob = collect_prob(retain_loader, model)\n",
    "    forget_prob = collect_prob(forget_loader, model)\n",
    "    test_prob = collect_prob(test_loader, model)\n",
    "    \n",
    "    X_r = torch.cat([entropy(retain_prob), entropy(test_prob)]).cpu().numpy().reshape(-1, 1)\n",
    "    Y_r = np.concatenate([np.ones(len(retain_prob)), np.zeros(len(test_prob))])\n",
    "    \n",
    "    X_f = entropy(forget_prob).cpu().numpy().reshape(-1, 1)\n",
    "    Y_f = np.concatenate([np.ones(len(forget_prob))])    \n",
    "    return X_f, Y_f, X_r, Y_r\n",
    "\n",
    "def get_membership_attack_prob(retain_loader, forget_loader, test_loader, model):\n",
    "    X_f, Y_f, X_r, Y_r = get_membership_attack_data(retain_loader, forget_loader, test_loader, model)\n",
    "    clf = SVC(C=3,gamma='auto',kernel='rbf')\n",
    "    #clf = LogisticRegression(class_weight='balanced',solver='lbfgs',multi_class='multinomial')\n",
    "    clf.fit(X_r, Y_r)\n",
    "    results = clf.predict(X_f)\n",
    "    return results.mean()\n",
    "    \n",
    "def plot_entropy_dist(model, ax, title):\n",
    "    train_loader_full, test_loader_full = datasets.get_loaders(dataset, batch_size=100, seed=0, augment=False, shuffle=False)\n",
    "    indexes = np.flatnonzero(np.array(train_loader_full.dataset.targets) == class_to_forget)\n",
    "    replaced = np.random.RandomState(0).choice(indexes, size=100 if num_to_forget==100 else len(indexes), replace=False)\n",
    "    X_f, Y_f, X_r, Y_r = get_membership_attack_data(train_loader_full, test_loader_full, model, replaced)\n",
    "    sns.distplot(np.log(X_r[Y_r==1]).reshape(-1), kde=False, norm_hist=True, rug=False, label='retain', ax=ax)\n",
    "    sns.distplot(np.log(X_r[Y_r==0]).reshape(-1), kde=False, norm_hist=True, rug=False, label='test', ax=ax)\n",
    "    sns.distplot(np.log(X_f).reshape(-1), kde=False, norm_hist=True, rug=False, label='forget', ax=ax)\n",
    "    ax.legend(prop={'size': 14})\n",
    "    ax.tick_params(labelsize=12)\n",
    "    ax.set_title(title,size=18)\n",
    "    ax.set_xlabel('Log of Entropy',size=14)\n",
    "    ax.set_ylim(0,0.4)\n",
    "    ax.set_xlim(-35,2)\n",
    "\n",
    "def membership_attack(retain_loader,forget_loader,test_loader,model):\n",
    "    prob = get_membership_attack_prob(retain_loader,forget_loader,test_loader,model)\n",
    "    print(\"Attack prob: \", prob)\n",
    "    return prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 797,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_dict={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 798,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack prob:  1.0\n"
     ]
    }
   ],
   "source": [
    "attack_dict['Original']=membership_attack(retain_loader,forget_loader,test_loader_full,model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 799,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack prob:  0.8\n"
     ]
    }
   ],
   "source": [
    "attack_dict['Retrain']=membership_attack(retain_loader,forget_loader,test_loader_full,model0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 800,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack prob:  0.8\n"
     ]
    }
   ],
   "source": [
    "attack_dict['NTK']=membership_attack(retain_loader,forget_loader,test_loader_full,model_scrub)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 801,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack prob:  1.0\n"
     ]
    }
   ],
   "source": [
    "attack_dict['Fisher']=membership_attack(retain_loader,forget_loader,test_loader_full,modelf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 802,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack prob:  0.96\n"
     ]
    }
   ],
   "source": [
    "attack_dict['Finetune']=membership_attack(retain_loader,forget_loader,test_loader_full,model_ft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 803,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=475), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attack prob:  0.8\n"
     ]
    }
   ],
   "source": [
    "attack_dict['Fisher_NTK']=membership_attack(retain_loader,forget_loader,test_loader_full,model_scrubf)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}