checkPerformance-reproduce_MMTLF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os, os.path, time\n",
    "import argparse\n",
    "import numpy\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.optim import *\n",
    "from torch.optim.lr_scheduler import *\n",
    "from torch.autograd import Variable\n",
    "from Net_mModal import Net, NewNet, TransformerEncoder, Transformer, MMTEncoder, LateFusion, videoModel, SuperLateFusion\n",
    "from util_in_multi_h5_unnorm import *\n",
    "from util_out import *\n",
    "from util_f1 import *\n",
    "from AudioResNet import resnet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    from torch.utils.tensorboard import SummaryWriter\n",
    "except:\n",
    "    from tensorboardX import SummaryWriter\n",
    "#from transformers import  AdamW, get_linear_schedule_with_warmup\n",
    "torch.backends.cudnn.benchmark = True\n",
    "from torch.optim.lr_scheduler import LambdaLR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()\n",
    "torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps, last_epoch=-1):\n",
    "    \"\"\"\n",
    "    Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after\n",
    "    a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer.\n",
    "    Args:\n",
    "        optimizer (:class:`~torch.optim.Optimizer`):\n",
    "            The optimizer for which to schedule the learning rate.\n",
    "        num_warmup_steps (:obj:`int`):\n",
    "            The number of steps for the warmup phase.\n",
    "        num_training_steps (:obj:`int`):\n",
    "            The total number of training steps.\n",
    "        last_epoch (:obj:`int`, `optional`, defaults to -1):\n",
    "            The index of the last epoch when resuming training.\n",
    "    Return:\n",
    "        :obj:`torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule.\n",
    "    \"\"\"\n",
    "\n",
    "    def lr_lambda(current_step: int):\n",
    "        if current_step < num_warmup_steps:\n",
    "            return float(current_step) / float(max(1, num_warmup_steps))\n",
    "        return max(\n",
    "            0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps))\n",
    "        )\n",
    "\n",
    "    return LambdaLR(optimizer, lr_lambda, last_epoch)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VICTIM MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mybool(s):\n",
    "    return s.lower() in ['t', 'true', 'y', 'yes', '1']\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--model_type', type = str, default = None, required=True)\n",
    "parser.add_argument('--embedding_size', type = int, default = 1024) # this is the embedding size after a pooling layer\n",
    "                                                                    # after a non-pooling layer, the embeddings size will be twice this much\n",
    "parser.add_argument('--n_conv_layers', type = int, default = 10)\n",
    "parser.add_argument('--n_trans_layers', type = int, default = 1)\n",
    "parser.add_argument('--kernel_size', type = str, default = '3')     # 'n' or 'nxm'\n",
    "parser.add_argument('--n_pool_layers', type = int, default = 5)     # the pooling layers will be inserted uniformly into the conv layers\n",
    "                                                                    # the should be at least 2 and at most 6 pooling layers\n",
    "                                                                    # the first two pooling layers will have stride (2,2); later ones will have stride (1,2)\n",
    "parser.add_argument('--batch_norm', type = mybool, default = True)\n",
    "parser.add_argument('--dropout', type = float, default = 0.0)\n",
    "parser.add_argument('--pooling', type = str, default = 'lin', choices = ['max', 'ave', 'lin', 'exp', 'att', 'h-att', 'all'])\n",
    "parser.add_argument('--batch_size', type = int, default = 250)\n",
    "parser.add_argument('--ckpt_size', type = int, default = 1000)      # how many batches per checkpoint\n",
    "parser.add_argument('--optimizer', type = str, default = 'adam', choices = ['adam', 'sgd', 'adamw'])\n",
    "parser.add_argument('--init_lr', type = float, default = 1e-3)\n",
    "parser.add_argument('--weight_decay', type = float, default = 0)\n",
    "parser.add_argument('--beta1', type = float, default = 0.9)\n",
    "parser.add_argument('--beta2', type = float, default = 0.999)\n",
    "parser.add_argument('--lr_patience', type = int, default = 3)\n",
    "parser.add_argument('--lr_factor', type = float, default = 0.8)\n",
    "parser.add_argument('--max_ckpt', type = int, default = 30)\n",
    "parser.add_argument('--random_seed', type = int, default = 15213)\n",
    "parser.add_argument('--additional_outname', type = str, default = '')\n",
    "parser.add_argument('--continue_from_ckpt', type = str, default = None)\n",
    "parser.add_argument('--warmup_steps', type = int, default = 1000)\n",
    "parser.add_argument('--gradient_accumulation', type = int, default = 1)\n",
    "parser.add_argument('--scheduler', type = str, default = 'reduce', choices = ['reduce', 'warmup-decay'])\n",
    "parser.add_argument('--addpos', type = mybool, default = False)\n",
    "parser.add_argument('--transformer_dropout', type = float, default = 0.75)\n",
    "parser.add_argument('--from_scratch', type = mybool, default = False)\n",
    "parser.add_argument('--fusion_module', type = int, default = 0) # 0 for early fusion, 1 for mid fusion 1 \n",
    "###MODEL INPUT PARAMS###\n",
    "args = parser.parse_args(args=['--model_type=MMTLF','--init_lr=4e-4','--fusion_module=0','--embedding_size=1024','--pooling=att','--n_trans_layers=2', '--batch_size=100', '--ckpt_size=2500', '--n_conv_layers=10', '--n_pool_layers=5', '--gradient_accumulation=3','--max_ckpt=3', '--scheduler=warmup-decay',\n",
    "                              '--additional_outname=late_fusion_all_additional_linear_42.7'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'x' not in args.kernel_size:\n",
    "    args.kernel_size = args.kernel_size + 'x' + args.kernel_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare log file and model directory\n",
    "expid = '%s-embed%d-%dC%dP-kernel%s-%s-drop%.1f-%s-batch%d-ckpt%d-%s-lr%.0e-pat%d-fac%.1f-seed%d-Trans%d-weight-decay%.8f-betas%.3f-%.3f' % (\n",
    "    args.model_type,\n",
    "    args.embedding_size,\n",
    "    args.n_conv_layers,\n",
    "    args.n_pool_layers,\n",
    "    args.kernel_size,\n",
    "    'bn' if args.batch_norm else 'nobn',\n",
    "    args.dropout,\n",
    "    args.pooling,\n",
    "    args.batch_size,\n",
    "    args.ckpt_size,\n",
    "    args.optimizer,\n",
    "    args.init_lr,\n",
    "    args.lr_patience,\n",
    "    args.lr_factor,\n",
    "    args.random_seed,\n",
    "    args.n_trans_layers,\n",
    "    args.weight_decay,\n",
    "    args.beta1, \n",
    "    args.beta2\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "expid += args.additional_outname\n",
    "WORKSPACE = os.path.join('/jet/home/billyli/data_folder/DayLongAudio/workspace/audioset/', expid)\n",
    "PRED_PATH = os.path.join(WORKSPACE, 'pred')\n",
    "if not os.path.exists(PRED_PATH): os.makedirs(PRED_PATH)\n",
    "EVAL_PATH = os.path.join(WORKSPACE, 'eval')\n",
    "if not os.path.exists(EVAL_PATH): os.makedirs(EVAL_PATH)\n",
    "if args.model_type=='TALNet':\n",
    "#     WORKSPACE = '../../workspace/audioset/MMT-embed1024-10C5P-kernel3x3-bn-drop0.0-att-batch200-ckpt2500-adam-lr4e-04-pat3-fac0.8-seed15213-Trans2-weight-decay0.00000000-betas0.900-0.999early'\n",
    "    PRED_PATH = os.path.join(WORKSPACE, 'pred')\n",
    "    EVAL_PATH = os.path.join(WORKSPACE, 'eval')\n",
    "    MODEL_FILE = os.path.join(WORKSPACE, 'model', 'checkpoint20.pt')\n",
    "    PRED_FILE = os.path.join(PRED_PATH, 'checkpoint20.mat')\n",
    "    EVAL_FILE = os.path.join(EVAL_PATH, 'checkpoint20.txt')\n",
    "else:\n",
    "    MODEL_FILE = os.path.join(WORKSPACE, 'model','checkpoint%d.pt' % args.max_ckpt)\n",
    "    PRED_FILE = os.path.join(PRED_PATH, 'checkpoint%d.mat' % args.max_ckpt)\n",
    "    EVAL_FILE = os.path.join(EVAL_PATH, 'checkpoint%d.txt' % args.max_ckpt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3x3'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args.kernel_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MMTLF\n",
      "/jet/home/billyli/data_folder/DayLongAudio/workspace/audioset/MMTLF-embed1024-10C5P-kernel3x3-bn-drop0.0-att-batch100-ckpt2500-adam-lr4e-04-pat3-fac0.8-seed15213-Trans2-weight-decay0.00000000-betas0.900-0.999late_fusion_all_additional_linear_42.7/model/checkpoint3.pt\n"
     ]
    }
   ],
   "source": [
    "print(args.model_type)\n",
    "print(MODEL_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model\n",
    "args.kernel_size = tuple(int(x) for x in args.kernel_size.split('x'))\n",
    "#model = Net(args).cuda()\n",
    "if args.model_type == 'TAL':\n",
    "    model = Net(args).cuda()\n",
    "elif args.model_type == 'TAL-trans':\n",
    "    model = TransformerEncoder(args).cuda()\n",
    "elif args.model_type == 'TAL-new':\n",
    "    model = NewNet(args).cuda()\n",
    "elif args.model_type == 'Trans':\n",
    "    model = Transformer(args).cuda()\n",
    "elif args.model_type == 'resnet':\n",
    "    model = resnet50().cuda()\n",
    "elif args.model_type == 'MMT':\n",
    "    model = MMTEncoder(args).cuda()\n",
    "elif args.model_type == 'VM':\n",
    "    model = videoModel(args).cuda()\n",
    "elif args.model_type == 'MMTLF':\n",
    "    model = LateFusion(args).cuda()\n",
    "else:\n",
    "    print ('model type not recognized')\n",
    "    exit(0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "if args.optimizer == 'sgd':\n",
    "    optimizer = SGD(model.parameters(), lr = args.init_lr, momentum = 0.9, nesterov = True)\n",
    "elif args.optimizer == 'adam':\n",
    "    optimizer = Adam(model.parameters(), lr = args.init_lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay)\n",
    "elif args.optimizer == 'adamw':\n",
    "    optimizer = AdamW(model.parameters(), lr = args.init_lr, betas=(args.beta1, args.beta2), weight_decay=args.weight_decay)\n",
    "if args.scheduler == 'reduce':\n",
    "    scheduler = ReduceLROnPlateau(optimizer, mode = 'max', factor = args.lr_factor, patience = args.lr_patience) if args.lr_factor < 1.0 else None\n",
    "elif args.scheduler == 'warmup-decay':\n",
    "    t_total = args.ckpt_size * args.max_ckpt / args.gradient_accumulation\n",
    "#     scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) Deprecated API style\n",
    "    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps = t_total)\n",
    "\n",
    "else:\n",
    "    print ('scheduler type not recognized')\n",
    "    exit(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LateFusion(\n",
       "  (alpha): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (beta): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "  (fc_prob): Linear(in_features=1024, out_features=527, bias=True)\n",
       "  (fc_att): Linear(in_features=1024, out_features=527, bias=True)\n",
       "  (branch1): TransformerEncoder(\n",
       "    (conv1): ConvBlock(\n",
       "      (conv): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv2): ConvBlock(\n",
       "      (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv3): ConvBlock(\n",
       "      (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv4): ConvBlock(\n",
       "      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv5): ConvBlock(\n",
       "      (conv): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv6): ConvBlock(\n",
       "      (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv7): ConvBlock(\n",
       "      (conv): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv8): ConvBlock(\n",
       "      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv9): ConvBlock(\n",
       "      (conv): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (conv10): ConvBlock(\n",
       "      (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    )\n",
       "    (enc_layer): EncoderLayer(\n",
       "      (slf_attn): MultiHeadAttention(\n",
       "        (w_qs): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (w_ks): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (w_vs): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (attention): ScaledDotProductAttention(\n",
       "          (dropout): Dropout(p=0.75, inplace=False)\n",
       "          (softmax): Softmax(dim=2)\n",
       "        )\n",
       "        (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (fc): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (dropout): Dropout(p=0.75, inplace=False)\n",
       "      )\n",
       "      (pos_ffn): PositionwiseFeedForward(\n",
       "        (w_1): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (w_2): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        (dropout): Dropout(p=0.75, inplace=False)\n",
       "        (layer_norm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (fc_prob): Linear(in_features=1024, out_features=527, bias=True)\n",
       "    (fc_att): Linear(in_features=1024, out_features=527, bias=True)\n",
       "  )\n",
       "  (branch2): videoModel(\n",
       "    (enc_layer): EncoderLayer(\n",
       "      (slf_attn): MultiHeadAttention(\n",
       "        (w_qs): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "        (w_ks): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "        (w_vs): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "        (attention): ScaledDotProductAttention(\n",
       "          (dropout): Dropout(p=0.75, inplace=False)\n",
       "          (softmax): Softmax(dim=2)\n",
       "        )\n",
       "        (layer_norm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (fc): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "        (dropout): Dropout(p=0.75, inplace=False)\n",
       "      )\n",
       "      (pos_ffn): PositionwiseFeedForward(\n",
       "        (w_1): Linear(in_features=2048, out_features=1024, bias=True)\n",
       "        (w_2): Linear(in_features=1024, out_features=2048, bias=True)\n",
       "        (dropout): Dropout(p=0.75, inplace=False)\n",
       "        (layer_norm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "    (hidden1): Linear(in_features=2048, out_features=1024, bias=True)\n",
       "    (fc_prob): Linear(in_features=1024, out_features=527, bias=True)\n",
       "    (fc_att): Linear(in_features=1024, out_features=527, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prev_ckpt = torch.load(MODEL_FILE)\n",
    "start_ckpt = prev_ckpt['epoch']\n",
    "scheduler.load_state_dict(prev_ckpt['scheduler'])\n",
    "# model = nn.DataParallel(model)\n",
    "model.load_state_dict(prev_ckpt['model'])\n",
    "optimizer.load_state_dict(prev_ckpt['optimizer'])\n",
    "# write_log('Loading model from %s' % args.continue_from_ckpt)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_valid_x1, gas_valid_x2, gas_valid_y, gas_eval_hashes = multi_bulk_load('GAS_valid')\n",
    "gas_eval_x1, gas_eval_x2, gas_eval_y, gas_eval_hashes = multi_bulk_load('GAS_eval')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_eval_x1, gas_eval_x2, gas_eval_y, gas_eval_hashes = multi_bulk_load('GAS_eval')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20098, 527)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gas_eval_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 1. 1. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "print(gas_eval_y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_eval_org_list = gas_eval_hashes.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "gas_eval_global_prob_multi = model.predict(gas_eval_x1, gas_eval_x2, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(gas_eval_global_prob_multi[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performance on Google Audio Set:\n",
      "   CLASS ||    AP |   AUC |    d' \n",
      "---------++-------+-------+-------\n"
     ]
    }
   ],
   "source": [
    "print('Performance on Google Audio Set:')\n",
    "print(\"   CLASS ||    AP |   AUC |    d' \")\n",
    "FORMAT  = ' %00007s || %5.3f | %5.3f |%6.03f '\n",
    "SEP     = ''.join('+' if c == '|' else '-' for c in FORMAT)\n",
    "print(SEP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.42654634009523607 0.9712865122083788 2.6870719826038076\n",
      " Average || 0.427 | 0.971 | 2.687 \n",
      "---------++-------+-------+-------\n",
      "       0 || 0.774 | 0.922 | 2.004 \n",
      "       1 || 0.088 | 0.862 | 1.538 \n",
      "       2 || 0.092 | 0.933 | 2.114 \n",
      "       3 || 0.435 | 0.976 | 2.808 \n",
      "       4 || 0.207 | 0.980 | 2.897 \n",
      "       5 || 0.062 | 0.919 | 1.981 \n",
      "       6 || 0.646 | 0.998 | 4.018 \n",
      "       7 || 0.458 | 0.985 | 3.065 \n",
      "       8 || 0.174 | 0.940 | 2.195 \n",
      "       9 || 0.094 | 0.935 | 2.143 \n",
      "      10 || 0.103 | 0.952 | 2.350 \n",
      "      11 || 0.180 | 0.977 | 2.812 \n",
      "      12 || 0.827 | 0.999 | 4.300 \n",
      "      13 || 0.476 | 0.995 | 3.639 \n",
      "      14 || 0.244 | 0.970 | 2.664 \n",
      "      15 || 0.738 | 0.992 | 3.435 \n",
      "      16 || 0.408 | 0.981 | 2.941 \n",
      "      17 || 0.468 | 0.990 | 3.305 \n",
      "      18 || 0.199 | 0.979 | 2.862 \n",
      "      19 || 0.154 | 0.973 | 2.714 \n",
      "      20 || 0.241 | 0.987 | 3.129 \n",
      "      21 || 0.149 | 0.974 | 2.742 \n",
      "      22 || 0.452 | 0.989 | 3.259 \n",
      "      23 || 0.518 | 0.994 | 3.552 \n",
      "      24 || 0.187 | 0.967 | 2.596 \n",
      "      25 || 0.091 | 0.954 | 2.381 \n",
      "      26 || 0.202 | 0.965 | 2.570 \n",
      "      27 || 0.377 | 0.969 | 2.642 \n",
      "      28 || 0.608 | 0.988 | 3.213 \n",
      "      29 || 0.468 | 0.991 | 3.324 \n",
      "      30 || 0.334 | 0.989 | 3.246 \n",
      "      31 || 0.365 | 0.985 | 3.081 \n",
      "      32 || 0.155 | 0.958 | 2.438 \n",
      "      33 || 0.259 | 0.982 | 2.978 \n",
      "      34 || 0.501 | 0.989 | 3.249 \n",
      "      35 || 0.461 | 0.978 | 2.840 \n",
      "      36 || 0.393 | 0.989 | 3.251 \n",
      "      37 || 0.395 | 0.974 | 2.754 \n",
      "      38 || 0.058 | 0.890 | 1.733 \n",
      "      39 || 0.313 | 0.980 | 2.899 \n",
      "      40 || 0.680 | 0.999 | 4.228 \n",
      "      41 || 0.158 | 0.948 | 2.306 \n",
      "      42 || 0.179 | 0.927 | 2.061 \n",
      "      43 || 0.750 | 0.986 | 3.107 \n",
      "      44 || 0.088 | 0.921 | 1.996 \n",
      "      45 || 0.229 | 0.904 | 1.846 \n",
      "      46 || 0.108 | 0.925 | 2.036 \n",
      "      47 || 0.328 | 0.981 | 2.928 \n",
      "      48 || 0.262 | 0.973 | 2.717 \n",
      "      49 || 0.459 | 0.965 | 2.568 \n",
      "      50 || 0.251 | 0.915 | 1.940 \n",
      "      51 || 0.572 | 0.983 | 2.991 \n",
      "      52 || 0.599 | 0.968 | 2.610 \n",
      "      53 || 0.204 | 0.946 | 2.269 \n",
      "      54 || 0.446 | 0.988 | 3.199 \n",
      "      55 || 0.113 | 0.922 | 2.010 \n",
      "      56 || 0.600 | 0.976 | 2.802 \n",
      "      57 || 0.616 | 0.958 | 2.441 \n",
      "      58 || 0.643 | 0.974 | 2.744 \n",
      "      59 || 0.566 | 0.988 | 3.194 \n",
      "      60 || 0.743 | 0.981 | 2.937 \n",
      "      61 || 0.483 | 0.974 | 2.746 \n",
      "      62 || 0.532 | 0.934 | 2.128 \n",
      "      63 || 0.473 | 0.974 | 2.746 \n",
      "      64 || 0.634 | 0.995 | 3.662 \n",
      "      65 || 0.350 | 0.983 | 3.007 \n",
      "      66 || 0.440 | 0.994 | 3.555 \n",
      "      67 || 0.523 | 0.972 | 2.711 \n",
      "      68 || 0.544 | 0.997 | 3.908 \n",
      "      69 || 0.575 | 0.993 | 3.488 \n",
      "      70 || 0.333 | 0.960 | 2.475 \n",
      "      71 || 0.555 | 0.996 | 3.784 \n",
      "      72 || 0.635 | 0.980 | 2.894 \n",
      "      73 || 0.643 | 0.989 | 3.217 \n",
      "      74 || 0.702 | 0.991 | 3.344 \n",
      "      75 || 0.523 | 0.995 | 3.675 \n",
      "      76 || 0.298 | 0.983 | 2.983 \n",
      "      77 || 0.694 | 0.998 | 4.020 \n",
      "      78 || 0.398 | 0.982 | 2.950 \n",
      "      79 || 0.367 | 0.980 | 2.898 \n",
      "      80 || 0.202 | 0.991 | 3.342 \n",
      "      81 || 0.748 | 0.994 | 3.596 \n",
      "      82 || 0.785 | 0.965 | 2.568 \n",
      "      83 || 0.579 | 0.990 | 3.301 \n",
      "      84 || 0.460 | 0.962 | 2.515 \n",
      "      85 || 0.631 | 0.997 | 3.822 \n",
      "      86 || 0.609 | 0.992 | 3.386 \n",
      "      87 || 0.667 | 0.998 | 4.003 \n",
      "      88 || 0.637 | 0.983 | 3.011 \n",
      "      89 || 0.698 | 0.988 | 3.192 \n",
      "      90 || 0.732 | 0.996 | 3.769 \n",
      "      91 || 0.726 | 0.996 | 3.760 \n",
      "      92 || 0.554 | 0.967 | 2.608 \n",
      "      93 || 0.470 | 0.986 | 3.092 \n",
      "      94 || 0.270 | 0.960 | 2.479 \n",
      "      95 || 0.586 | 0.995 | 3.637 \n",
      "      96 || 0.496 | 0.991 | 3.367 \n",
      "      97 || 0.552 | 0.995 | 3.624 \n",
      "      98 || 0.784 | 0.983 | 3.005 \n",
      "      99 || 0.776 | 0.985 | 3.077 \n",
      "     100 || 0.611 | 0.983 | 3.002 \n",
      "     101 || 0.870 | 0.996 | 3.758 \n",
      "     102 || 0.742 | 0.988 | 3.179 \n",
      "     103 || 0.720 | 0.989 | 3.227 \n",
      "     104 || 0.283 | 0.944 | 2.244 \n",
      "     105 || 0.352 | 0.964 | 2.536 \n",
      "     106 || 0.674 | 0.973 | 2.723 \n",
      "     107 || 0.626 | 0.962 | 2.502 \n",
      "     108 || 0.415 | 0.989 | 3.228 \n",
      "     109 || 0.520 | 0.986 | 3.122 \n",
      "     110 || 0.526 | 0.971 | 2.685 \n",
      "     111 || 0.465 | 0.967 | 2.606 \n",
      "     112 || 0.408 | 0.971 | 2.680 \n",
      "     113 || 0.182 | 0.965 | 2.560 \n",
      "     114 || 0.250 | 0.903 | 1.836 \n",
      "     115 || 0.826 | 0.995 | 3.631 \n",
      "     116 || 0.830 | 0.991 | 3.355 \n",
      "     117 || 0.713 | 0.991 | 3.373 \n",
      "     118 || 0.739 | 0.977 | 2.828 \n",
      "     119 || 0.693 | 0.986 | 3.115 \n",
      "     120 || 0.665 | 0.941 | 2.209 \n",
      "     121 || 0.654 | 0.991 | 3.332 \n",
      "     122 || 0.397 | 0.988 | 3.182 \n",
      "     123 || 0.406 | 0.985 | 3.082 \n",
      "     124 || 0.099 | 0.894 | 1.768 \n",
      "     125 || 0.448 | 0.985 | 3.081 \n",
      "     126 || 0.705 | 0.970 | 2.654 \n",
      "     127 || 0.402 | 0.978 | 2.849 \n",
      "     128 || 0.568 | 0.976 | 2.805 \n",
      "     129 || 0.751 | 0.971 | 2.682 \n",
      "     130 || 0.047 | 0.831 | 1.358 \n",
      "     131 || 0.754 | 0.988 | 3.208 \n",
      "     132 || 0.748 | 0.982 | 2.977 \n",
      "     133 || 0.744 | 0.986 | 3.093 \n",
      "     134 || 0.607 | 0.971 | 2.671 \n",
      "     135 || 0.064 | 0.838 | 1.394 \n",
      "     136 || 0.541 | 0.990 | 3.296 \n",
      "     137 || 0.823 | 0.937 | 2.163 \n",
      "     138 || 0.422 | 0.969 | 2.638 \n",
      "     139 || 0.578 | 0.991 | 3.356 \n",
      "     140 || 0.703 | 0.992 | 3.394 \n",
      "     141 || 0.567 | 0.994 | 3.554 \n",
      "     142 || 0.495 | 0.993 | 3.468 \n",
      "     143 || 0.537 | 0.991 | 3.339 \n",
      "     144 || 0.353 | 0.986 | 3.115 \n",
      "     145 || 0.496 | 0.994 | 3.529 \n",
      "     146 || 0.276 | 0.989 | 3.239 \n",
      "     147 || 0.774 | 0.996 | 3.781 \n",
      "     148 || 0.535 | 0.994 | 3.578 \n",
      "     149 || 0.458 | 0.982 | 2.975 \n",
      "     150 || 0.582 | 0.988 | 3.211 \n",
      "     151 || 0.568 | 0.987 | 3.133 \n",
      "     152 || 0.515 | 0.991 | 3.336 \n",
      "     153 || 0.373 | 0.986 | 3.123 \n",
      "     154 || 0.347 | 0.990 | 3.280 \n",
      "     155 || 0.783 | 0.991 | 3.365 \n",
      "     156 || 0.376 | 0.991 | 3.342 \n",
      "     157 || 0.485 | 0.996 | 3.717 \n",
      "     158 || 0.365 | 0.977 | 2.832 \n",
      "     159 || 0.231 | 0.969 | 2.645 \n",
      "     160 || 0.731 | 0.989 | 3.252 \n",
      "     161 || 0.561 | 0.982 | 2.964 \n",
      "     162 || 0.598 | 0.993 | 3.471 \n",
      "     163 || 0.300 | 0.984 | 3.033 \n",
      "     164 || 0.550 | 0.986 | 3.110 \n",
      "     165 || 0.573 | 0.994 | 3.529 \n",
      "     166 || 0.529 | 0.993 | 3.445 \n",
      "     167 || 0.505 | 0.981 | 2.935 \n",
      "     168 || 0.573 | 0.990 | 3.292 \n",
      "     169 || 0.671 | 0.995 | 3.613 \n",
      "     170 || 0.546 | 0.997 | 3.916 \n",
      "     171 || 0.458 | 0.991 | 3.374 \n",
      "     172 || 0.492 | 0.995 | 3.662 \n",
      "     173 || 0.552 | 0.985 | 3.082 \n",
      "     174 || 0.684 | 0.993 | 3.508 \n",
      "     175 || 0.388 | 0.961 | 2.490 \n",
      "     176 || 0.304 | 0.970 | 2.651 \n",
      "     177 || 0.576 | 0.993 | 3.471 \n",
      "     178 || 0.177 | 0.974 | 2.747 \n",
      "     179 || 0.252 | 0.982 | 2.979 \n",
      "     180 || 0.289 | 0.974 | 2.747 \n",
      "     181 || 0.284 | 0.983 | 3.015 \n",
      "     182 || 0.383 | 0.994 | 3.558 \n",
      "     183 || 0.717 | 0.988 | 3.201 \n",
      "     184 || 0.410 | 0.985 | 3.051 \n",
      "     185 || 0.768 | 0.994 | 3.534 \n",
      "     186 || 0.670 | 0.998 | 4.029 \n",
      "     187 || 0.500 | 0.991 | 3.361 \n",
      "     188 || 0.622 | 0.995 | 3.660 \n",
      "     189 || 0.507 | 0.986 | 3.110 \n",
      "     190 || 0.417 | 0.990 | 3.311 \n",
      "     191 || 0.365 | 0.991 | 3.348 \n",
      "     192 || 0.725 | 0.990 | 3.285 \n",
      "     193 || 0.605 | 0.995 | 3.603 \n",
      "     194 || 0.636 | 0.992 | 3.416 \n",
      "     195 || 0.691 | 0.988 | 3.208 \n",
      "     196 || 0.565 | 0.991 | 3.340 \n",
      "     197 || 0.595 | 0.995 | 3.651 \n",
      "     198 || 0.544 | 0.994 | 3.543 \n",
      "     199 || 0.822 | 0.999 | 4.230 \n",
      "     200 || 0.173 | 0.973 | 2.731 \n",
      "     201 || 0.584 | 0.987 | 3.146 \n",
      "     202 || 0.545 | 0.985 | 3.087 \n",
      "     203 || 0.659 | 0.981 | 2.936 \n",
      "     204 || 0.512 | 0.972 | 2.713 \n",
      "     205 || 0.657 | 0.984 | 3.019 \n",
      "     206 || 0.708 | 0.977 | 2.827 \n",
      "     207 || 0.854 | 0.999 | 4.534 \n",
      "     208 || 0.844 | 0.999 | 4.241 \n",
      "     209 || 0.808 | 0.997 | 3.858 \n",
      "     210 || 0.886 | 0.997 | 3.954 \n",
      "     211 || 0.819 | 0.997 | 3.837 \n",
      "     212 || 0.800 | 0.964 | 2.539 \n",
      "     213 || 0.396 | 0.978 | 2.840 \n",
      "     214 || 0.679 | 0.997 | 3.840 \n",
      "     215 || 0.595 | 0.986 | 3.107 \n",
      "     216 || 0.199 | 0.965 | 2.566 \n",
      "     217 || 0.347 | 0.991 | 3.352 \n",
      "     218 || 0.513 | 0.980 | 2.903 \n",
      "     219 || 0.133 | 0.978 | 2.840 \n",
      "     220 || 0.239 | 0.980 | 2.918 \n",
      "     221 || 0.145 | 0.985 | 3.062 \n",
      "     222 || 0.208 | 0.989 | 3.225 \n",
      "     223 || 0.214 | 0.985 | 3.080 \n",
      "     224 || 0.216 | 0.979 | 2.863 \n",
      "     225 || 0.181 | 0.988 | 3.196 \n",
      "     226 || 0.150 | 0.972 | 2.696 \n",
      "     227 || 0.112 | 0.974 | 2.758 \n",
      "     228 || 0.274 | 0.979 | 2.886 \n",
      "     229 || 0.204 | 0.968 | 2.623 \n",
      "     230 || 0.278 | 0.974 | 2.738 \n",
      "     231 || 0.488 | 0.991 | 3.366 \n",
      "     232 || 0.112 | 0.969 | 2.635 \n",
      "     233 || 0.118 | 0.962 | 2.509 \n",
      "     234 || 0.168 | 0.963 | 2.522 \n",
      "     235 || 0.054 | 0.938 | 2.170 \n",
      "     236 || 0.226 | 0.983 | 3.000 \n",
      "     237 || 0.078 | 0.962 | 2.516 \n",
      "     238 || 0.552 | 0.981 | 2.925 \n",
      "     239 || 0.112 | 0.966 | 2.587 \n",
      "     240 || 0.246 | 0.986 | 3.123 \n",
      "     241 || 0.311 | 0.987 | 3.151 \n",
      "     242 || 0.374 | 0.974 | 2.738 \n",
      "     243 || 0.277 | 0.966 | 2.588 \n",
      "     244 || 0.212 | 0.980 | 2.893 \n",
      "     245 || 0.369 | 0.985 | 3.058 \n",
      "     246 || 0.311 | 0.987 | 3.164 \n",
      "     247 || 0.242 | 0.990 | 3.272 \n",
      "     248 || 0.268 | 0.989 | 3.240 \n",
      "     249 || 0.526 | 0.980 | 2.892 \n",
      "     250 || 0.409 | 0.988 | 3.212 \n",
      "     251 || 0.187 | 0.971 | 2.688 \n",
      "     252 || 0.188 | 0.982 | 2.962 \n",
      "     253 || 0.261 | 0.984 | 3.024 \n",
      "     254 || 0.173 | 0.985 | 3.080 \n",
      "     255 || 0.537 | 0.992 | 3.427 \n",
      "     256 || 0.431 | 0.986 | 3.114 \n",
      "     257 || 0.457 | 0.994 | 3.513 \n",
      "     258 || 0.125 | 0.956 | 2.408 \n",
      "     259 || 0.343 | 0.977 | 2.834 \n",
      "     260 || 0.104 | 0.963 | 2.528 \n",
      "     261 || 0.448 | 0.992 | 3.421 \n",
      "     262 || 0.411 | 0.977 | 2.819 \n",
      "     263 || 0.329 | 0.986 | 3.107 \n",
      "     264 || 0.100 | 0.973 | 2.725 \n",
      "     265 || 0.143 | 0.980 | 2.915 \n",
      "     266 || 0.103 | 0.952 | 2.350 \n",
      "     267 || 0.258 | 0.973 | 2.717 \n",
      "     268 || 0.218 | 0.967 | 2.606 \n",
      "     269 || 0.161 | 0.965 | 2.558 \n",
      "     270 || 0.101 | 0.935 | 2.142 \n",
      "     271 || 0.384 | 0.986 | 3.095 \n",
      "     272 || 0.369 | 0.986 | 3.109 \n",
      "     273 || 0.123 | 0.969 | 2.641 \n",
      "     274 || 0.240 | 0.981 | 2.947 \n",
      "     275 || 0.138 | 0.957 | 2.429 \n",
      "     276 || 0.160 | 0.967 | 2.593 \n",
      "     277 || 0.072 | 0.924 | 2.027 \n",
      "     278 || 0.236 | 0.966 | 2.583 \n",
      "     279 || 0.110 | 0.949 | 2.315 \n",
      "     280 || 0.069 | 0.940 | 2.200 \n",
      "     281 || 0.602 | 0.996 | 3.700 \n",
      "     282 || 0.400 | 0.957 | 2.429 \n",
      "     283 || 0.517 | 0.985 | 3.085 \n",
      "     284 || 0.157 | 0.854 | 1.491 \n",
      "     285 || 0.464 | 0.982 | 2.977 \n",
      "     286 || 0.767 | 0.998 | 4.098 \n",
      "     287 || 0.759 | 0.998 | 4.024 \n",
      "     288 || 0.576 | 0.977 | 2.828 \n",
      "     289 || 0.641 | 0.993 | 3.444 \n",
      "     290 || 0.443 | 0.979 | 2.890 \n",
      "     291 || 0.479 | 0.992 | 3.404 \n",
      "     292 || 0.328 | 0.988 | 3.171 \n",
      "     293 || 0.661 | 0.996 | 3.730 \n",
      "     294 || 0.448 | 0.993 | 3.465 \n",
      "     295 || 0.403 | 0.993 | 3.476 \n",
      "     296 || 0.452 | 0.987 | 3.128 \n",
      "     297 || 0.132 | 0.932 | 2.112 \n",
      "     298 || 0.546 | 0.984 | 3.048 \n",
      "     299 || 0.044 | 0.878 | 1.650 \n",
      "     300 || 0.580 | 0.971 | 2.686 \n",
      "     301 || 0.856 | 0.993 | 3.452 \n",
      "     302 || 0.852 | 0.997 | 3.898 \n",
      "     303 || 0.797 | 0.995 | 3.693 \n",
      "     304 || 0.712 | 0.984 | 3.026 \n",
      "     305 || 0.622 | 0.998 | 4.184 \n",
      "     306 || 0.101 | 0.971 | 2.679 \n",
      "     307 || 0.437 | 0.984 | 3.034 \n",
      "     308 || 0.353 | 0.991 | 3.320 \n",
      "     309 || 0.503 | 0.979 | 2.867 \n",
      "     310 || 0.551 | 0.983 | 3.006 \n",
      "     311 || 0.422 | 0.969 | 2.646 \n",
      "     312 || 0.631 | 0.990 | 3.312 \n",
      "     313 || 0.259 | 0.989 | 3.251 \n",
      "     314 || 0.218 | 0.987 | 3.160 \n",
      "     315 || 0.407 | 0.995 | 3.664 \n",
      "     316 || 0.405 | 0.984 | 3.046 \n",
      "     317 || 0.315 | 0.983 | 3.011 \n",
      "     318 || 0.598 | 0.997 | 3.815 \n",
      "     319 || 0.425 | 0.983 | 2.999 \n",
      "     320 || 0.746 | 0.981 | 2.942 \n",
      "     321 || 0.361 | 0.979 | 2.864 \n",
      "     322 || 0.889 | 0.998 | 4.171 \n",
      "     323 || 0.612 | 0.998 | 4.088 \n",
      "     324 || 0.695 | 0.998 | 4.118 \n",
      "     325 || 0.793 | 0.984 | 3.040 \n",
      "     326 || 0.496 | 0.989 | 3.228 \n",
      "     327 || 0.164 | 0.962 | 2.502 \n",
      "     328 || 0.797 | 0.998 | 3.991 \n",
      "     329 || 0.819 | 0.998 | 3.977 \n",
      "     330 || 0.522 | 0.995 | 3.678 \n",
      "     331 || 0.587 | 0.988 | 3.188 \n",
      "     332 || 0.820 | 0.998 | 4.040 \n",
      "     333 || 0.830 | 0.998 | 4.184 \n",
      "     334 || 0.646 | 0.996 | 3.714 \n",
      "     335 || 0.622 | 0.994 | 3.550 \n",
      "     336 || 0.697 | 0.999 | 4.280 \n",
      "     337 || 0.611 | 0.991 | 3.360 \n",
      "     338 || 0.237 | 0.982 | 2.950 \n",
      "     339 || 0.622 | 0.997 | 3.931 \n",
      "     340 || 0.650 | 0.996 | 3.774 \n",
      "     341 || 0.690 | 0.985 | 3.062 \n",
      "     342 || 0.839 | 0.994 | 3.579 \n",
      "     343 || 0.353 | 0.978 | 2.844 \n",
      "     344 || 0.337 | 0.988 | 3.208 \n",
      "     345 || 0.117 | 0.860 | 1.525 \n",
      "     346 || 0.646 | 0.996 | 3.743 \n",
      "     347 || 0.626 | 0.991 | 3.375 \n",
      "     348 || 0.124 | 0.978 | 2.847 \n",
      "     349 || 0.121 | 0.970 | 2.660 \n",
      "     350 || 0.241 | 0.984 | 3.025 \n",
      "     351 || 0.294 | 0.978 | 2.861 \n",
      "     352 || 0.203 | 0.983 | 3.008 \n",
      "     353 || 0.266 | 0.982 | 2.953 \n",
      "     354 || 0.116 | 0.960 | 2.475 \n",
      "     355 || 0.439 | 0.969 | 2.637 \n",
      "     356 || 0.138 | 0.934 | 2.132 \n",
      "     357 || 0.173 | 0.935 | 2.140 \n",
      "     358 || 0.126 | 0.925 | 2.041 \n",
      "     359 || 0.480 | 0.945 | 2.257 \n",
      "     360 || 0.444 | 0.959 | 2.466 \n",
      "     361 || 0.368 | 0.857 | 1.508 \n",
      "     362 || 0.324 | 0.954 | 2.386 \n",
      "     363 || 0.249 | 0.967 | 2.609 \n",
      "     364 || 0.245 | 0.981 | 2.922 \n",
      "     365 || 0.320 | 0.985 | 3.060 \n",
      "     366 || 0.609 | 0.975 | 2.777 \n",
      "     367 || 0.658 | 0.997 | 3.847 \n",
      "     368 || 0.311 | 0.952 | 2.360 \n",
      "     369 || 0.872 | 0.999 | 4.388 \n",
      "     370 || 0.447 | 0.994 | 3.542 \n",
      "     371 || 0.582 | 0.997 | 3.855 \n",
      "     372 || 0.557 | 0.994 | 3.568 \n",
      "     373 || 0.809 | 0.992 | 3.407 \n",
      "     374 || 0.821 | 0.997 | 3.927 \n",
      "     375 || 0.465 | 0.923 | 2.019 \n",
      "     376 || 0.346 | 0.932 | 2.106 \n",
      "     377 || 0.682 | 0.987 | 3.133 \n",
      "     378 || 0.427 | 0.963 | 2.526 \n",
      "     379 || 0.208 | 0.914 | 1.933 \n",
      "     380 || 0.335 | 0.963 | 2.525 \n",
      "     381 || 0.460 | 0.976 | 2.794 \n",
      "     382 || 0.474 | 0.972 | 2.705 \n",
      "     383 || 0.873 | 0.999 | 4.353 \n",
      "     384 || 0.614 | 0.991 | 3.326 \n",
      "     385 || 0.489 | 0.979 | 2.862 \n",
      "     386 || 0.665 | 0.993 | 3.486 \n",
      "     387 || 0.744 | 0.989 | 3.260 \n",
      "     388 || 0.173 | 0.944 | 2.246 \n",
      "     389 || 0.253 | 0.985 | 3.056 \n",
      "     390 || 0.602 | 0.964 | 2.537 \n",
      "     391 || 0.443 | 0.963 | 2.523 \n",
      "     392 || 0.526 | 0.988 | 3.198 \n",
      "     393 || 0.569 | 0.988 | 3.215 \n",
      "     394 || 0.500 | 0.967 | 2.595 \n",
      "     395 || 0.327 | 0.975 | 2.770 \n",
      "     396 || 0.864 | 0.994 | 3.553 \n",
      "     397 || 0.854 | 0.997 | 3.852 \n",
      "     398 || 0.217 | 0.958 | 2.437 \n",
      "     399 || 0.717 | 0.997 | 3.957 \n",
      "     400 || 0.713 | 0.994 | 3.558 \n",
      "     401 || 0.222 | 0.963 | 2.533 \n",
      "     402 || 0.310 | 0.971 | 2.674 \n",
      "     403 || 0.613 | 0.994 | 3.524 \n",
      "     404 || 0.175 | 0.928 | 2.070 \n",
      "     405 || 0.125 | 0.941 | 2.213 \n",
      "     406 || 0.435 | 0.966 | 2.590 \n",
      "     407 || 0.092 | 0.884 | 1.692 \n",
      "     408 || 0.355 | 0.944 | 2.245 \n",
      "     409 || 0.233 | 0.957 | 2.424 \n",
      "     410 || 0.152 | 0.936 | 2.154 \n",
      "     411 || 0.799 | 0.994 | 3.537 \n",
      "     412 || 0.501 | 0.966 | 2.574 \n",
      "     413 || 0.308 | 0.964 | 2.550 \n",
      "     414 || 0.673 | 0.983 | 3.009 \n",
      "     415 || 0.665 | 0.993 | 3.504 \n",
      "     416 || 0.498 | 0.974 | 2.754 \n",
      "     417 || 0.347 | 0.966 | 2.586 \n",
      "     418 || 0.322 | 0.979 | 2.867 \n",
      "     419 || 0.486 | 0.985 | 3.066 \n",
      "     420 || 0.515 | 0.960 | 2.484 \n",
      "     421 || 0.452 | 0.986 | 3.102 \n",
      "     422 || 0.587 | 0.994 | 3.566 \n",
      "     423 || 0.673 | 0.977 | 2.831 \n",
      "     424 || 0.273 | 0.986 | 3.122 \n",
      "     425 || 0.480 | 0.988 | 3.180 \n",
      "     426 || 0.239 | 0.982 | 2.965 \n",
      "     427 || 0.692 | 0.983 | 3.008 \n",
      "     428 || 0.732 | 0.997 | 3.816 \n",
      "     429 || 0.515 | 0.995 | 3.617 \n",
      "     430 || 0.754 | 0.996 | 3.791 \n",
      "     431 || 0.720 | 0.997 | 3.893 \n",
      "     432 || 0.738 | 0.991 | 3.328 \n",
      "     433 || 0.483 | 0.992 | 3.398 \n",
      "     434 || 0.033 | 0.905 | 1.850 \n",
      "     435 || 0.355 | 0.969 | 2.637 \n",
      "     436 || 0.366 | 0.992 | 3.424 \n",
      "     437 || 0.492 | 0.979 | 2.864 \n",
      "     438 || 0.699 | 0.992 | 3.401 \n",
      "     439 || 0.376 | 0.946 | 2.272 \n",
      "     440 || 0.065 | 0.884 | 1.688 \n",
      "     441 || 0.203 | 0.956 | 2.417 \n",
      "     442 || 0.144 | 0.924 | 2.022 \n",
      "     443 || 0.283 | 0.968 | 2.626 \n",
      "     444 || 0.067 | 0.934 | 2.127 \n",
      "     445 || 0.579 | 0.994 | 3.528 \n",
      "     446 || 0.184 | 0.973 | 2.735 \n",
      "     447 || 0.034 | 0.888 | 1.720 \n",
      "     448 || 0.157 | 0.950 | 2.330 \n",
      "     449 || 0.378 | 0.973 | 2.726 \n",
      "     450 || 0.185 | 0.924 | 2.031 \n",
      "     451 || 0.263 | 0.962 | 2.516 \n",
      "     452 || 0.102 | 0.931 | 2.102 \n",
      "     453 || 0.452 | 0.961 | 2.492 \n",
      "     454 || 0.247 | 0.983 | 2.985 \n",
      "     455 || 0.771 | 0.999 | 4.269 \n",
      "     456 || 0.321 | 0.978 | 2.857 \n",
      "     457 || 0.308 | 0.982 | 2.977 \n",
      "     458 || 0.797 | 0.993 | 3.460 \n",
      "     459 || 0.261 | 0.957 | 2.423 \n",
      "     460 || 0.215 | 0.925 | 2.039 \n",
      "     461 || 0.198 | 0.954 | 2.376 \n",
      "     462 || 0.527 | 0.990 | 3.293 \n",
      "     463 || 0.576 | 0.991 | 3.351 \n",
      "     464 || 0.402 | 0.994 | 3.557 \n",
      "     465 || 0.747 | 0.995 | 3.633 \n",
      "     466 || 0.087 | 0.922 | 2.007 \n",
      "     467 || 0.534 | 0.988 | 3.188 \n",
      "     468 || 0.185 | 0.973 | 2.716 \n",
      "     469 || 0.283 | 0.972 | 2.711 \n",
      "     470 || 0.231 | 0.969 | 2.639 \n",
      "     471 || 0.203 | 0.949 | 2.316 \n",
      "     472 || 0.646 | 0.978 | 2.859 \n",
      "     473 || 0.059 | 0.924 | 2.025 \n",
      "     474 || 0.381 | 0.949 | 2.315 \n",
      "     475 || 0.014 | 0.816 | 1.274 \n",
      "     476 || 0.643 | 0.991 | 3.367 \n",
      "     477 || 0.189 | 0.970 | 2.654 \n",
      "     478 || 0.212 | 0.848 | 1.456 \n",
      "     479 || 0.472 | 0.995 | 3.686 \n",
      "     480 || 0.271 | 0.969 | 2.647 \n",
      "     481 || 0.242 | 0.953 | 2.369 \n",
      "     482 || 0.348 | 0.927 | 2.053 \n",
      "     483 || 0.261 | 0.980 | 2.912 \n",
      "     484 || 0.265 | 0.968 | 2.624 \n",
      "     485 || 0.038 | 0.860 | 1.529 \n",
      "     486 || 0.030 | 0.833 | 1.365 \n",
      "     487 || 0.147 | 0.907 | 1.867 \n",
      "     488 || 0.074 | 0.924 | 2.024 \n",
      "     489 || 0.103 | 0.940 | 2.197 \n",
      "     490 || 0.678 | 0.995 | 3.630 \n",
      "     491 || 0.708 | 0.993 | 3.488 \n",
      "     492 || 0.530 | 0.997 | 3.920 \n",
      "     493 || 0.333 | 0.952 | 2.353 \n",
      "     494 || 0.774 | 0.991 | 3.364 \n",
      "     495 || 0.214 | 0.975 | 2.778 \n",
      "     496 || 0.265 | 0.951 | 2.334 \n",
      "     497 || 0.167 | 0.972 | 2.701 \n",
      "     498 || 0.828 | 0.998 | 4.044 \n",
      "     499 || 0.386 | 0.959 | 2.460 \n",
      "     500 || 0.136 | 0.936 | 2.147 \n",
      "     501 || 0.460 | 0.977 | 2.821 \n",
      "     502 || 0.022 | 0.908 | 1.880 \n",
      "     503 || 0.162 | 0.950 | 2.333 \n",
      "     504 || 0.068 | 0.910 | 1.895 \n",
      "     505 || 0.095 | 0.851 | 1.473 \n",
      "     506 || 0.185 | 0.869 | 1.589 \n",
      "     507 || 0.071 | 0.861 | 1.535 \n",
      "     508 || 0.106 | 0.948 | 2.293 \n",
      "     509 || 0.146 | 0.898 | 1.796 \n",
      "     510 || 0.075 | 0.898 | 1.798 \n",
      "     511 || 0.242 | 0.954 | 2.388 \n",
      "     512 || 0.189 | 0.936 | 2.151 \n",
      "     513 || 0.087 | 0.898 | 1.797 \n",
      "     514 || 0.411 | 0.961 | 2.494 \n",
      "     515 || 0.358 | 0.956 | 2.406 \n",
      "     516 || 0.406 | 0.945 | 2.258 \n",
      "     517 || 0.519 | 0.994 | 3.557 \n",
      "     518 || 0.112 | 0.963 | 2.528 \n",
      "     519 || 0.101 | 0.929 | 2.081 \n",
      "     520 || 0.063 | 0.921 | 1.997 \n",
      "     521 || 0.483 | 0.990 | 3.310 \n",
      "     522 || 0.249 | 0.981 | 2.925 \n",
      "     523 || 0.143 | 0.954 | 2.390 \n",
      "     524 || 0.272 | 0.936 | 2.148 \n",
      "     525 || 0.311 | 0.911 | 1.906 \n",
      "     526 || 0.199 | 0.910 | 1.895 \n"
     ]
    }
   ],
   "source": [
    "classwise = []\n",
    "classwise_dict = []\n",
    "N_CLASSES = gas_eval_global_prob_multi[0].shape[-1]\n",
    "for i in range(N_CLASSES):\n",
    "    a, b, c = gas_eval(gas_eval_global_prob_multi[0][:,i], gas_eval_y[:,i])     # AP, AUC, dprime\n",
    "    classwise_dict.append({'mAP':a, 'AUC':b, 'dprime':c, 'id':i})\n",
    "    classwise.append((a, b, c))\n",
    "map, mauc = numpy.array(classwise).mean(axis = 0)[:2]\n",
    "print (map, mauc, dprime(mauc))\n",
    "# for c in classwise_dict:\n",
    "#     print(c)\n",
    "print(FORMAT % ('Average', map, mauc, dprime(mauc)))\n",
    "print(SEP)\n",
    "for i in range(N_CLASSES):\n",
    "    print(FORMAT % ((str(i),) + classwise[i]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 28800x7200 with 0 Axes>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD5CAYAAAAOXX+6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAACEwklEQVR4nO1dd5wURdp+qntmNgfYJaclShAQRAQRRcWAijmHz5xOvVPPU089784znXqenmdCzzOnUxEjBqKgZAmSc16WhV02T+zvj5rqqa6u7umZnWVnl378+WOnY3V31VNvPe9bbxFN0+DChQsXLtIXSnMXwIULFy5c2MMlahcuXLhIc7hE7cKFCxdpDpeoXbhw4SLN4RK1CxcuXKQ5XKJ24cKFizSHx8lBhJAtAKoBhAGENE0bYXd8cXGxVlJS0ujCuXDhwsWhgsWLF5drmtZOts8RUUdxgqZp5U4OLCkpwaJFixK4tAsXLlwc2iCEbLXa50ofLly4cJHmcErUGoDvCCGLCSE3yg4ghNxICFlECFm0d+/e1JXQhQsXLg5xOCXqMZqmDQcwAcCthJDjxAM0TZukadoITdNGtGsnlVlcuHDhwkUScETUmqbtiv5bBmAygJFNWSgXLly4cBFDXKImhOQQQvLY3wBOAfBrUxfMhQsXLlxQOIn66ABgMiGEHf+epmlTm7RULly4cOFCR1yi1jRtE4ChB6EsLly4cOFCAjc8rxmwp3YPZm6f2dzFcOHCRQtBIhNeXKQIl399OfbU7cGKq1Y0d1FcuHDRAuBa1M2APXV7AAARLdLMJTl48If9+MtPf8G++n3NXRQXLlocXKJuRoS1cHMX4aDhuy3f4ZP1n+DpRU83d1FcuGhxcIm6GRGOHDpEzaDBXaPThYtE4RJ1M+JQkj5cuHCRPFoMUS8qXYQD/gPNXYyUIqSFmrsILly4aAFoEUQdjARxzbfX4Obvb27uoqQUh6T0obnShwsXiaJFEDVr3Gv2r2nmkqQWh5Iz0UUM6yrWYe7Ouc1dDBctCC0ijpo5oCJoHZquQhREtAhCkUNH+oimIHAB4PzPzwcAN47ehWO0CIuaOd1ai/ONgJLWoWhRu1EfLlwkjhZB1K1N12TW5aGkUbPOqTXCH/bj0XmPpr2zOxgJojpQ3dzFcJEEWgRRtxZLmoGR1iEZ9dG6+lwAwFebvsIHaz/A878839xFscUdM+7AMe8f09zFcJEEWgZRtxJtmkEh9LW7FnXrgEehrp66YF1C5x1sA2T2jtkH9X4uUocWQdStTvo4hDXq1ogMNQMA0BBuSOg8f9jfFMVx0QrRIoi61Ukfh6BGzdAanYmZaiYAoCGUGFEHwoGmKI6LVgiXqJsBh7RG3QrBpKxELeREid3FoYsWQdStzQo7FDVq5mdobd8SiElYrkXtoqnQIoi61VnUpPVr1Cd/fDJu/O5G/Xdr+4Y82MSl+nB9Quclqmm7OHThEnUzQJc+WvHMxNLaUvy8+2f9d0sdPUS0SNxoDiZh+UOJSR+iRa1pGhaWLmx1znMXjUeLIOrWVnF16aMVW9QiWGfb0r7ls4ufxdHvHW2rP7NOyKmFzDpq8fhP13+Ka7+9Ft9t/S7J0jpDS/sGLloIUbe2OGo9PK+FWpnJoKV2Sl9u+hIAbGcdspGRU43aq3gBmJ2P26u3AwC2Vm1NuJyJoDX6CVo7WgZRtzbpgxx6UR+6Rd3CSCLLkwUAqA9Z68+sE3Ia9eFTfQBgyqDHJs4EI8GEy5kIUtmeDvgPYPR7o7GodFHKrunCjBZB1K1tqNYaLOple5dh2NvDUF5f7uj4lmpRZ3pojLQdUTOL2ukzMov6rVVvGXRqRtSLShfhxaUvJlVeJxDbU22wNulr/Vr+K2qCNZi0fJKj48ORMILhpu2IWiNaBFG3NovaiUb94ZoP03rK75sr30QoEsLiPYtN+2Qda0v9hsyitiMzRtSKw+bkVb3633wHwEh70Z5FeGnZSwmX1Sn4Uc3k9ZMx6r1R2HRgU1LXYqNDp/LkH2b/AcPfGZ7UvQ5ltAyibmEadUOowVbTjBf14Q/78cj8R3DrtFubpHyphCyHRyBijg9uqaMHZlHbETXrcJ3m3GYWNWDUtQ9WZju+02TGwIaKDUldixkdThWt77d+n9R9DnW0CKJuadLHi0tfxHXfXme5P14c9dKypQAAlagpL1uqwBq73lA5yLTalip9JGJR+8N+TJw8MW6nxKacA8bIj5pgjeG4xtb7zzZ8hgfmPGDazhM1+37JGkOso25pxlRLQ4sg6pY2bC6vL8eeuj2W++Pl+mDD0H5t+qW+cCkCIxGZFSmLJ25p35DBCVHzndCWqi1xw/Syvdn637xFXRMwEnVjO7c/zf0TPt/4ue0xOlFHGkfUiXYqLXWE1VxwiboJEIqEbEO1mJZp1RDZ86bz8lVM55RJH4eqRe0UES2ij5Z4jbo6aJQ+muqdpdSijtZRJ9E8fEfU1JEtrQ0tgqhbWkhXSAuhIdxg2cHo4XkWDZxZJ83ZQZXWluLtVW9b7mffRCZ9yHJYtKQJL8FwEPN3zwcQS2GaCFHHe8aIFkG2h1rVBulDtKhTZHVOXj/ZWD6uPelEnWRdc2pRf7flO4x+f7T+2yXqxNAiiJqvRC1hJfJ4EyDi5aNOhzUifzv9t3hy4ZPYVbNLut+uM7GzqFuCZf3M4mdw/XfXY+W+lfo2UT/mIT5TPOtUg4YsL7XUG0IN+rsUQwBTFWf/0E8PobS2NFY+mUXtsK5pmmYgZacW9U+7fjL8dhNSJYYWQdR8xbjwiwvTOmwN4IjaQquMp1Hrq643I1FX+isNZRHByiazjGREnQ6dj1MwH0FlQ6VeXrt8H+J3jKf38hb1rppdGPLWEHy+8XMT4adSx60LxcpvINo4FnFFQwVun347KhsqAQDjPhqHCZ9O0Pczoo9nUYsjL9eiTgyOiZoQohJCfiGEfNmUBZJBbNybD2w+aPd+Z9U7WFq2FPN3z8c7q95xdE48i5pVWiuLKR1ITdfJLZbQYgQuk2/sLOpZO2Zh2rZpqSpmk4OV245YxH3xRg0RLaI7FLdVbwMAPDDnAdO7TOXog59kwne+qqLqZZLhndXvYOb2mTj909Oxp3YP9jfsx86anfp+p1EfYj1KdNLLFxu/wGsrXkvonNYETwLH/g7AagD5TVQWSzRn6M/fF/7d8PuKgVfEPYc1MKvZbPFmJqYDUTMLydLqj+53bFFzVuaf5vwJJ112UiqK2aTQoOnfwI40xX3xZABN02IaNdeZi9dJZXbFv/78V/1vvl7FI9oqfxUA6ui85ttrTPt1h3cc14PoGE/Uor5/zv0AgOsHX5/Qea0FjixqQkhXAGcAaJYurSU4oHg4TdJj1fjTQfrQLUlN3qDsLGqZ/sg/a32oHg2hBmw5sCUtw7R464+Vz07OEJ8h3jNFELOoeSdlIBzQp5EDqbWoV5Sv0P+WOhMtno+PRGFJo3jo0omEqeuCdXobEKUP2aSoROEP+5s8gVW6wKn08SyAewBr05YQciMhZBEhZNHevXtTUTYd6aRr8p3GtqpteGz+Y6aGqSeSt7Co7UgOSBOLmivjo/MexeA3Bxv321jUTJv3kBjp8M8S0kI46t2jMPGziXhl+SspL3sqob8HG8eeuM+JRS0L+6sKVCHPm6f/DoQDeH/N+yl3vPF1OF54nhiJIkK3yCV19ej3jsa4j8YZ7sOQinwfD855EGdOPtM2D0s8HKzZoI1FXKImhJwJoEzTNHNSBw6apk3SNG2Epmkj2rVrl7ICAqkhrJpATaM+KANPrn+Y/Qe8v+Z9rKmIRaJsObAFm6uohm51v3jDaZGoZ2ybge1VZmumKcHKFoqE8MHaD0z77Tob5njK8GSYridCliskncDKzb7Fz7t+xsTJEw3yTqLaMoujzlQzDU4+AMjzxYj6y01f4rH5jzlOeOQUiUR9xCOyeFEfrCMyadQpcCbO2TkHQPJrT87cPhPHvH8Mfin7pdFlaWo4sajHADiLELIFwAcATiSEOPOqpQipiKMe/f5onPbJaY2+jmy5Jd5CmfjZRL1yW0V9sEbx+q+vS60lPfQtauX8dsZvcdaUsxpX8ATByhgv1lvW4H7Y9gMAoDir2HQ9q/ukK5gkwEZNj81/DFuqthjCFk3heQ6iPhSiINOTaYomyfHm6H+zd8scjqmCwZlI7J2J4iQcy2smKE+mQvpgSNYAm7VjFgBg3f51KStLUyEuUWua9kdN07pqmlYC4BIA0zVNi+9RSyFS1Zj3N+xv9DVY711WV4ZV+1YBsC6fVU/PHy+LU2YEzTf4g71sVzyiZmWU7V+xd4XhGuLfLQWapsVGFlF5g5EnryWL78BJHLVCFPgUn4lkWK5qIEaiVYGqJJ/A4v6SOGir78OcifGuFc+YagrpQ18pJ0mLmhlU+RkHPT4iYbSIOOrmmiQhq7ysUlzw+QW2xwHWPT3/PLJzdZJspoUFwpGwlKj5Bm4XR8228c8Wz3GaVuBG6aJjlz2bzOGo/3YgfRAQeBSPaUTFZ9ZjRFLtT62OynckcS3qONKHU8d3UzgT2SdI1qJmz5brzW18WZoYCRG1pmkzNU07s6kKY3Nf299NBVmDY5Wiwl8RK48F2Vg6EzVNdxrJKjh7vlAkdNAjXsrry3HE20foZTcQNfeczCIy6bORsH6cjNhFpHtEDyNh9i97Xv65xQ6VjYRCkZAh5ljfH5U+vKrXFMpYmFGo/82sWbsEX8nAEJ4Xx6KOt2qN05V7mkKjZhB1fidYX7Feny3JYsnTGYnEUTcbmmvYzN9XJSrCWliqO1vJA3bSBxs6yzoD3po9mJLH0wufNgy9ASMJhbWwbhkxi0gcwhpGC2ihFjUHcXTDCIZ/Hivp45nFz+DtVW9jxkUzDHq9plHpw0M8JstyUPEgFGQU4JP1n6AqGCPqadumobyuHBFEcGn/Sxv3UNwrj2dRJzK13A7i90+l9JGMRf3o/Ef1v1nZX1r6EvIz8nH5gMsbXbZUo0VIH83VmPkhLSMwGflahU/ZORPZEFcqfSAWFWIlf2iahq82fZW0PieiKlCFN1e9iVdXvGrYzls+fGPULWqhfDxpGTRqCwdbOlvUGjRT1Ad7vnOmnKNby6Yp5NFjWWKnvXXGcNUIIiBELn2oRMUZvc4AYJQ8dlTvwCPzH8Fj8x9r9HPxHahdeB3gINRQMnoCzJ2X+DuVFnUyRN0pp5P+N/vG07dPx487f0xZuVKJFkHU6WBRM2LdcmCLybljRdSW4XmIWdQyxxMvfVhZ1Iv2LMJ9P96Hpxc9HecpnGFZ2TLpdv7+vFXELMF3V79riFywIvaWblEzMuYtQZbr2SR9RM9hmfdE+SCiRaBAgUfxmPapRDU4EVm8dSpHVvx3iTeF3Om1xO8oPpdY/lTGhidD1N3zu+t/8wnGDrbT3inSlqh/3vUzbp92OzRNM1Wig9W4eXJhje6R+Y9g4uSJhuOsHCO8tfvR2o/0ZD+RCEfUEkuTj7O2qjiscso0UB63TrsVTy18yvYYAFhbsVa63cqZyDc0fiFWK2JP1mJrDvB6qih98KQskri4XSTq/Q37EQgHoGmablGLUBVVl5eqAlV6XHUqCWRj5UZTtFGyDnsrZ6KJqDVnFnV1oNpxbD3T15MhavZtAOO8hXQl6rTVqG+ffjv8YT8awg1mZ2IzEDWv3Yphfk4s6r/N+xsy1UwsvGKhwaK206gBa2cOOz/eEHL2jtmYjdm4aehNyPdZhyFZSShWxMvfVzb5w0M8hu9mGcGSfjxtAN+ITcmXIsbQPX179D2xCT91wTqEIiEc/+HxGNVpFCKgzkQZUXuIR99eFahCSX4JyurKUhr5dMfMOwAAK65aEcvpkixRW0hX/Co/X2760tQxWNXbu2behXm752HeZfMMMeUyNEajloWO2hlGzY20tah5ImqupEwy6UMGq0rHKhCrzEyz5p2Jsk6H32ZF1Kw8YsUqry/H8788b7LydlYbLe8pG6YYJByrUYGV5sx3Tszq21q1VW/wXtVr+G4tyaLmwc/Q/MOsP0j3ie+afW+2NuJvZ/wWo94bBQCYt3teLOpDUqdUJSZ9hCIh5HhzoBK1yQjEalTgFLpGLXxH3j/zxx//iHm75xn2Wxk36yro5BOres/XI3bPZPw0fAdzx8w7MHn95LS2qNOWqFllDYQDzaZRO628ls7EaAUyzVzjoz4k9+Cf16oSsvPFivX4/McxafkkzC+db3T+cZ3JyvKVeHDug3jk50di+y288E6J+stNX+LMyWfq03q9itcURz2g7QB0yO5guH46OxMBo7Ulpme1SgXAfvOjMJ54+DhqESpRDeFi2Z5sStSNiKn3EA/aZ7eX7nOSy8QOukat2WvUIqz2s3Zv1fYMfpJoHUzGoha/2cfrPkY44lrUCYNZG8FwsNniqHmi2VK1xfK4eFEfYkcT1sK2FrUT6cPKomYVsCZQYziXEbWmaZi+fToA4EDggGm/CDE8j12Dt8A9ige/lv8KILYCj0/1GeOoIxF4Va/JikxHi5pPps9HfYiTNnhrmwf7fvxq4+J+S+lD8RiSWWV6MqEqaqOzDPLXlJU1WYKy0qjtrNyizCJUNFRg84HN+ixWBvaOraQYPZuhFtEJOhmilsmpGjSXqBMFq8SBSOMt6mSJ3aluZyUbsApkylesaXrDCUaCJjLmy2tF1MyRsnLfSuyr36dvZ3mOa4O1hnPn7JyDYCSIyRsm60l+sjxZCEfCeHf1u5ZrAsqcibLQK1HrzFAzTM5EZi0ZnjUNiZohokUMz9w2s61hP08aPsVnOA8wOqx4aJoGlahS6cOjeAwWdaaaCY/iabRGLesUWFmA1EsfdhZ1cVYx9tXvw1mfnYXLvr5MWs54Ccv46ycTQSKTU+3CYZsbaetM1Ik6HHCsUb+7+l1ommZK7p8s0Ts9z0o2YFaFzKL2qrSR3j7tdgQiAay4KmZZ8M9rZZnwZH7D9zfg07M+BRBbNXtfwz7Dua+teA0ExKBLZ3uyMWXjFDyx4AnT9Ud2HIkFpQsMlrbVaieBcMCUTc+reE3heQpRTA06naWPCCKGtR6LMotQXl+u7+f3+VSf3mHLpA/DdbVoHLXEyuXD89g1PMTTaEvPiqgbm64gGemjOKvY8B55MIvaUorTzLnek2nfMr9CJBJBiKQnUbcIi9ppY35iwROmFVmA5D3aTs+rClThtE9Ow8+7fjZsl1nUZXVlAIzPBxjX5OOfd0HpAuk9+cq59UAseTq71966vabGsqVqi6GCZ3myLHM5PHnckwDkFjWzYErySwBQ4hatba9q1KiDkaDl8Ptg4LgPjsO/lvwroXP40NDyunKprwGgjZ63juMlKmJRH6yz5qEqqoFUfaoPqmJ0JiZs/RKbTsMmuZYTOI2jZvjTqD+hKKsI5Q1youYdqTKMeX8MHpr7kEHuSGZUZjIYopOb0nUtx7QlavbBguFgo6WPpInaYYNYV7EOO2t24h+L/mHYXumv1HtqhpP+R5eg8hJjI+U1cL68b6x8Q3pPvqLxDZ5JGLtrd2Pp3qWGc1SiGhoQs75lyPRQfVUWnsc6lysHXgmf4kNpbSlW718NILZat0/xGUYGu2t3o0OO0ZEIHLzJTBX+CtOsy3gIa2FD9rwNlRtM+9m/PBGKsxlFaJpm6Uz0EI/Bos5QM0zSRzIJjUZ2HGlZFgDYULlBdwQnAsuoD8lIcHSn0bjosItQlFVkkOt46Ba1DWFO3jDZdgkzJ5DNzUgk6mPZ3mV6hMrBQNoSNavE/rDf9FKfXfIsJq+f7PhaTgh3YelC/LjDOH3UaQVg1rDY8PxhP877/DzpVHLx2I2VG/W/nZAXb3Xz+igj6hnbZ+CBOQ8YzlGIoq8uDgBZXiNR8xM9fIoPHsVja1Eza2/KxilYtpfObDzgP6Dv0zQNb658E4PfHIyyujJ0y+vWbDHxiUBPhi+ZbNU1t6v+N5+wiSfqeCFvujNRJn0oxqgPn+qjeWa4ayWjyd4w+AbLsgDAqn2rcMsPtxj2ORnJWjkTZYTH6nxxZrElEbNnt5rhysDnhU+ms7dK3eCUqK/4+gqc//n5Cd83WaQ9UfP6J4/PNnzm+FpOCPfab6/Fb6b9xrDNaQWoDVFylDmHNlRuwMp9K03bRaLmM6Q5aSBWMd61wVpLS1khiiHvhBjFwOfl9SgeeBWvrUXtU3ymBqcTteJDRIsYprh3y+smHXKmKyJaxFQH2FqHgNGi5r+BGLYnfmuWj9oyPE9iUfPfIZnheWFmIV4a/5Jpu10dd9Ju9LqqAbf8cAueW/KcZRnZ87LRmgwy6UOaxTLISR9J+DlMFnV05NtcKZXjIf2JOgVRH8nqb46JOmrFWjlsZEMkMbUi7zxx4jw1ELUgfYjRCQwKUQyatGjx8TMXmbPLMGU6Wq4Fu6lu7lN9pmswZ6VX9ZpIuFteN1OZ0t2ZKNYBvhNkMpIoffCzGdtltcOdw+80XdtqCrlH8Zg0ag/xGFIFJJsnQ5E0d/EbsbYSjoQdhb3pFjUimLNzDl5bQde/tiNq0UCQreEoc2Lz4EepqZA+2HXc8LwEoYevWWjUdpaYaaov9yF/P/P3WL1vtaMyOK0AjPz4BnbxYRfj/TPeB2DMhcEgNjZxQkQ8GDRqwaLmcxrzUIiiW/+y+/Dr9QEwWXKRSAQ7a3bqKSJ9qs/0HVgOZV6O4a8vHl8fqsfsHbOl5U0VGhP1I9YBKVFHwtLwvLAWhqqoUkceS8okQmZRq4qK5eXL9W1JJzQi5k3iu2Hk/IfZf8Ax7x/j+NJieKcsaoPVUzFMk2+vfNgqg0xC4jXqVFjUbJtL1AkiXtSH3ccR16HjP/R3W7/DPbPvcVQGpw2cVW6+Aka0CAp8BZbniNEWPFE70gZ5jVo1atSFmYXScxSiGBqUGJIVl6gRMUSYyKQedk0ZOck02Z01O3HrtFsNGn2qkWyMsFT68MSkD96i5kmX16hVYkHUdlPIFYGoBWJLdnUU0ZLly8rw4tIXMWv7LHy/9XtH12Tni9a3rIxWFjXf8bB9dgsz8PfL8mSlRKNmUR9hLZyWo7z0J2qLOGo7i1ok6mRjRBMdUjEnFEBD10RnHY+aQI3hN19ZnVQ8/p0wa27LgS2o8FegZ35P6TnMCvn9kb/XJ7vwDYKtOsPgVbwGyyaiRbC1OkbUViFfVvv4PBYiUr0uIEBX8Rj85mBTtEY88DMTxW/B66ss8VBYCxsmt/DatRUhi9IHIygx6sOrmGdzJpt034n08c7qd3Db9NscX1MkNSafyaQPNlPTblkuqfQh6WgbS9QaNIPhoGma5YSudEDaE7Us6gOIVbAH5jxgWL8QAHbVGjN1iefzhGqHRCsAa2Djuo7DFQOvsM3+xcLYGAwWNTT0LuiNm4bcZHm+zKL+Zss3ICA4v6/cG82WLMr2ZsNDPIhoEUMjEck125tt0NfDWhibD2zWf/ME8t9T/4uTup8k3cfgUTx4cbxZBgKSt3qtUFpbivM+Pw8A9CnziUImffAkY5A+LDRqlajSGYqiM5GRBp/mFKAWdarWG5TV+0bP+hWIviirCIC8M2GdnJ1FrUd9hONo1FGjI8ebk1TStogWMfmJrCZ0pQPSl6g5jVo6FIlu+nzj56ZcyldPvdrwWyQBcf02K7AP17NAbqGKYA1heIfhUIhimesBkAwVBYuaEII2mW0sz+ffCSPFmkANMj2Z6JjTUXrOzO0zAdDKrSgKQpGQoUGI5Hpp/0v13B3snny6Sp6csjxZho5JOqwnKnoV9MLNQ2827UvGITT6vdF4ZvEz0n1//fmv+t/JLl4qcybydYdflssq6sNS+oDR0makIY44MtQMExElq1HL6n2qiVqf/yAhO6bv261Irksf3ChYZuEyZ2KONydp6cMgVyHS6ARVTYm0JWpWca2iPuKFdfHkLL54x0QdieWtcIRokVhlk1kw7bPb4+LDLsYjxz5i2C46ExWi2N6Xb7ws6iMUCcGreA3E0Kewj+ncXG8uVKJSizpsbVGLHVRYC2NvfSy8j3egZXmyDPqtrOysYcjkj2SIuiZYg//++l/pPv4b8yF1iSASMRM1j7BGV2vXoBkib3iNWlEUqWNVlD70dyNYeT7VZ1pcIlGiZu9CplE3Wo8VTmd1Xip9WFjUfN3nDTQGq0WmPcSjh4GK2Fu31zRTmIeJqLl37EofCYBPcyrN2SzRD3nwyf1NFrVD6UPP2SBpaLLICtbLW+mwALUAHhz1oElH9keMzkTZkJlvVPzfvKdcDO/68+g/m8qQ7c3WF+vlG4loBZsy3WkayutiU395chItaqlGTeRWI5C49BGPrPjRiNV6jVbQ1xGUWNSGBXsjYb3csqgPJn3IpoqLE17Y36LDVUxuBQA/7/4ZV359ZcJatVT6aGSud7Ftsncn+z7MorZzjsqciVZRH5meTChEkfLAld9ciRu/v9Gy3GKSMLvFitMBaUvU7OXbWdTiSis8UfCWX7JB7Oy+sjAqGVEz0pNZLgxWwz+D9BFd/FRs4LKk6ex4IGZR89eWkUSON0fPH8FbPvGIOhgJYl9DbOovT06Znkyj9GGRx4L/l0ei34iPXpm6eSpW71uNZxc/q3dgBRmxiJtkNV2ZRs2TPj/FnH9e3ploG/XBnWMVEeFTfaYyvL3qbSzduxTba7Yn9DxNIX1YSUN20ofYYRg0ahIbSU9eP5mujmMR9ZHlyaKJviSjAquFh/ly8+063Yk6bbPnMSKyWjhg04FNuOTLSwzbwpEwhrcfjiVlSwzZucQXn7BFLWlo/Cw+BidEbeX5NkkfUJChGC3qsBaGCvNipPrCq1GLmodMK3YqfYjX2tewz3J5sixPlkFikDoTibX0kehwnnfG/mF2bOWViw+7GJ1yOxmOjZfE3gqyUZvYoGWjLj7lq0pUufQBufTB6mauNxc1wRoqfViQaaJ5mJtC+jBZ1HbSR7Tu28VRszL+vOtn/LjzRyzbuwyXD7jcdC1/2A+f6gMhxLaT/2HbD2iT0QYjOxlznUQQkfoVAJeoEwIjHyvpoz5Ub6qoIS2kJ/4pry9HKBLCo/MfxfD2ww3HOdWo2ceTEbUYygbEwrVsLWqvxfCPIypN06QadSgS0svCW/t8Avt4VjEQTUYfXTWEtzi8ihdfnPNF7LdgFe+u2W157Qw1wyh9SMiJWdKyEQpPpn/56S/I8mTh3pH3mo5jEEMwGbZUbUGn3E6G9yk6au2+Dw9ZHLWhg+RmssmSMrHwPCuLmpc5WJnY9QsyCihRK2aLmqE2IM8hDtDcHSJSaVGzBXrFtmm3EEEiUR8sFcG++n3S5w9GgvAqXstlypi0d/esuwHAkEaYlZNvg4a47TQk6vSVPhDznDtNUhSOhPWlnvbV78MvZb/g43Uf4/4591ue92v5r7hnlnwCjD6stSA7ETKN+vNzPjcck6XKh3+mpZoIMTVwfgjIGkimmmkg6ngW9biu49A+uz1URTWF53lVL0oKSlBSUCI9953V7xgImC8fISSuRs0aqIwo+cb6yfpP8M7qd/Tfsu8vhjcybK2icd78zLVkk8xHtIhp6MyTRjgSlspjvEatELkzUQzP657XHUDsnXfNo8mfgpGg5fDdLvb84i8v1v+2dSYmmWtFP084nTewRDgiak76EK/JQECoUaJ6pZ0FYL/GKWDusMWUvHZojgkx6UvUXM/shKhDkRA0aHrkQSASsMy9wVsWd868E99s+UZehqgeKWtoPsWHJVcuwd0j7ta3yaQPMXKCSSZ2FjXLVyyzqBlYZfEqXoP0IVZQkTDP6nOWfv+aQI0esseuxUP8XVZXhom9J+Kd09/Bpf0vhVfxGrT6HE+MqGU6tL5PIn34w35sqtyErzd9bdg+d+dcDH1rKNbuN4ZgWq1Iw0I1/WG/rony79aRDBKtHpHofzx+c0QscVeFvwJPLXwKAEyNfvL6yVhYuhAexYPCzEIoREHbzLZok0GdnIQQw/t96vin8My4Z/TQyifGPoHfDP0NDmt7mKVFzazOeNCJTDKQbMysTcO12fVsYpFZ2xQn3vCkzK6rO0qJOWpLIYqe31yBIn0/Vm2fv4+VM5HPr251rn5eJGzylTUF0p6onU7pZGkPWdRDMBy0jL7grVkZCTPIHEUMbP0/Xpdl0ocdSTHJxE6jZvmKRZLlGxVvyYW0ELZXbXckfbDnVYmKubvmGvZ1yjFquzKrZGzXsRjabijuP/p+EEIw+ezJ+HjixwCMYXB2kS+yRrSnbg/OnnI27v3RKHfM2D4DALCkbIlhu4yoO+V0wmcbPkNpbSn8Yb9ODMla1FsObDE0ynuPuhf92/Y3HPPFJioViekDHvrpIQD0O+f58jDnkjmYfuF0/R0pMEoiBRkFOLnHyfrv4qxi3HLELZZRDYC1RW31jLKZiclGfVit7KITrV14nmJtUTPit5uZSEjMolaUmDNxfcX6uAnS+PLzPMAbQZd+dameXEoG/nu8s/odHP/h8Vhfsd72fo1F+hN1JOyoMvH6MEu/abWMlSHvss00aFYGq1l2gDGcikkfdhooy6cR15kosaj5Xp9VaK/ixfK9y3H65NMxd9fcuNIH63RknUmfNn2kx/IY2Hag4XdxVjEOa3sYACNR2/kBZO/n9V9flx7LT+dm/07bNk26Ms2VA69EKBLCrppd8If9uhTDE4EsN7gVJm+YbGiUTqJ5AOAfi2MLSDACz/PlGRI0WWnXMiRqUe+p3SPdnkpnImuTYtvULWrZzEQHzkQZ0YvPrxJVHz0qUHSJ6rzPz8OdM2imwnhEzfwH4n0Z/rfuf5bn8hb+9urtcY9PBdLWmchb1E6kD9YAPYoHXtWLGdtn4K1Vb1kef/XUq6V5FHjYRX2w83gSYJXLzppkRC0eUx2opkl8otoxew4efOXXpQ/hGBNRC/sZ+cvKyHRS/VzJu7Ejl+KsYgDAFQOusCU1q/fTp7CPKS+HnsQfGg74D+CumXdZLk/GRgRXTb0KAHTrl/9GpbWlhnSromVlB7vGX5RVhGdPeBZ3zLhDWn4GRlai9GEHq/rPryLPo7Su1FgGi05T5ix1Cl360JxLH8yiFsvDdzjiyu4ExGRwKURBMBxEhidDH3GwjptlGYz3bpnDXrwvfw8r8BY+y22SzEroiSBtLWo+QYqTXl+XHaKrO7OeTgZCCBbvWYx5u+dJrUYGq8TvQKwiyCw0u4+c68vVyyDikfl0tqIGSh7ts9pjdKfReniSbFULcYKEWEE9xKOnWwWM0gfDLUNvwZjOYxyF9tm9r7aZbTHjohm456h7bN+BFeF1yOlgGkXwFvVDcx+yJGkApqnzMumDz12yv2E/hrw1BF9t+srymjzYO3t7wts4teRU077juh5nOkckQt6itnuXdtdg4Kfz84i3zBVDWHM2Wk2kTMyvI/MRWc2+ZA5ggFtFiNOt+dS8AOdMjM4ZiGgRXQbSNA0NoYb4GjWMzsREtHqe1O2knlQibYmaD3FKxKJWFTVubyouOcWDvxerdLL0nOwezIrkYWdRW83OAoBZ22fpZVBAG/KkUyZhbJexAIQFcKPSh1Vj0MuiqDi8+HD9NyMKvrHcMOQGvHzyy6by2HVQVijOKgYhxNZKtSLxoswitM9ub9jGrhPRIoZJTADQLqud4beYVpZJMTxR805JttrN4wsetywrAEzoOQFtM9vi+G7HAwCOaH+EaWq+VWZAMdSLdUQEzi1qq3CxVftWSY0YK3lHfO8RLZK89GHhTGSyQDAc1COchhQPwbFdjo05UgWLml8vlJc82bFiKKbuTFQ8lKgRs6jrQnU4Z8o5zqQPWEsfthY1H/ljI/WkEnGJmhCSSQhZQAhZRghZSQj5a7xzUgFD1IeDXp8Nj5hFbQc7jVr2EaxW4gCA03uejkknT8LFh8XCoew+MrPyCCF6Oc7tcy4A6IQqDsuY1lobrMWi0kW4/8f7LUMH4z27rlFHSYWAJLQ6uFNykTmuGKzu1yajDTrndtZ/M6cqQMlHJMJL+19q+C2mlWXJmHii5q1Q1gmIWq9IXv3b9sesi2cZOmVTh0hUqcRgRdRWYXsyWBkqlf5KXPfddabtVs5EsXx8eGGieHHpi9hUucnWmZjhoc86pssYvDT+JWniKYUo2HJgi+l8vh2KRK07E5lFHYkYZKCdNTtN30e0eMU2lkg+IN76Fpenayo4saj9AE7UNG0ogCMAnEYIGdWkpYKxZ3XS6xs06nhEbRP1YbCoo3/LHG+M8AghGN15tDETmgOLGogRepvMNjii3RF6Z8PiqMVz5u6ai2u+vQZfbPoCH6z5wFAOhniWhCh9eBWvY42WPy8eRM++4RoWUTFtMtvg4WMe1n8HIgG9gdUF60wdIHNiAnRCA58UCoApPC/bk42t1Vt1q9puerGhvJJnFrepRJW+R5EADBa1Q+lD5kzs16YfALoos7iCkFUIoli+Cn8FFpUuclQGEe+sfgc3fHeDyaL2h/244bsbsLNmp/6ORDmLrxtdcrtgX8M+Q+w5wGnUhJgifJhFzYh644GNeG/1e4ZjRA4Qde54E58cW9QRa00+lYhL1BoFm13gjf7f5BHfev4KzVkctUGjjtMA+N5SPJa9+PpQvb5it6yhmhIWca/E7iOzmZP8cSpRkenJ1B0Son7GLOq3V72tb1tRTmdaxZM+ROgaabSxOI08YHBK6nYWtRXZt81si865nXHvUTREryHUoL+TulCd6dky1Ay8cNILmHTyJADmSUgZagY8xKNbO7m+XJTWluKCL2j+citJQRzByb6n+AxW7128B3vfYS3s2KKWdSidczvjjF5nAABeWmZctFa0qNk3E5/jiq+vaFRKz5Bm9h/Vh+oxb/c8ALEFm8XvwtcNcdTD561hEDVqBZz0Eb3WrB2zDMeI7VPsvESLOhHIRt2hcNPOZnRUUkKISghZCqAMwPeaps1v0lIhpg87HZ4xR5sTjZof1YhExe516sen4pXlrwBwptXy1pwVEf1z3D/RJbeL/ltf1UPxINOTqY8KNE0zVGa7BQgSlT4YOTD5wamUkSjsGoFV+lamKbNv4g/7Y0Qtsagz1Awc1/U4jO48WnrPTE8mVEXVG6k47d8qEY9IPlKiFkYFVs9rJX34w37H737SKZNM23yKz/I9OpU++Hw4ycCjeGxnNl7U7yIAEouae1csCopZvHoYJnfdumCdYWIVL31YGQ5imxWjMsJaOCk/CmDsOJkl3ewWNQBomhbWNO0IAF0BjCSEHC4eQwi5kRCyiBCyaO/evaZrJAp+CrkGzZL82GomzKL2EAfSh40zMayF8ZsffoMKf4W+TXZvsSJcc/g1+t9Ww/7D2hxm+M0qpUIUZKlZltJHIkTNynVCtxOklU13JjLpw+HwO1HYNQKmXTL0LOiJKwdeiTGdxwCIWWD+kN9gUYvkGC9PeJvMNlCJqhMXi7hh4EnUbik02XsUdXar+ilaw3wn5HQ0c2SHI3F0x6MN2zLUDEPd5e8jsx4B62/Sv21/3HaE8+W3GLyK11KW/Hjixzi7z9l6WXnw75NZ1KzuizIPcybmeHN0KUvTNEOuD6uy8WD8wCDOTBRhp1HzoxBWb9KCqBk0TasEMBPAaZJ9kzRNG6Fp2oh27dqJuxMGH54npiRkOKrjUfrwj59skghRi5U3rIXx484fDdtkH5RphAz8wrBWFcCKwJlFbSV9+FSf5dDaKqTuXyf+C8v+b5npeHHCS5NZ1NGqJc2TIqx8k+PJwT1H3aMTKWvYp08+HXN2zgEQlT7EXM0ee6I+s9eZ8If9espLkah5UnhiwROWscGy7yl+S6tvLkoL7NkD4UBi717gDZ9qtKj5+1hp1FZWYkSL4KahN5k0/nhQiWppURdlFel1TJQ++HfFvgmr+zLirw3WItubja/O/Qo98nsgrIV16cOpRS1GwrAQWCvY7eNT3aYNURNC2hFCCqN/ZwEYD2CN7UmNxCPzHsHq/asBxMLzpJMvlBiB6VEfigON2mLqKEArrRgiJlpyD41+CEd1PMry+lYNQmzMrJKrREWWJwv1oXqs3b+WrgwiXMPKqk5UoxZTjSaqUTsFe2eyctslmwLMRA4A9cF6qfRhhfdOfw/ts9sbyNgkfXBW6OQNk/WV0J1o1CaL2sJBaqVR+8P+uN+KBzMu+O/Gv0er0YHsGlZlTFSztZI+CAgKMwrRp7APzu97PkZ0GGHcz7U/ZuAw6dKU35oQ1IZqkePJQbvsdhjbZayetdCrei3LbCJqwZkohueZnsGGqPk6xfwfzR6eB6ATgBmEkOUAFoJq1F82ZaE+XPuh/jfTqK0mX7DtzIrwEE/ccDO+wpryDUfCpo8qEqzVKt92S00B5obArAfmTKwKVOGCLy7ApgObTI3Kytqxkj6sIDqWmsqiZuWXEbVIxGIll3UeVs5EKwxuN9i0jS+LpmmWIVmOoj4EYra0qAWi1mWdsD+haBuGvm36AqCTWgwWdcTaorZyJjIw4km0PB7FI7WACzIK4FE8yFAz8Jdj/mJa+9NgUQvSh8wfxaQPds+IFklY+hA1ak3TbCOTWpz0oWnack3ThmmaNkTTtMM1TXs43jmpREgLoS5UJ23wfPJ1fsJLIquFiBpiRIsYiNqreE0fzcp6YgH+Vg3CRNTMolZUs2fcwqLmw/sAa+kjHth5TiMPEgWfAF9EXItakkK2LlhnapR2iwfLwMtTfC5pBvY9RPKREZgsPE8G8R7sfTjNic7A6sOgokEAgI2VGw1EzRNFooswJGtRq0SVEjX/nmWwcyZaEbWezCo6fZzJoU47F5Go2aQyJ2XksaN6B77d8q3++2ARddrm+mAIRUKoDdaa9EWATlTQp3JzE14SSfzNQvD4+/HB62wVCR5WHzHTk4nqYLVlo7WVPlQjAYv3ZASd480xVDqnFvXfx/4dP++OLfbZ1NIH6wClFrVAxCarUyDgPF8e6kLyqA8R75/xviVR8Z1GMBI0dejsvKQsaivpQ+iELux3IfY17MN1h5snqtiBETvLX3Ji9xMtpY9ENWpGMnbEJYNX8SaVC9rOmSjKTt9v/R4AMKBoAADoC16w+1iVWexAxFjseHHUVh3pRV9eZEgIpksfhzpRh7UwaoO10ga/v2G/Tky69KF44r40ngR58gJiMZs+xYdARO7wsaocVonR9fOspA9FNTV0kxVPYpovH1blNDzv9F6n4/Rep5vK0lTSB9McZR2sSLCi9MGS5jOc2O1EfL/1exOxyjolfrq8CN53IUvIL8byMjQm6kPshLyqF7cPu92yjPGQ5cnC/MvmI9OTicnrJ0vvwy+UzMNS+og0QvqQEHU8+c1A1MyZaKFRM+jJxLh2Iq4PykMsV13IOLsxLlFbvAsxayOru80ufTQ3whFrot7XsE8nGmZlOrGo7Yad++tpEnA+y53JorbQtvTsYBYf2c6iZhVATOupn2vhnEtUoxaPa6rwPGYh8YsJiPdmEK1Ofui84PIF6JTbCfWhehOhJ0os/HcPRUKm+zKidhL1YSV9LL1yqWF7son5TYgWXSEKsr3ZpjSpjZI+tFimOjssuHwBHjv2Mf23qsilj4SI2oFGDZjDStl97CJZeIjT0MXIKhFOpSk9jjoNnIkHFbK0iVZEXR2ojlnUXNL+uERt08Bv+uEmADGy4HNyMFhZT0y+sMqDbRcNwiQYFnEi9tDMghPfQ6JRH/w9gfgW9aSTJ+GRMY84uiYP9j1yfNYx4Ax21kiWJwvZnmxo0CyX33IK/v3vrduLh+Y+ZNivW9QO4qitpA9xe6rW35Mtp+XYmWizFBd/bo/8HrZlyPJkGeqLh6TOoo5H1LL0vLYWdTzpI5L8FHIerFNMh1wfBxWmBWsjIdQEakwENaFkAl446QVTulFew7rnqHtwYrcTTfcQibd7Xne0zWxr2MbyzMqOt+ptmUVtlb3MSsdUFRVXDLgCh7U5DOf0OQeAmbz0hWGJPTE7TppEnE0hH915tCmlpxMwS10MdZRBRmZTzpmCl8fTjH4s4sXp0lNW4LPtXfDFBaZ3nAhRm76DRaTRxN4TkyqrFZwQdSIrvAAxq/+5E57Ds+Oetb0/b+RYRX3Ei7oyOBOjIZP6HIJ4FrVD6UOUr0zSByK2VrNTi5oRtNN0zMki7TRq0WoKa2EEQ0GDI+h/E/+nO1WYZCBzJg7vMByapmH69umGa4pWQFFWEe468i7cMfMOfZu+EgsUkwVuZVFfMeAKLChdgN4FvaX7LWM+iQfd87vj47M+xsfr6LJWInmxe4qyS6JTyPV7MunDwfHJ5EQ4q/dZqA/V46J+F5mSBomQWdS9CnqhV0EvALGp5bJVXRLBid1PxGklp2HqlqnS/SyHuVg/HE0h577LX0b/BYFIAGf2OtMUpZMsGHEYEool6Ey0GkmycwszC3FSj5MM+8TFHPh3oSryCS/x6hTffpiME4+oZRa1KH0wv5LsOqJFrWnWs50Be/nSEEfNdYostrspkH5EHTAStT/sRygSMizzxK9bJ8ZR89JHlpoltWJlqxqLjVHXSYl99i8eJ3Q/wbQsPQ8n8dXsXsla1PGmVTOw53OiZyZD1B7Foy94EA/x5AFmUVcFqqAQBWf1PispAvQoHozvMd6SqJ9b8hwGtB2QdPY8hvP7nZ9w2eKBEQd/HwNRS2J7ra4hwiox00dnfoQueV0w5v0x+jbeKvcqXunCrnZrhgLG+uRRPMhUaZ6bhaULDbmpeUilDzWW6+PKgVdi/u75+sIQsmRRPMQ0DSKsLGqP4kE4zOX64LTpYCR4CBG1YFGzRiyLxwViRMV6TJ/i08+xcjaIVkBEi5gql74SCwhOLTkV26u349XlryIQCThO9SnCUqNWzI3PRNTMoiYKPjv7M5wz5RwAZqJ2Gm43sIiufchm4yVT7oMFlme6JliDMZ3H4G9j/pbUdZzkKr/5h5tNx0iljyR9AyIKMgoSygltJV+wOv/91u+xs2YnvIrXVIcSDb9jIXE8eHJjoXMiEtGoWbbLYDiIa7+91vIc9k0M0geJTXgR26T4Tr/f+j0GvzkYv1z5CyVbzTz7l4fTgABem27KyI+006ithm1WU6hZwna2FE+ON0dfkinTkyklVTHMSxaqwzsTPYoHNw+9WS9DohU+Hvi4YWY5WEkfYhRKY4ma5cGwQ3MTNT8rM1lCBJyvU2jVSdptS/YdzbhoBmZdPCvucTLpgwcrM8u1feOQG83XSGImpAgnz5mIRu1R6EzieJPULC1qmEcagPXq6kyrjpfmVGZRhyNhM1FL1kxtCqSdRW0VMcFLHzwYkVb5q/Tjnhn3DOaXzkf77PbSjyHL72GaCRgNK5N9MLupp8mAX2yVzRS0kj4Uopg83zycatRtM9viyoFXStf5awoMaGu2zhh+M/Q3tufyMkeycd/TL6R+imSIXjoz0eEU8nhw+jw6UXP1kR/eszodioTgUTy4achNyPXm4u8L/266RmNgR24l+SXYUrUlIYvao3igKmpcg0EWnlecVRzL6S7KLRZ+vYZQA/J9+abwPA/xGCQg2Te/+MuLUR00+kn40XlThuilHVFb6Wt8FIYIr+LVe8pMNRM53hycVkIT/MkakEiCYc3cUxZlFQGQV+5kG6UV+OWndOkjLA/PUxXVVNF5ONWoARoVczAw46IZliMiO02foUN2B3gUj05CyaBdNo34SIbopdnzBMJKdZ2wAk8Mg4oHIUPN0P04AAx5ms/qc5aRqFNgUVuR/fzL5mPatmm4f879CRO1h3jw066fbM/hlzBjaJ/d3qDd8++Gt6jZArgAF10ihOflZ+Qb9HbTQsCRMNZWrIUdXOkDMaIe3Wm0aR+rGFkes/PQiUXN57vO8+Zh/mXz9bAyvnKzv1NhmfDgK3Y8Z6JK7Im6qaaEA+bFZJ2iOKvY5Pz77vzv8NW5zlb/LswsxCWHXQLAOirAKWQk8vqpr9ueI/VzWExIajJIqlyWJ0tfYZ7Vl5AW0jt1UaJLhYRldQ2WzB+IP2oxRG4Qj6POl7ULvsNvk9HGMtc2X0/4uscmvogWtWgIiiOXNRXxE4YmOtEoEaSdRW1H1FbWF7OSZJEAsoolWqu8M9GrepHtzba1vFLVKPkFcRkY0VqG5wnSh6gHNtWU8NdPfR0l+SUpu16n3E4JHc9W/Wa+iGQh88rbpawF5N9bdEgnskBwMtBnrIr3jZJcaW0pzp1yLjrkdNCfkdV9PXteknbZCye9gNLaUsO1ZOVj0kHcDI6crkwIcdSeWLvg18lUFVUnZNF44tt4ppqpBxvwYYD8++jXpp8h4oS954gWwdC3hlqG3ALUh1IXqjNFlqQSaUfUVr2SXUYuVjFkqUCdSB+8Rs2Ijl1TZj0nalG/d/p7WF6+3LT9wVEPmrZZRX3wKVT5zkfUy5vKoo5HZk0NtgJ4opNeHj7mYeyo2aH/TqYjk31vJ7HWTQJBe2X14rMNn2FD5QZsqNyA9lkWk4ySHAjyfgwnFrWTxaUVosRNC8yDWdS8P4eHQhTDKIdvb7wBxxM13+k8POZhjOs2DvfPuR9ALISXPdPGA9bRUSxpmDipJpVIO6K2sqgdEbXE4Shz/MmImlUWcSKIzIKwWydOhsHtBkvzI8tgJX3wThMxvEl2fmtDcXZyRH1u33MNv5PRuKVEIlSBpiZqK+OA1VN+BGb1jKmIVrK6BgHRyc3JO1YQI2onx+uLMhMFZ/Y6U/c5MIva7v2z8E7ASNT8d83x5mBi74k6UYurodshz5eHPXV7TJNqUom0I2qrOfN2H5NV1lRa1DpRp1iPjod40ofJoiYHx6JubhRnUqIW09ImCitr75WTX8FN398k3Sfr7BlRMKTCUecEVtIHX6dFAmTrLVqRWeecztLtMlhKH5xF7cRC5i1qJ7IR/90eH/u4/rdVGB4PPoUwv+SXHbkzonaS257p201J1GnnTLQKz7ODrUUt+RjiVGQDUauC9MFVTDaEakryjid9NJdG3dxom0VzsViFaTqF1fs5pvMxht9s0WRATjzts9vjuROea1RZEgGrh1baOC8Z6otCqD58fs7nePL4Jw3X4PHcCc/ho4kfOS6HpfSRgEbNrqMbHw40akvy1+zLBcilj3irkDOCtrKo2TwEgFtO7FDXqOOt5JGoRc0wsddEfLHpC4S1sD5ss7OoJ508Cd9u+VYP3WsKWEZ9WEx4SWQdwZYMr+LF38b8DUe0O6JR13Eqfdi9Y4amrAciLKWPqGFR4a+IbeM6o54F8mXjGPoU9kFBRoFh29Tzp1oSlB1RH9nhSADQV5O3Q6IatRV06cPG5uQXqmBRHxo0KFAwpvMY6bR1RtQyi3pIuyF49eRXcdL/TkJNsEYn6kNK+pBp1PkZ1jHUAEfUDi1qBraKNVt+HrBvyN3zu+OGITfYlqWxYBNeDmtzmGE7H57HV2yxkrdW6QOAnlmwMXCcXZBr+FYkkOoZqk4ghgWy+upEo5ZB1j665HaxPN6qwyCEYFj7YVhyxRJH+S746fxOvgkbUYlg0oeddWzlTFQUBS+f/LL0nDX71+C91e/hxO7m7Judcjoh25uNwoxC1ARr9PQWh5T0ISPqB482R0fw4OOoRfBEdm6fcw1x2MxSZ8vPA7FKw4aYB0t7ZCCE4M3T3sSkkycZtuuxsUSxjfpo7une6Q5WV+LNyDRY1BYzUVM9Q9UO8aQPw7YEiDpR56qVpc0I3GlSIjajGIgvfUy7cJopDTGDHkdtI0fyFjUvfcTraB9f8Lj0eZkVzxbt9SgePUSvqZB2rZqXPnoX9MaSK5bghO4n2J7j1KJ+eMzDho/GZAJN09Axm+YHOb7r8QBilm2H7A7JPEajMLzDcBRmFhq2scrMT84BDt6MuNaCHG8OPjjzAzx13FO2x+2p26P/bZn1MA2aj6qoJpKKR7785I5EO3YrMkrUoOFHhvHqcFGmtcTEOi675+Cl03i5Pv5zyn8Mv2XSB+scCjMKAdDyZ3uzTavIpBJpLX2EtbCjHtqjOo+j5jsCJn1EEEG3/G6YcdEMvVJ0y++GR499FGO7jE38IZoArFJpmuZIP3VhDbaStx2Wli3V/3a6BubBgNWKKrKoDxlePOlF9G3TFyd/fDKAxDv6VFmNvEUddyajjcWtT3ghxDJxP++34ReHkH2/kZ1GYkLJBHyz5RsA8qXURItaVVTkeHOalKjTrpUHwgFdwnA6XdhLbJyJwkfmOwLW07L7FGcVG0jwrN5n6R+juWEVO+1a1KnHhf0uxKPHPqr/Tgei1q1mCRexyUAMdprv2K5j9eySQOL1p2tu1/gHOYBKVEMkE2CvjVvBifTBdwRsJRYN1uF5fHImWf4OposbLGpPtr4wdlMg7YjaH/brXmgnyzgBMV1MJn2IFZEnatbTNjZ/xMEAXxHtJry4aDweGv2QQcN2suBDU2Nou6EAzCu0A9ATkLFjEtGdE02HcHjx4fju/O8wvP3whM4TwVvUDEPaDUn4Ok6kD/4+wUhQnzRllSiMJ2cn0gdb2ERc9CSVSEvpo3ted9x15F0Y1WmUo3OYQ0VmUYsf0GBRe2LOxHSHVa6Hg+nQOlSRDhb1lQOvxNiuY6XhdrcPvx2nlpyKbzZ/g2V7lyUUS59MR98pt1PCs3Nl92XtlhGfGIZ7VMejcPOQm22vw+f6sCqTaFFvrd4KwHoxX96BaCd9MIOyOliNPF8edtXssi1rY5CWRJ3lzcKEnhMcn5OsRa1HeDThopSpgpXXn38+u5zPLsz4y+i/WKYs4JEORE0IsYyJ9ipeDCoehBnbZwBILEFUsgnGGjsK5eOoWZ0W5wB0z+uOkZ1G2l7HSXSWaFFvq9pGr5/fXXo8b1HLlilj0geLn67yVyHfl4+1Afs0qI1B2pljwUhQj7hwCrukTGJj4mc+suMT6RTSDfzzJTLDzAVd2/CyAZfpv0/pcYqUDNNB+nCCZEaIyT5DYy1qnqgZxPBaJxKOVZpTHvwIIxgOYlv1NihEQbdceYInPvOeNBwx+ugseqY6QC3q3bW78cGaD+KWORmklUW96cAmlNaWWg5JrJDIhBcx6mPOJXMstap0hCkPsqtRpwz/GPcP6fZ0sKidgBFdIkSdbP1p7CiUdyYy8KGzgLOJMLpFLXEmFmUWYV/DPhxefLi+LRgJYlfNLrTPbm8ZUcaTs10ctW5RB6r0v19d/iou6X9J3HInirSqaRd/cTGqAlUJB+EnkuaUH+oqREFBRkGj1uE7WLDyaqcbWbRGWC5KnGadJCNqJxnfGJKtP42VPgghMY06SrYdczri8gGX6yPcRCxq2XOM6TIGK65aYUj7GoqEUFZXZhuoEM+ZyKQPtpBG59zOyPNSom5sLhorpFUrT2RKqew8J2lO+Uqcbg0tGTR1wnoX1vXkYGdWjAfmjGvKJaEYGit9jOw4EsPaD2MXA0Db8X0j79OT9DvhAauFA3jwOnwwEsTeur3WObshELXEmcg6h065nfDKya/g4WMe1i1q2ezoVCCtiFrMBZ3oefGmkAPAWxPe0v9uDdaoG/XR9LC0qJt6+a0EwaSDRCzqZNFY6ePBUQ/i6sOvNmwTU7Y64YEBRdSB3i2vm6nzkMVYByNBlNWXmdLU8jA4E22kD4BmXcz15ep6dVMRdVqZY+LqKk6RoWaAgDhyJg4qHoQhxUOwvHx5i7KonUR9uGga2GWMSyccVKJupEUtA6vL+koxDmYlX9b/MozoMMKwRBcDKyOfHbAmUIPqQLW99BGOE/UhkX1Yp80vUpBKpJU5lsiKDzzO7XMu/jHuH9IPKyOydLOEnECPoxYsmdYwKkh3tBSNmhkqLcGiNlwrSqiiRe1E1iOE6CR9xYArDPsYoWZ7s7HiqhU4o9cZ2FVLY53F2Zw8rCxqlhhKlsWRRar1yEssEMIp4r4JQkg3AG8B6AggAmCSpmlNkjFdT9qfoO7aIacDTs45WbpP1shYA2sJE10YrCxql6ibHpYa9UHOrBgPrP20BI1aBlaXE7GoeVx02EU44D+Af/3yLwDmMvJSCnP+ycC/Pz6ct0N2B8y6eJb0nNGdR+PhYx5uslBfJ4wYAvB7TdOWEELyACwmhHyvadqqVBcmWenDDnYW9cGwPJoa6WbVtUZYEXK6vXtm4MiG66lGKtMuiISaiEUtgje+rHJ3A7GEbDLwvMAnobLTzAkhpvU5U4m45pimabs1TVsS/bsawGoAiWdPcYBkpQ87yCxOvUK3BqJugTJOa0G6jWaYBdripA/B6cdWvB9UHD/LoQgDUdtY1HYrIY3rNk7/uymX10oECTEiIaQEwDAA8yX7bgRwIwB07y6fmhn3+tEPdbAs6pYkfTC4E17SB+lG1CxF78k95DJgKpFK6UO81oSeEzC602hTTnYn4MPpRKufJ2q75f0ePuZhdM/vjheXvmhIXdoUco9TOGZEQkgugE8A3KFpWpW4X9O0SQAmAcCIESOSeiI2xEypRS0JX2OpGlvSjETLpExpRhaHEtLt3RdkFGDupXP1paGaEk2dcTIZkgac5/ywkz68qlfnCF76aM4RuKOaRgjxgpL0u5qmfdq0RUrtStoyi/OuEXfhqeOewogOI1J2n6YGS28p5qJwLermQ7oRNUDzT6RjuZwgFc7ZqwddHUv2pFlLH1mqfRgdk0b41KWltaWNLl+ycBL1QQD8B8BqTdOeafoiNb1GnaFm4LSep6XsHgcDx3U9Dm9NeMu0CndLbZStAS353b9w0gvYXbM76fObUvpoDPJ8eXj4mIdx/5z77TVqG4saiGXWW1exTt/Gr/R+sOGEEccAuBLACkLI0ui2+zVN+zrVhdEXyGxii7qlQp9yy6E1PV9LQ0sm6niL+8ZDOi+2wdqEyaJWnTkTAbpeq1fxYm1F06UuTQRxiVrTtDnAwZ2C1dQWdWtCusXyujg0kEqivmnITVi+d7m+Qk1jwfxSLHkSAx/uJ2bqE+FVvehT2Aer968GABzR7oi4ubGbEmk1hfxgRX24cOEifTCs/TD8dOlPKbueE4vaSc77zrmdsXr/aqhExdunv52y8iWDtDI39aiPFGaEcy1OFy5Sj3SWPtgo2k6jdsILbMp4Ohh7aUXUDC0hP7QLF4cymjOmOB4sLeoEfV+MqNNBPm3+EnBoCukDAG4eejPentC8QxcXLQ9HdjjS0XH92/Zv4pKkH9J5nVFGrCaNOkFeYUTdEG6Ic2TTI71M1+hoJNVEfesRt6b0ei4ODfznlP+YGruIr8/7Wm/QhxLSmaj19LNCERO2qLPS57umF1FH4UofLtIBqqJChb0+2S1PvkBqa0eime0OJqwyTVqtOm6FwoxCAPaZ9g4W0kr6YEhlHHVrxYX9LsTZvc9u7mK4OETx0viXcPPQm5u7GFJY5W7v16ZfQtdhnfCdI+5MTcEagbQ0XV2LOj4eGv1QcxfBxSGMngU9cesRt+LlZS83d1FMYBa1KFspRMFtR9zmeDTQJbcL5l02Ly1yAqUVI6bb0kYuXLhoedDD8yQ6+k1Db0roWulA0kCaSR9WQxYXLly4cAqrTJMtGWlF1C5cuHDRWLRGgy+9pA8Lb60Le3gUD45s7yzm14WL1o7WyCNpRdQFPrqsu+tMTAy/XPlLcxfBhYu0gWtRNzEeHvMwPt/4OYYUD2nuorhw4aKFwrWomxhtMtvgqkFXNXcxXLhw0YLRGi1q15nowoWLVoXWaFG7RO3ChYtWBdeiduHChYs0R1FmEQBgYNHAZi5J6pBWGrULFy5cNBbd8rvhgzM/QL/CxHJ7pDNconbhwkWrw6CiQc1dhJTClT5cuHDhIs3hErULFy5cpDlconbhwoWLNIdL1C5cuHCR5nCJ2oULFy7SHG7UhwsXLpLGHcPvQG2wtrmL0erhErULFy6SxnWDr2vuIhwScKUPFy5cuEhzuETtwoULF2kOl6hduHDhIs3hErULFy5cpDlconbhwoWLNEdcoiaEvE4IKSOE/HowCuTChQsXLoxwYlG/AeC0Ji7HoYdIpLlL4CKVSPR7ut/fRQKIG0etadpsQkjJQShLDA1VwOrPgcVvAqc8AhR2A149CajdC2hhoG0vYOSNQPk6wF8NdBgEHPNbYMajwPYFQG4HoN+pQM0e4MBO4LDTgPmvAO36AzWlwNE3A8F6eq+cdsCqz4D6SiAjFzjuD8D+zcDuZUB+Z2DqHwF/Fb1XfhdA9QKeLMCXA1RuAzLzgZJj6d8lxwKbZwNlq4FgAy13YXcgHAD2b6Ll9lcDgVpgz0qguB8w6Fxg3VSg51igz3ggs5Ce/+Ud9H7tB9BrdxwCjL4VKFsFeDIA1QfMfhoI++mz11cAhT2A+S8DwTp6j0gYIITeU/XRsma1Bap20e3H3gns+RXYtRTILAB2/UKfq+EA/QbeLKDHaMCXS/f1PhGo3g3UlgP1+4GKrUBOMT2XqLRcBd2A0hX0fXU+Auh5PLBxOn1vu5YCvmxAzQB6jQP2rgZ6jKHftetRQE0ZoKj0uxT1BYr7AiE/sGkGMOxKeu6vnwDhEBAJAvs2ANnFQKCGPis0wJsNZLcFMvKALXMATQNGXAsseYuWs+RYet2OgylZ/vIWUNAVaNMTKOpN7731Z+DTG4EhFwGhBmDha/S7FHSl77l+P71Wz+OBbkcD2+cDv7xD66YvFxh1C32eha8B5evpu8/rCERC9F+iAiv+R991cT9g30b6fsIBWi89GfR+bXrQuqCo9H6qj77vw84AytfSMncYCGQXAZ2HxdpPfQUw72Vg7ddA6XL6zQmh70TxAJ2OANr3p99zwzRazpx29PlCDUC3UfTf+v20Xaz9Olr2zrROdz0K6H0C0Hk4QBTgxdHAwLPpN+8zHsjrQOvG0veA2jKg+DBAiwAVW2g5Qn6g6wj6DfZtBI64HOh/Oi37ppnArKeAUx8BVn0OHHk1ff4v76LPlduefhdfDj2uYjOtG33G07qcXUTbiv8AsPoLoMPh9LqHn0/vzbBhGvDZLUCgjr6X8yYBq6YAoXqg41BgwEQgv5OZm8Ih2r5+ep5+i74nA0vepmVqP6BxvGcD4mS5mihRf6lp2uE2x9wI4EYA6N69+5Fbt25NrkQVW4EXRtKKAlBSLOxOK6YdjrwaWPxGtDAKrRhWyGlHyaEx6HQErUDBelpBFS/9yACQUQAcfi5QXUpJny97lxH0vA6HA4v+Y3+PnsdRcvNXA07Xf8vvQjuE8vVAXTltdA2VtBIDdF9xP9o5MCgeSp49j6Ok582iRHlgG63sPNr2AjLyacPtMpw+H2tAIT8l6Zx2lPTWf2c8d9gVlFC3zAEObJcUnlg/py8PCFRbP7cnk5J0/X7jNlaPnKL9QNoZ8mjTk3Zuue2pUVC2GqgU6ne/CfS5ZGXsfyZ9L1qEdoD+A8CAs4CsNsCSN+kxh19AiazHaEoGZavoPUvG0E59x0L7ct/yM7D2K/r9v7mXGhc8hl1BidOuXSSKzEJ6PfFeDJ4s2iYyC2gd63EM/TtQC6ycbDy283Ba98rXRc+N8+282dQgq9gs35/XGajeFfvdcQjt+M5+kRosH18HbPieGiUy9DgWuOJj2hYWvEo7wvmvUDIP+2PH9T8TWPMlvd8tc6mRkCQIIYs1TRsh3ZcqouYxYsQIbdGiRQkVEuEgfSELXqEVlkdGPnDJe9EerwHoewrtqbuMoNbRU31oL9fhcOC674G9a4Cl7wK9T6KWy8pP499/6KXAsvdpRe92NO0pP7uFWi/nv0o/UvVuYMEk4PppQJcjYz20ptH/dy4Gdi2hFpzqjV17wavUKjn/P8YPOfV+2iDPfoF2Luum0gr62c3AkdcAE5+l1yUEmHIb8Mvb1DrpOoI+HwCc8AAdSTDctihmiQbrKBkAQN1+oGontWIA2vh/fAYYcjHQ75TYfUTsXk5HJss/oqOUwRfEf5cA7cB+eYdabcvej71T9r7mPEPLWF9JjynuSzsBgDbYdd9SAvDlAnmd6PGbZ9PGOfhCoN1hwJBLKDF6MikRKCodudTtp++//UDgx38Ac58DjroOGPt74Ps/x8iRoe8p5k6FxxWfAL1OBBROKTywA/gnl5z+gT2ANxPYuYR+05CfftM+46kVyhCow8qF07HSOwgXjewJrPyMvveBZ1vfX9OA/11NR34Tn6N1tOtRQN0+4NMbaL0Tcf5/gI0zgKXvAKc+RkdjdfuBJ3uajz32Lro/HAS2zqV13F8DHHUt7XQrtwP7NwJDL6P3mnqv+RrtBgADzgRmP0VHwQMm0tFV5VagsMT47uorgb/3oH//cQfw8lhrwmW4ZiqtI7uXAl/cCeQUARe9BUx/BFj+ISVuEEA2nb2oDx19AcAxt9P69u6FwOCLqBG1czEw6DxaVv5d9jsNOOt54Om+kgJxRkVBd9opeDJpPRv/V3lbioOWQdSaBvy9hFp/g86jw4hB59Jhf4dBdMhohc9upRXylp/pUJDHtnnA66fSv2+YQQlz9lP097mvUIvxqOvpUG7DD3Qo7s2i+9d9B3QfRXtggA6V/VVAVmFizybg39PX4+nv1mHz46frKyYbsH8TJSdWDgCY80/gh78Aw/8PmPAU8GgHuv2aqcB/oy6E8X+hckYa4kBdEFe+Ph9PXzgU/TrkJXeRhihx840+GZStpp3/2q9pgzrqelr/5r9CJa9OQ2mn8vxwevzvllNrTMTOJcCrJ9C//2JhmUlQct9XAIAtT5zhvMx1+2n5xv4e8PiM+1ZNAbbMpR3c0nepNXgNvQciEfqMrJ5tmUtHVE/3ob/v206H/nw9DPmppezNwrTVe1CQ5cWIkqiBEQ4BC18FZj5B2yrDb+ZTOaV6D5U+4uHF0bRjvuRd4M2JtBPuNJSOztqU0N+DL6IG0YAzqSwjw4qPgU+uo8bTkjepxMWgeIFOQ4AznwVeGUu3ZRRQMm/bm967oCsdHWZE62R1KfD8kfRdArTD+0SYJn/U9cCEJ4GtP9GO8vSnaUfww59ph3X9D1RGTRB2RJ0+uT4IoT1wQyX9YMfeQbcXy3ozAac/SS0CkaQBWikZugynkkXP44HvHqCW1NBLYvv7jDee2+8U429FaTRJA8A/f1gPAKisC6JNjs98ALMseQw4K0rUV1HLjYF/vjQlaQDYsLcay3ccwCNfrYZHIXj5iiPh8yRIuKzDbCyYljjyhtg2QoBRN8uPL+gm387Vzfs+WY4nzh+SmvJx0DQNL8/ahIlDO6HrCX+UHzTwbPp/yE+t4lG36LvCIKisDaAoN4NuKBlD/x18EbVOZe/Uk6H/ed2b1ODSOxXVQ68/51njOWyk6ISkAeCWn2J/9zyOEvPEf1G/Rm059b+c+GB8wht8AfU75HWkBl2PMcDkm+i+7qOAq7+kf9+2mPoj5j5Hf1/2IdBWMrrI6whcOxX4+h5g209Gkh54NvVvdTmSjt56jgXuWk3rTiRC61WPMUmRdDw4Cc97H8DPAA4jhOwghDRdFhbmSGrXP7HzfDlykgZoBTr6FuD/Po/eQ6Ev+KbZjdKTGoM22VQW2XWg3vlJRb2p1dY12uFmFtChFvcM9YFwKouZUlTWBQEAs9ftxfQ1ZVi2o7J5C+QE571Kh8pWFnxGHnDEFbgpcAc+WLgdn/2yM6HLj3liOp6cusb2mC376vD3qWtw23u/xL+gJ4PKdF2G65se/3o1jnzkB1Q1BI3Hnv8qcNtCLNlWgUEPTcW+Gj8SwvArjb+ZxOYUvJV/7F3ATT9Skgaok27CE1LC21vtR0NQqOdstO3NoobXJe/R35FQ7JjiPtTYAahEKiNpho6DgWu/AS58g/7uNgr47S/UMu9xjKEj059BUag02AQkDTggak3TLtU0rZOmaV5N07pqmhbHA9YIDLmY/iuQbiSioeS+r/DyrI3JXXfCE0Cv4xtZuNShMJta0aUHrJ0loXAEtrLUnauAP2ygFaXPeLytnocBD021Pv4gY+u+WsPzMaJmeHLqGoQjDh2kzYUhF1G91Q7nvIBvIyMBAI98tTqhy++srMeLM2N12h8KY+n2SsMxB+rpe0v2XX3zaykAoKo+iC+W7cLjXxvLOGnWJtQGwliweb/sdB2mujjufuDemEO1rLYRRoKiUonCAY569Adc9foC+4NYpxEWOqeuI4DblwCXfuCsXIPOpcdf9QUd5TaTYQek2czEv5Ufj+ln/UyjPDgEwtRTHc/6SDdommbo/ffV+HHtGwvhD9FtuyyI+ueN+9DngW/w1y9WSfcDoD13Rh42l9ei4eKP8Kdah06+g4Tjn5qJUY9P039X1hsbzcItFVix07mum84oyKIjpPIaP+oCoThHm3H9mwtxz8fLcP5LP+GcF+ZiGUfWjKhzM5JTKdlgoKzaj9vf/wWvzN6Eaav3YOEWSsy6PzzOdarqhecSZMB7PlmeVPmSwfw4nQo6DKK+jHH3mfcV9TY6d+OhqLfZJ9AMSCuifm/Bdvy827w9GCXqiAb88/t1CITSb7LA1n21+N8iY8jZuS/+hIEPTcWOijoAwI/ryzF9TRm276eSx0NTfjUP4wCs3k3DnWavtw4hbAiGcf2bC3HC0zMxafamVD2GYxz16A+4+3/LTNs3lNVIrb8DdQHTtmpxOJ6GqPGHTFYuD03TUOsPoWsb6vjdWZGAnBXFD6vL8NGiHfh1J/3u2/bX6fuYJJGXmSRRR5l4B1eux79ZgxdnbDDsj8QJKogn05XX+DFjTZnl/g8XbsOMtWW4dNI8vPXzFidFNyAS0XQDh8eUpTuxYofQ4WcWAPfvjEUZtQKkFVF7VIJg2FxhQty256atxwcLtyV1/UAoggte+inuMC8ZnP7cj/jDx8v1ytQQpMPYiBZreJle4+vWNGMDYmAd06a9tXj627X4aKGxA9hcXovRj0/DD6tpw1guVlQbhMIRfLx4ByKNlB32Vvvx8eIdhm2b9tZg/DOz8NwP60zHV9YHkZfhwcBO+fjtSdQJV5fGmjrDizM24JwX5uLnjfuk+/2hCEIRDf07Uqfc9oo66XEM4reUYW91TC8u14naa3W4LVgsB3/NyroAavwhwwGaRvXslbvkdemxr1frBgePZRfNx+iG5/Hrzipc88ZC/GoxSrr3kxW45r8L8fOmfXhoykrsrEysQzv3pZ9w2INmae93HyzFxH/PSehaVthQVoPt++2/X3MhrYjapyo6SfEQtyXjNKsPhLG+rBqLtlbgvhQN00oPNOBf09ZTqypapvIaajluKKvRj6tuoI1CRkwyi5ofMfx7xgbc88ly1EYb1ls/b8Ht7y9BBaf57rKp9Mt3VBq04jd+2oK7/7cM/1u8Hf5QGPM3GQlo+/46lNz3FWatczYhaH8tfd7N5bVYFR0JfL/abFlV1gVRlOvD178bi3OO6AwASckEBxvMCSd2SgyM8AZ2ouFdK3fGJn+EwhETITmRCMo4Ut0XrU9eNfG4XCBmMRuJOogaf9iwf1dlPV6ZvQk3viWJyQYdDR779xmm7eHcDtiNIv33mtLYhJ9wRMN/526W1vHJS3ZEyxLAL9sq4j7HMptRTWNQURtAyX1fYcrSnRj/zCyMfdL8jOmAtCJqj0oM1jNDULD+wg5iv0UMeGgqzvgX7XmTiEU3QdM0/PaDX/DM9+uwcEusopVX+6FpGv47d4u+TUbUGdHQNL9ExpF1Vhv31qCsugEPTVmpD5EBoEthlk6QMtzyzhI8Ny1m4TJiLa8J4B/frcPFk+YZho5stMEakgx8+TaX0w7phKdn6pEJfARBJKIhEIqgsj6IgqgTNSeqt6azRc2cZw1B+qzb9teiLhCCPxTWpSkAegfaoygHY/sWY9LsTQhF38+jX6/GmCemJxxRUVYd61j3VNG/k5H7whFNbys8UYciGmr8tANiTWHLPjpRhB/1hST1UJS1xN/b9sUmnCzdXom/frFK2ulvKqfHXfrqfJz74k/2jnMLyEaFCzbvx7o91dhQVoMP44y8v1+1B18up7MX3/hpS8L3P5hIL6JWFAQlyWqCQiVt7LBdVeyZeun2SpTc9xUWb92P535Yj7kbygHQRqlpGjaU1aDnH7/WSY2RH0AbxFs/b8UnS3ZgcJcCADEtlh8JsKHsvE37TNp2gOusHjqTRsCs31ODmgazBdqrXY7ht0jyVfVBg0XNLChN0/Sh7KbymPXPOo4Mjyp7NQiGIzjzX7Gh5p4qMwnx7+Oxr1ej34PfYMf+OnTIo2FNWT567am/lkqtrebG5F92oOcfv8aOijo9WmXhlgoc+bcf8PAXqzDhuR+xO6rZMos6J8ODY/sUo9of0t/h96v2AIh11E7J6NMlO/HiTKohL9pKjQB/KILfffALzn5hruPnuOaNhdi6j37jvUJnweoSM1o27aXEWZQTCz0LSIha7DDE+rZlX0w6YJ0M68wYsryqruWzTo91iImgQaJZX/TKzzjln7Nx5vM/4t5PVlie++P6vbjhrUX405SVAGLtIl2RVkTttbCoQwJ5Nzaqi8D4Uf49fT2mrd6j/2ZOkVlr9+KfP6zD5a/Nx/er9mDQn7/F+wu244HJxgrAOzn2VDdg0uxNOLpnW3xyyzEAYh5z3oL0RDuLp75diz98vBzbuAoeCEWQm+HBsodOwf+N7gGvSrBhb40eAcAjP8uoXb43fxvW7YkNP/2hCHYfaMBfPl+JsuoGsD4qHIk1Sp5YmRyxvaIOf/jfMnywYBu27qvF1F+pl3dnRT3Wctf/zbtLsKXcOG03xH2g1+bQqcGbymtxWEcqD2R7KVH/uL4cf/vSJrKlmTBlKbWy1pZW40B97N3UB8NYHCVO9s5qoxJCboYHXpU2J0ZerC4zq9aJb6RjPp3M9OTUtfhq+W7dh+EPhTFl6S4s216Jsipn+Utmc5bsbMGqrRHIk1m4bbkJWDIrXtwmtle+LulELYyc+nbINUlCychg7N3LwIjfKqxxLSfRAEB603Q6zUwE4FEVEykDQCBkP9xieDvqTb5ydIntffjOc9WuKjz93Tpk+1R8dNNo/GfOZkyOTlzg78Ks3jd/2mIgKiBmMQHA50t3YWdlPe6d0B8+j4Isr6pb1HXB2HGiUf/tylLccBydkRgMR+BVCQqiE2MKsnw4UB80hbgB1Drh8efPV8KnKlj36AQqO4QjWFNajTWl1dh9oB4DOlGnV1jT9Eb51y9WYfyADujWNhv7og3tp6jz7H+cNrvliTMMJMzwymxn8e2MqD1qzD6wi6hoLniiMW2hiGaK//ZEtWJGEsxazM30wBuVswLhCL5cvgulUaJio4aLJ83Tr3PusC5ok+3D63ONOS6ev2wYuhRm4di/T8et7y1Bm2wvsn0eg0S2cEsFzhgiyezGwS5GHwCCYRpFwYiXSSN8dImMqKlREjMOTCM4LpKHjbaqhHrbp30upizdZZBW6gJhTul2Bie+qkAooo/gDNuFcqe5QZ1mRK3Eoj6C4QgO1AdRnJthIm+rISQbxsQjal76mLOBWhr9O+bh6v8u0J2BgNECZpYNI+kMj6I3Ht7Snb95P/IzPThlIJ1Km5fp0Ym8gbuemOODv0YwHDFMr872qajzh3CgzkzU2TaVUKyM5TUBg/TB3+PteVtRUpSjW0EAjQ/my8VC0URYySQd8zN1sgKAvu3NOT5Eyy4dwBx3obBm6hzV6PurqAtg2fZKPP4NnUCSm6HCFz0vGNYMMwkbghGT3tujKFtqcBRmedG5MAv5WV5U1gUxuncRKmqD8Acjep1zIheVOrC6axpCpmvxZZJJH6JPRYzS4kmZWf7iSLBXcQ7CEQ37uZDN2iQsat7wsYI/FJYSdVAw/qQ5d9IIaSZ9KHqF/mDBNpz49EyEwhFTr/2v6RuweKvzEDuxQfDfhJFxhkc1hUBVcMM4Pmyne9tsdC6MJUwSK+KoXkXIjFq6+Vle3crgiV+clVzdENTLGQhF9GE0QMm4NhCWSh+ySsggNkI+bjkc0Qxa+H/nbsb9k1fg0yWxadAXjTAmwgmEI1Ki3i2JsT1/eFcTCXdvm206Tna95gbryF+YscHghANiQ+rKugDOfmEu1u2h+n6mV9U7PtGn0hAMm4bpXlXR6wgPJmWxkRK7Li+vyQhURJ2D91rjD6FeqCP+UER3Lsss6vpg2NCeRCOKr6N7ok5Rvh0B0J3K/PWTcSyL71TW8Vk5YUVOSW+aTjOi9qhEH1qXVftR1RDCgfqgNLb66W/NsbpWEAmLdxywhucPhdG/o9Hi42dAVXMVv2/7XD1fB0AbLY9OBbGkSbxFXceVQxV68K9W7Ebv+7/G2tJqBMIR+ASirg+ETcNwwCx9MGiaZrJ+qjkLyh8ydoCyd9xXyHLnD0UM7+FvZw+CqhDdYQUAQ7oWYMbd45DlU0xELetU7HTGRHHvx8uTmkwhgvkPWDTNXSf3w7F9igHERlTit/B5FJNGzdAQDKPabzw+HNGko6H8TCNRZ3lVw+gNcBYBIurC/DUZKFEbr/XVit2Y+O85mLZ6j7RDOOWfszHu6VgIm1n6COkjXiZ9iFJhTvS5+TpXl0A9IITWb/Fby0YasqgqWbmTNag3lFUnn9oiAaQVUXuVWBx1gJMVZOFqdpnXIhENwXAEU5buRCSimawGYiBquq8uEDZ5fq2C8tvnZ6JNdszpwhptUVTzbZ8fI+pOBZlYu6caf57yK2avjTl0RGuKSS7zNu2LatSx58vJ8KA2EEJlvXl2n6yxA8Cox6cZYrkBRtT0XS7bXon6QNgQnzu4SwH6dYgllTlMIOpAyGhRD+1WiDF9irE56oh6/tJhmHLrGPQszjGU3w6BcAR//HQ5/j19vaPj7fDhou14KCp/NQYeoewFWV5cfUyJYVuFSNRqjKhFcqgPhk2dViiimYjTqxI9PI7Vj2yfigyvmjhRSyzqRQ8as0PWNIQMchyPx75ejRvekqcqZjNrAXMHH45ounXMZDRxQhZruzyxMuljT1UDJv9iDA0V279HIfh82S7d6cuQCFGLnRDf9hOJKvti2W488c2aJp8TkFZEzcdR8/qvLBJEJGpet65uCOHtn7fidx8sxSdLdpicDrwjjw0p15RW47tVpRjU2TqVJiPF3AzVkJ6UDfeYo6kDR9TnDuuKvdV+vPnzVlT7QziyRxvcMLYnXrriSOk9agMhBMNG/TjLSy1qqfRhYVHvqfLrTlGGGn8I+2uplbNoawXe+GmLHtMMABcc2RXf3RlLXtWjyChV+AWizvKq6JifoX+r3EyP3gn6HBJ1OKLh/QXb8fR3zkdIyeD3Hy3DJxaTVkR4BE9vplfRHbsM4ijK51H0Z94tOPIagrH3dlQJTRgUjpidXPmZXv39McLWLepgfOnj0yU78K9ptMOTab5yi5ped2CnfBTnxkLzNu6tNRCyFWRtc2+1H5e/Ns/gZOfB3tO+GnO00Y1vLcKdHy4zyCUs0oZBIQQz1xqjWGauLTMYZOwTWnVqdp2dFbnbHWv1rKlCmhG1ok9uYZWxsj4orZjsY3+8eAf++f06g8ZVWR/QG8am8lpH0gdArYMMj2KKTQZo+FXPYro92+cxhDHpRB0VnnnP+eFdjMSfk+HBA2cM1K8lYmdFfVSjJoZz1pRWG/RjBpnOGXtO87Y5G4wzEXlCbR+Nc/7gxlG4//T+yPYZfc0BQfrI9KoY1Svmq8/mysKsy2N6F+Ge0w7Du9cfbbjWVaMlifgdoD4QxiNfrjJZMKKDeUNZDe77ZLneID9ZsgO/F3KT1AfCuPI/8/U4eQaPKhK1qr8bhp82mt8je+adwlTrhmBYb8gsgZPMouZDLZlOnumLSR/sCXki+WljOR77ejVC4Qju+mgZnvmedngyKUERKsTGvTWoC4Rx6chu+Pp3YzGgU+ILOshGuy/M2IC5Qj3jwaJjdlbG3lOdMLO3uiGEnZX10DQNl3DRMgBtv6LxdfV/FxraOWvjsvwgsnLzHY44AudR4w/h2jcW4orX5uOLZbv064uRLalGWhG1VyEIhSP4eeM+vYFV2Ugfmqbh7v8tw3PT1hssvYq6oG4BVdQGTC+e1ddPl+wwWZ0ZHhXTfz/OdL/DOP06J0NFIWdhMaIe0pVOcOEtapHs2mTb52zYtr8OAUH64C2v/EwPbj2ht+X1faqCJ86jy219tMhoQaoK0XNHMPD3YcnlR/Uqwo3H9YbPoxisS38obLSofSpOGhBLFM+XhZGdz6PgN+P6YExU42X469mH4/FoOe1Q6w/huR/W63XgzZ+34LU5m/Hq7M34cOE2PbeE2Jn/feoafLBwO+Zs2GsZJbS9og4/ri/H5a/NN2z3CJ7eTK+KHkU5eOe6WGezTcgJoSpE71xf52alAsD0NWX4MJrjoyCLdvDhsCaxqM1BWNSiVg0+Bd4avOzV+Zg0exOWczk2aEqD+BbeY1+vQXmNHx3zqWPc6SiIB982Wcczb7M1SfP34S121rGwkcTCLfsx5onp+M8c8xJdCpFPdqkPRLhj6LewdiZqhvfNT7Szi6p5Y+5mTF9ThjkbynH7+78YJNqmRFoRtUclWLmrCpe+Ok+fdmolfRBA97gDRmffvhq/rp1N/mWnIQgfiGnUd31kzv6WEa0oY/oYozpLinJ0r3JOhgdtJRr1vaf1x3vXH40juhXq+3K4xtilMAv3n25eqZgnw52V9ebwPM7y6tchD9cfG1sBRtSoi3N9uPgoeRpHmRXP36c415zOkbf6qEYdq8TZPlW3EAEgO8NsUYukZ3VtKzz7wzr884d1+DyqR7KIioZQGPd+sgJnPk9nSfIWVq0/hC7RqJw3ftpqsED5RmjVIMVIBjZqGd6j0LKchBDdUmQkPuFwmtB++poyPS8068xLinNsLWo2KSvLqyLDq6A2EALrb2TkU85Fp1T7Q6YoCtFRzmNkT5pnma8LbFatFVjnx8fVM+PFSjJRFYKh3Qr1usEnsGIdC+u8WHy9LBlWRJN/u3qJPLR0e6XJV8P286Nivv7MXrfXMmxUjAJidcu0MEOKkWZEHSsOeyGVdXLpoy4QNsSK8pbe89M36OFJ/lAEvxcI2W4GOcvB8fa1R2P1w6fhvGFdANDQMp2ofR6DRs0qSHaGimMEy5F/pr+eNchgbTPkcj176YEGBELGqA++MWR6VcPQXJQ+zhjSyTImVHQOAsZkP8XC8B4wWvP+UMRQITOF+Gm+0/DpRG39su1kGwY2AYfN7mPDd77BlNz3lWGoXV7j1+Wn2ev24qeNMWmDt4R52Yt3IIkxtpnROpHt8+CJ8wbjk1tGS8sqWqSPnjvYVNfOGdYFn9wyGpcf3d30/OIsU4C+/0yPCn5QEAibSYqfIr6/JmBoD5leBVNuo0twyarGsO6FAGJ1HwDevm6krYV9+r/mYMaaMkMoIt/xnHNEZ3xw4yjDOR/cOApTbh2j1zkW8kpIjCjZNViMNV/eB04fgCtGdUcwHEFDMIKxfYtx6chY7nqZZPH4N2sw/plZpu3BUMTw/nlivu/TFbjzw6XS5xanujOirqgN4qWZG5ssdW9aEbVX0qitLOq6YNigCzEv/IgebbBi5wEDoewTLOp5m/bjR4tcz+zjKQpBlk/Vyatfh1ydLLJ9qiHqgyFDtSce3uLkwcjs1EEdUBcIo7zGb5AkDOToVSxlkZl3j8N9E8wWOwNbVHZot0I9ppm3ovIkyen56++qrMc3K0pxUv/22PLEGTppnh3Nhscnt2eN0S6vil0MOEMs94ixqooZA9/4KTZE3lcbMFiUvIVXxuUmMVjX3FBalNr4Bn3JyO44skdb3HS8eV1LMdKlMMtrIuPcDA+O7NEWhBDTaChfkso006vi3KixwCCzqPmOa38dJWr2DTI8qj4pSexc37p2pF5GVhc65GegMNtnkPdErN5dhbs+WmpImBYIR3SiLSnOwaheRejFjeIY8bP3tGRbJdrm+NAm26db1KwsMWdirP4UZnujE+A01AfCNMacMzQSyaopjlpFC3rVLnmiM1FyYU7ejxfvwN+nrsHT3651XIZEkFZELYZFAdSilmnU9YGQwdO6J+ppH9ApH+EITZzUJtuL350kXxz3yv/Il/MRQ/RuP7EP/nrWIJw6qKNudeVmeNA2x1yJvR77YMwcn3wi6NvXHY3bTuiDM4ZQwttT5TdWIu45M7yqwUrlG3tJcY6UGF++4kh887ux+tB9xY5KPdqDJxeZJc7v/90HS1EfDOMiQVp5+sKh+OGu4w0Thti3tCVqBxa1P2jUZVkRxdBJnryqG4xDf57E9tbERmFGGYQLf7MhagbZtxRTkSoKiTkFvQoGdykwvA/WUbFt+Vlyjbp7UXbcHBwGoq4JoDYQ1qM4+NFChpATnS14AMSImvkaZE51HoFQRI8yAWhbZVbtoM5UOvksasnz1+frdof8zOjMW6ZR03fCUr2yBQ26tsnCucO66PWxuiEYHV3GrnVAEr7K8OvOA4bZocGwZqjbYjijlWInSi7MkGAjtabKCJlWRC3LuSuLox4/oD1q/WHDMGNHtOGyKIu1pdXI9nkcWW08xFwWeZleXHVMCRQlNhknO8OjW9R83HE8Z4xVzPOATvm4+9TD0KUwJovwlYiPrMj0qMbG7oDshnYrwIBO+RjRg2qREY2GGLL7/HjPCfica1B2uPG4Xvr0eL6sfdrnCttoGe2kD6v3wRAKR3SvOmtITLsVV1Lhdeiq+iDqAyFdP+fThhosau4cftgsjuDEBR+syi6L7WflfujMQfji9mMN+9i3y/apGNOnSP8+PNhIgh9RyKRA3km8vzaAukBIr6O8pCJa1IXcyJBZ3axcL1w2HH8602LRaJgn1XQqyNSd5cf3a0fvzXXe7P3wdftvZw9Cjs+jExzrVNj3nR5NkHb1MSXwqIr+HqobQsj0KAYZkE//IOLM5+foKSYARB32sXPFeHCrbHoNwYju/yjM9sZypUTfv9gRpgppletDNkmiSjIzMcvnQT0X8gRAn3RxZA8ap7rrQAP6dch1RGQ87ILdI7pGTZ1oCgF6FefqTk3ZiIBHtiAtzPrDOAPpts+LEbWPs86vGVOCX3cdwKdLdiLTqxgsXyudl8/TwRpnlk/Fvaf1R8/ibLy/gEYh+FQF3dpmo5tkejdgDju6fmxPR3kRGKHaWdRi2cMRTT8+EIqg34Pf6PsYKWgwO7EAY9a26gYaH9w+LwMH6oOG6Bc+Kb+VY9FO+mDIkchEfEfdrS1tzAohiGia3FEbJfsMj4J3rx9l2g/ERjkGoraxqD0K0RcnOGVgB1wysptOmvRZjHWUj3xgRJoT7cSLcjNw3bE9HWU4/O/VR2FAp3wEon4MvtNSFYJwROOkj1idyM/yIsun6tIH63BFvZl9A92i9oeQ6VXh5Uzfp+LIDu8v2IZLjuqGod0KEQhFkJfpwaUju+P9Bea81Va1tiEYRtc2WTh9cEe8M2+bbkiwb+J0oleiSCuLWhYhUFkfMCW0yfGpqAuEDBb15vIaFGR50aMoNlxLxqK2W+2Zt6g9qoLnLhmG207s4/jaOUJZehTloGubGEHylo8oSZREn0usCFZW/LTfxyau8I3mlnG9cdrhnXQ92W6GJwBTljY+X7Ed2LtKRKNuCIYx/plZeHLqGtMszBp/CNv31+HJqfLGyBNwVUMQdYGwwUnLwEsE/CQSZsF9s2I39tcFdAcbYLZCAblFzX+bWXefAAD48KbRGNgpH0O6FpqO96kKFGIRGSO8Nj7xlT8UwUcLt2MetzrPkm2VyMv0GJ65c2EW/m90iaFNiJ0Ob1ywuuTEySvihP7t0bEgE92LsnG4EDHCRlWss+HrbJZXRU6Gioq6APyhsGUkTpZA1LScisk5Ko72RDz42a8Aohq1quDx8wbj/yQx/ZYWddQJyYxFcQp+MqtPOUFaEbWd9MG/t6yoplXFW9R7a9EhPwNeVdGHXzkZqiOLun1eBl6OzhS0Wz2GDaVZOScO7YyBnaxnMooQY55F5GV49CgBkYBlw2/AWhcvzs3Ai5cPx8BO+dJ3kKNLH/bW8UNnDsTCB2JTj+MtusAQjoa4JaJRb9pbiw1lNXhx5kaTJV/rD+Guj5Yato3ta4ywYVi9uwo/ri83kalXJSirbsDm8lq8MGODwWr7v9cX4KeN5bjl3SX4ZVulgRBkw1mpRs11eszRemSPNvj6d2PRscAc7UMIQZZXlX5DRjhdohoyX4Zfdx7APZ8sN00EqW4IGSJ7Ohea73lMb/k74+/h9Bs7hVe3pM3SR5ZPRbbPg193VuH4J2dazgpknQc/ssj0qoaRlU9VMKx7G9uysJEN70yUNnmLV9AQCCPTq+h1V6ynTRWml1ZELc4IA6JEHdEMQ5zi3AxU+0OGySq1gbAuHbCJG9k+jyOiDoYjeuW0kz5ev/ooPHjGAINEIc72skM861VRiG5Ve4VjVQvvhl2c8umDO+Hr342VllHmTJSBEJKU7sYakJ1GLX6buVwY3bxNxuyItf6wKTSqc0EW3r5upOm6LAeEmH61W9ts7K32496Pl+Opb9caljQDjOsd8h2YGHECyCNWklnXMMunSr/Bdcf2xNKHTtb1UFYGhZjzjPB4/rJh+t8dC7JM++8/vT+m3jEWn982Bq9fPcKwj6Wh7VWcazqvMThpQHsAnEYtpEdgHWppVQNWWCyOywyVLpzzM9OrGkbAXdpkSb8Bu/5RJW30OsQ7E8+KRi3xsLaoabRJVrQ84kQXtkhIqpFeRC0hnYZgBDUNNNRo3GHtcMf4vtJ0mUBsCjRLjpSb4bF1WLGKc/YRXWIrn9hY1N3aZuP6seawrHg4qX97x8eyYXa2QGJWFJDsoqcsIkAW+igimRlrYV36sD5XHCW89uMm/W+2jBVDrT9kGuZ2LszC2L7tMPPucfo2fuanuHRZ97bZ2Li3Fgu20E6Aj68GjFne4kXDyDogr82zWiHLp0rPI4RIHX3x/CDt8zL12H+ZkeJRFfTvSKWYE/sbZYKTB3bAir+cggfOkId4njusC566YAim3jHW/qEEPHnBEMy8exxnHBid4fFGmkDMouad1hkexUDUw7oVSr/LjLvH4bs7j0NeplfPYsinEj6qxOzEFa+iaRqWba/E/toAMj2q3lGLYX1NZVGnmTPR+HpoOkNgX60fHlXBG9dQ62n5jkr9mMM65OkNjDnEWIXo3jYbmTZE/cg5h+PUQR2Rm+HB7GhctZ1GnSxevvJIx4lemO4uOvcYV4jToZNNeM5mKe6S5JIWIbMo4yFG1NbHEEIwfkAHZHoVfLl8N8prAnjivMF4fvoG04KoK3cfMM14Y6FlJcU5mHH3OHy5bBcm/7JTtzjLa42zyMQOXowS4OtVvJGGbPSXyOiKwUr6EKF/AwfV88EzB6J9fibGHdYu/sECxJzsPP558RH6373b5WDj3lrLY3lkeFSUSOKpAfrOfA6MDUbURVyYIm9R9++Yh0fOPVyfqs9wzZgSdMjPRIf8TORmeLBxLyVWKn3E7sscnnq5ou1qf20AbXN8mL95vy41ZfmoRi1DU+X8SC+LWmgc7KPsrKg3kEWPtrGP/tmtsbCy3tHelg1H+saJ+mBToFWFYEBHqjVfcGRXy+OtcNqgjobAfhFeVTFMBrEDi27oLmSuY1UqVd0II+odFfGJOpnO4JxhXdCzOAdXjiqxPe61q0bg/OGxd37RiG76lGaGXsU50mnJvTnrqmdxDm4/qa/BFNpXE9CncQPyhQt48CkJ4o0iZBOekkGWV7WVrxjYkJ/5FoZ0LcB5w40TYVjdbZvjw30T+jdZBAIgz1/uFGJ9cmLEsNEXIUSXg6hGTc+9dGR3ZPs8BrIdP6AD/jxxkP6bzw0vphIuFGaFEkJDfIf/7Xt8vHiHYUZrBqdRi2iqnB9pZVGLw5a+7fNQXrMPy3Yc0MPuAKAg24vj+7XD2Ud0NmiFvaMB+iwFZc/iHFvpg9ehOhZkYssTZyRV7pevlKcsbQxEUmGV2+FC1nHBokicWgCPnzcYw+M4anh0yM/EDE6SsEOnqNPrqQuGQFEIjippq/sf7hzfD3uqG/TFV3nIJmQ8dcFQlNf4cdPbi9G9bTZeuuJIlNz3FQD5FG0reFWCL2471pDhzXjvXLx+9Qhc+4Y8Z7NTtMvLcDSK++OEAejfMR9LtlXg48U70D4vE/ed1l/PqPjHCf1x0/G941wlORTl+Eyze2WT0JKFXbY6Bj4S5fAu+dhZWY9IRAMrBvMx8R1IRGgseZleVDcEoWkaTSXMEXVBttfwjOGIhpW7qF7+w6o9BmewGnUCM/g8CgKhCB4/b3BShp4TpBVRixbAgE75mL95HyIaTBbrm9fGnEjjDmuHmWv36k6Qpy4cipdmbkS/DnmmZYB4pNa3nRr0aZ+LDWU1hiEeEJuebTetNxFk+VTcOb4fju1bFP9gwJBTIdXo3zEfy/58ih5VcyyXL+V34/vi7XlbpefJplyzDv29G47WnWPDuxdiybZKDOdC7opzM0yZBHl4VAWDuxZgcFfr5ERiRsBk8Ph5Qxwdl+VTcdnR3bFpL7X6M7wK2ucnb1wkgln3nGCK3b5oRDc8N209PrhxlK3D2AmsQvLOGNwJX63Ybdo+7rD2+HblHoQ1TXf+M6IOcwm1zJPXPNFFfSNUo+ZG6Yd3LsAmTsrZW+PXV6iZurLUcJ3K+iCKuLj4bJ+KQCiC7m2zm2wUk1ZELep+eZk073N5TQA9baazvnDZcGzbX6db18O7t8Gr/0c92nYadVGus5jgg4mPbhqNPVUNpuHhOcO6oKohmFLC/N14+fT65gCfha97UTZuOq4XtuyjDUcmWfybi26QgQ9De/PakdhZWY8+7fPw3Z3H4ZR/zrbVzgH7aBUGmTzy54kDDSlx46GdJBGWHc4c2hmvzdl8UNeazM3wAEIx7xjfF7eM651UzLWITtHolDvG98WzP8SmpPdul4MnzhuM+z5dYXhPlxzVDW2yvThpQAfdr8AIkidnMYKLTe65/s1FplTCT5w/GKcO6ohb31sCgE6H//vUNdLyVtQG0L9jHu4/vT8e+3oNKuuCUBViWmgjlUgroma6X9c2WdhRUQ+fR8HlR/fAc9PW40ibYXdOhgcDLOKZ+SHKXSf3w56qBrw7fxv+e/VRJi00HdA2x2fI68CgKgTXjOnZDCVqHvyRSwd7TO8inDW0Mz5fFlt66cwh5pAqK+RletG/I+0I2BB2WLc2WLBlP8LRpdruPqUfvlu5B4uiq4msLq22vB6DTLtv6m80tGsB/nbO4TjOIob8YIEQkhKSBoD7JvTH0T3b4sQB7fHKrE3I8qnYXxtAXqYXl4zsjksE44QQgtMOpxOx7jm1P3yqgolD6W+7hXeZo3ROdKEIfsZmts+DM4Z0wsuzCixDBGPXoSsZXX50D7y/YDsePedw9OuYZ1ghJ9VIK6KecHhHTP7NMcjyqTjt2R/hUxXcekIf3HpCn7gxyFZgveaATvn47Ul9oWka/nLWoCZ1tDQHZOTeWuBVFTxz0VCdqD+6SZ5m1AnyM72YcusY9GmfyyVMooTD8nw/8NmvcWe48RjV6+B1+IQQXDkqudVx0hWZXhUTBlOiXf230/DtylLc9PZiHO3gvbbJ8eGvZx+u/y7hZmGK0j8/Y/LDG0cZ/F4Mn9xyDCY+Pwdr91Qjx6fixAEdENE0fLV8NwZ0yscNY3tifLRu5GR4HPthGgtHRE0IOQ3AcwBUAK9pmvZEUxSGEIJh3dtA0zT8ZlxvU6B8svjqt8eia2G2fo9kY4/TFYsfHN/od5Tu8KgKPv3NMehZlGPIBZ4MhnILO/Bg4XVOVp5hWPjAeMPSay4aj1MHdcSvfz3VcaQUj/OGd8G+Wj8e+3qNSfro0z4Xd5/SD4pCcHQvuW/G51H0ejDt9+P0Edjfzg4gw6NIc7wcDMS9KyFEBfACgJMB7ACwkBDyuaZp8TO1JAlCCO45rX/KrsdSLrYmvH3dSH2Yl45ae1MgkaiTg4VENWYXFFNuHWPbwSVD0gDljqN7UhKWGS+3nRjfL/Pvy4bhh1V70CE/9m2be8Tq5G2MBLBB07RNAEAI+QDA2QCajKhdxMfYvolPZnDhIl1gNapJBYZ0LcBvT+qLy5J0vPdul4vex6d2Gn1j4WS83AUAP91nR3SbAYSQGwkhiwghi/bula+e4sKFCxdNDUII7jq5nzQRVkuFE6KWCbqmCH1N0yZpmjZC07QR7dq51p4LFy5cpApOiHoHAH7tpa4Adlkc68KFCxcuUgwnRL0QQF9CSE9CiA/AJQA+b9piuXDhwoULhrjORE3TQoSQ2wB8Cxqe97qmaSvjnObChQsXLlIERzEwmqZ9DeDrJi6LCxcuXLiQoHXPknDhwoWLVgCXqF24cOEizeEStQsXLlykOYi4tFNKLkrIXgDyJMLxUQygPO5RLR/uc7YeHArPCLjP2dTooWmadBJKkxB1Y0AIWaRp2oj4R7ZsuM/ZenAoPCPgPmdzwpU+XLhw4SLN4RK1CxcuXKQ50pGoJzV3AQ4S3OdsPTgUnhFwn7PZkHYatQsXLly4MCIdLWoXLly4cMHBJWoXLly4SHOkDVETQk4jhKwlhGwghNzX3OVpDAghrxNCygghv3Lb2hJCvieErI/+24bb98foc68lhJzaPKVOHISQboSQGYSQ1YSQlYSQ30W3t6pnJYRkEkIWEEKWRZ/zr9Htreo5Abr0HiHkF0LIl9HfrfEZtxBCVhBClhJCFkW3pfdzaprW7P+DZuXbCKAXAB+AZQAGNne5GvE8xwEYDuBXbtuTAO6L/n0fgL9H/x4Yfd4MAD2j70Ft7mdw+JydAAyP/p0HYF30eVrVs4IunpEb/dsLYD6AUa3tOaNlvwvAewC+jP5ujc+4BUCxsC2tnzNdLGp9XUZN0wIA2LqMLRKaps0GsF/YfDaAN6N/vwngHG77B5qm+TVN2wxgA+j7SHtomrZb07Ql0b+rAawGXaatVT2rRlET/emN/q+hlT0nIaQrgDMAvMZtblXPaIO0fs50IWpH6zK2cHTQNG03QAkOQPvo9lbx7ISQEgDDQK3NVvesUUlgKYAyAN9rmtYan/NZAPcAiHDbWtszArST/Y4QspgQcmN0W1o/Z3JrsqcejtZlbKVo8c9OCMkF8AmAOzRNqyJE9kj0UMm2FvGsmqaFARxBCCkEMJkQcrjN4S3uOQkhZwIo0zRtMSFknJNTJNvS+hk5jNE0bRchpD2A7wkha2yOTYvnTBeL+lBYl3EPIaQTAET/LYtub9HPTgjxgpL0u5qmfRrd3CqfFQA0TasEMBPAaWhdzzkGwFmEkC2g0uOJhJB30LqeEQCgadqu6L9lACaDShlp/ZzpQtSHwrqMnwO4Kvr3VQCmcNsvIYRkEEJ6AugLYEEzlC9hEGo6/wfAak3TnuF2tapnJYS0i1rSIIRkARgPYA1a0XNqmvZHTdO6appWAtr+pmuadgVa0TMCACEkhxCSx/4GcAqAX5Huz9ncHljO63o6aNTARgAPNHd5Gvks7wPYDSAI2iNfB6AIwDQA66P/tuWOfyD63GsBTGju8ifwnMeCDgOXA1ga/f/01vasAIYA+CX6nL8CeCi6vVU9J1f2cYhFfbSqZwSNLFsW/X8l45p0f053CrkLFy5cpDnSRfpw4cKFCxcWcInahQsXLtIcLlG7cOHCRZrDJWoXLly4SHO4RO3ChQsXaQ6XqF24cOEizeEStQsXLlykOf4fccc35Wx0tqEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 28800x7200 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(classwise)\n",
    "plt.figure(figsize=(400,100)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adversarial Routine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch in range(1, args.ckpt_size + 1):\n",
    "        x, y = next(train_gen)\n",
    "        global_prob = model(x)[0]\n",
    "        global_prob.clamp_(min = 1e-7, max = 1 - 1e-7)\n",
    "        loss = criterion(global_prob, y)\n",
    "        if args.gradient_accumulation > 1:\n",
    "            loss = loss / args.gradient_accumulation\n",
    "        #if n_gpu > 1:\n",
    "        #    loss = loss.mean()\n",
    "        train_loss += loss.item()\n",
    "        if numpy.isnan(train_loss) or numpy.isinf(train_loss): break\n",
    "        loss.backward()\n",
    "        global_step += 1\n",
    "        if global_step % args.gradient_accumulation == 0:\n",
    "            optimizer.step()\n",
    "            if args.scheduler == 'warmup-decay':\n",
    "                scheduler.step() \n",
    "            optimizer.zero_grad()\n",
    "        if batch % 500 == 0:\n",
    "            sys.stderr.write('Checkpoint %d, Batch %d / %d, avg train loss = %f\\r' % \\\n",
    "                            (checkpoint, batch, args.ckpt_size, train_loss / batch))\n",
    "            tb_writer.add_scalar('lr', optimizer.param_groups[0]['lr'], global_step)\n",
    "            tb_writer.add_scalar('loss', train_loss / batch, global_step)\n",
    "            \n",
    "        del x, y, global_prob, loss         # This line and next line: to save GPU memory\n",
    "        torch.cuda.empty_cache()            # I don't know if they're useful or not\n",
    "    train_loss /= args.ckpt_size\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(opt, model, adv_jam, trainloader, max_batches = 100):\n",
    "    model.eval()\n",
    "        \n",
    "    batches = 0\n",
    "    total = 0\n",
    "    error = 0\n",
    "    loss_ = 0.\n",
    "    torch.manual_seed(0)\n",
    "\n",
    "    for X_batch, y_batch in trainloader:\n",
    "        # forward pass\n",
    "        X_batch, y_batch = X_batch.cuda(), y_batch.cuda()\n",
    "        # plot(X_batch[0], y_batch[0])\n",
    "        output = model(adv_jam(X))\n",
    "        opt.zero_grad()\n",
    "        neg_loss = -nn.CrossEntropyLoss()(output, y)\n",
    "\n",
    "        neg_loss.backward()\n",
    "        opt.step()\n",
    "\n",
    "\n",
    "\n",
    "        # Evaluate performance\n",
    "        adv_out = model(X_adv)\n",
    "        adv_prediction = sigmoid(adv_out.squeeze().cpu().detach())>0.5\n",
    "        adv_accuracy = (adv_prediction.numpy()==y_batch.cpu().numpy()).mean()\n",
    "        print(\"Accuracy = {0}\".format(adv_accuracy))\n",
    "                # project into feasible region\n",
    "        adv.jam.param1.clamp_(0,1)\n",
    "        adv.jam.param2.clamp_(1, 40)\n",
    "        adv.jam.param3.clamp_(0,224)\n",
    "        total += X.shape[0]\n",
    "        error += (output.max(1)[1] != y).sum().item()\n",
    "        loss_ += nn.CrossEntropyLoss(size_average=False)(output, y).item()\n",
    "            batches += 1\n",
    "            if batches > max_batches:\n",
    "                break\n",
    "\n",
    "        return error / total, loss_ / total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'x_min' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-1faa4862a84c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m def projected_gradient_descent(model, x, y, num_steps = 20, step_size = 0.1, step_norm=2, eps = 10, eps_norm=2,\n\u001b[0;32m----> 2\u001b[0;31m                                clamp=(x_min, x_max), y_target=None):\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;34m\"\"\"Performs the projected gradient descent attack on a batch of images.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mx_adv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#.to(x.device)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mV\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'x_min' is not defined"
     ]
    }
   ],
   "source": [
    "def projected_gradient_descent(model, x, y, num_steps = 20, step_size = 0.1, step_norm=2, eps = 10, eps_norm=2,\n",
    "                               clamp=(x_min, x_max), y_target=None):\n",
    "    \"\"\"Performs the projected gradient descent attack on a batch of images.\"\"\"\n",
    "    x_adv = x.clone().detach().requires_grad_(True)#.to(x.device)\n",
    "    N, T, V = x.shape\n",
    "    targeted = y_target is not None\n",
    "    num_channels = x.shape[1]\n",
    "    criterion = nn.BCELoss()\n",
    "\n",
    "    for i in range(num_steps):\n",
    "        _x_adv = x_adv.clone().detach().requires_grad_(True)\n",
    "        \n",
    "        out = model(_x_adv) # N x T x 1\n",
    "        out_flatten = out.reshape(-1) \n",
    "        y_flatten = y.reshape(-1)\n",
    "        loss = criterion(out_flatten, y_flatten)\n",
    "        \n",
    "        #prediction = model(_x_adv)\n",
    "        #loss = criterion(prediction, y_target if targeted else y)\n",
    "        loss.backward()\n",
    "\n",
    "        with torch.no_grad():\n",
    "            # Force the gradient step to be a fixed size in a certain norm\n",
    "            if step_norm == 'inf':\n",
    "                gradients = _x_adv.grad.sign() * step_size\n",
    "            else:\n",
    "                # The grad is normalized here.\n",
    "                #gradients = _x_adv.grad \n",
    "                # Only normalize w.r.t. to feats at a timestep\n",
    "                gradients = _x_adv.grad/_x_adv.grad.norm(step_norm, dim=-1).view(-1, T, 1)\n",
    "                gradients *= step_size\n",
    "                \n",
    "            if targeted:\n",
    "                # Targeted: Gradient descent with on the loss of the (incorrect) target label\n",
    "                # w.r.t. the image data\n",
    "                x_adv -= gradients\n",
    "            else:\n",
    "                # Untargeted: Gradient ascent on the loss of the correct label w.r.t.\n",
    "                # the model parameters\n",
    "                x_adv += gradients\n",
    "\n",
    "        # Project back into l_norm ball and correct range\n",
    "        if eps_norm == 'inf':\n",
    "            # Workaround as PyTorch doesn't have elementwise clip\n",
    "            x_adv = torch.max(torch.min(x_adv, x + eps), x - eps)\n",
    "        else:\n",
    "            delta = x_adv - x\n",
    "\n",
    "            # Assume x and x_adv are batched tensors where the first dimension is\n",
    "            # a batch dimension\n",
    "            mask = delta.view(delta.shape[0], -1).norm(eps_norm, dim=1) <= eps\n",
    "\n",
    "            scaling_factor = delta.view(delta.shape[0], T, -1).norm(eps_norm, dim=-1)\n",
    "            scaling_factor[mask] = eps\n",
    "            #print(scaling_factor.shape)\n",
    "            # .view() assumes batched images as a 4D Tensor\n",
    "            delta *= eps / scaling_factor.view(-1, T, 1)\n",
    "\n",
    "            x_adv = x + delta\n",
    "            \n",
    "        x_adv = x_adv.clamp(*clamp)\n",
    "\n",
    "    return x_adv.detach()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}