TF_1_x/SelfNormalizingNetworks_CNN_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Tutorial on self-normalizing networks on the MNIST data set: convolutional neural networks\n",
    "\n",
    "*Author:* Guenter Klambauer, 2017\n",
    "\n",
    "Derived from: [Aymeric Damien](https://github.com/aymericdamien/TensorFlow-Examples/) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-1-f7315b4644ea>:17: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/kai/miniconda3/envs/snn_tf1_compat/lib/python3.7/site-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/kai/miniconda3/envs/snn_tf1_compat/lib/python3.7/site-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/kai/miniconda3/envs/snn_tf1_compat/lib/python3.7/site-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/kai/miniconda3/envs/snn_tf1_compat/lib/python3.7/site-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/kai/miniconda3/envs/snn_tf1_compat/lib/python3.7/site-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "1.15.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "import numbers\n",
    "from tensorflow.python.framework import ops\n",
    "from tensorflow.python.framework import tensor_shape\n",
    "from tensorflow.python.framework import tensor_util\n",
    "from tensorflow.python.ops import math_ops\n",
    "from tensorflow.python.ops import random_ops\n",
    "from tensorflow.python.ops import array_ops\n",
    "from tensorflow.python.layers import utils\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Import MNIST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (1) Definition of scaled exponential linear units (SELUs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def selu(x):\n",
    "    with ops.name_scope('elu') as scope:\n",
    "        alpha = 1.6732632423543772848170429916717\n",
    "        scale = 1.0507009873554804934193349852946\n",
    "        return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (2) Definition of dropout variant for SNNs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dropout_selu(x, rate, alpha= -1.7580993408473766, fixedPointMean=0.0, fixedPointVar=1.0, \n",
    "                 noise_shape=None, seed=None, name=None, training=False):\n",
    "    \"\"\"Dropout to a value with rescaling.\"\"\"\n",
    "\n",
    "    def dropout_selu_impl(x, rate, alpha, noise_shape, seed, name):\n",
    "        keep_prob = 1.0 - rate\n",
    "        x = ops.convert_to_tensor(x, name=\"x\")\n",
    "        if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1:\n",
    "            raise ValueError(\"keep_prob must be a scalar tensor or a float in the \"\n",
    "                                             \"range (0, 1], got %g\" % keep_prob)\n",
    "        keep_prob = ops.convert_to_tensor(keep_prob, dtype=x.dtype, name=\"keep_prob\")\n",
    "        keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())\n",
    "\n",
    "        alpha = ops.convert_to_tensor(alpha, dtype=x.dtype, name=\"alpha\")\n",
    "        keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())\n",
    "\n",
    "        if tensor_util.constant_value(keep_prob) == 1:\n",
    "            return x\n",
    "\n",
    "        noise_shape = noise_shape if noise_shape is not None else array_ops.shape(x)\n",
    "        random_tensor = keep_prob\n",
    "        random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=x.dtype)\n",
    "        binary_tensor = math_ops.floor(random_tensor)\n",
    "        ret = x * binary_tensor + alpha * (1-binary_tensor)\n",
    "\n",
    "        a = tf.sqrt(fixedPointVar / (keep_prob *((1-keep_prob) * tf.pow(alpha-fixedPointMean,2) + fixedPointVar)))\n",
    "\n",
    "        b = fixedPointMean - a * (keep_prob * fixedPointMean + (1 - keep_prob) * alpha)\n",
    "        ret = a * ret + b\n",
    "        ret.set_shape(x.get_shape())\n",
    "        return ret\n",
    "\n",
    "    with ops.name_scope(name, \"dropout\", [x]) as name:\n",
    "        return utils.smart_cond(training,\n",
    "            lambda: dropout_selu_impl(x, rate, alpha, noise_shape, seed, name),\n",
    "            lambda: array_ops.identity(x))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (3) Scale input to zero mean and unit variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler().fit(mnist.train.images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.025\n",
    "training_iters = 50\n",
    "batch_size = 128\n",
    "display_step = 1\n",
    "\n",
    "# Network Parameters\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "keep_prob_ReLU = 0.5 # Dropout, probability to keep units\n",
    "dropout_prob_SNN = 0.05 # Dropout, probability to dropout units\n",
    "\n",
    "# tf Graph input\n",
    "x = tf.placeholder(tf.float32, [None, n_input])\n",
    "y = tf.placeholder(tf.float32, [None, n_classes])\n",
    "keep_prob = tf.placeholder(tf.float32) #dropout (keep probability for ReLU)\n",
    "dropout_prob =  tf.placeholder(tf.float32) #dropout (dropout probability for SNN)\n",
    "is_training = tf.placeholder(tf.bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create some wrappers for simplicity\n",
    "def conv2d(x, W, b, strides=1):\n",
    "    # Conv2D wrapper, with bias and relu activation\n",
    "    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')\n",
    "    x = tf.nn.bias_add(x, b)\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "def conv2d_SNN(x, W, b, strides=1):\n",
    "    # Conv2D wrapper, with bias and relu activation\n",
    "    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')\n",
    "    x = tf.nn.bias_add(x, b)\n",
    "    return selu(x)\n",
    "\n",
    "def maxpool2d(x, k=2):\n",
    "    # MaxPool2D wrapper\n",
    "    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1],\n",
    "                          padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create model\n",
    "def conv_net_ReLU(x, weights, biases, keep_prob):\n",
    "    # Reshape input picture\n",
    "    x = tf.reshape(x, shape=[-1, 28, 28, 1])\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv1 = conv2d(x, weights['wc1'], biases['bc1'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv1 = maxpool2d(conv1, k=2)\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv2 = maxpool2d(conv2, k=2)\n",
    "\n",
    "    # Fully connected layer\n",
    "    # Reshape conv2 output to fit fully connected layer input\n",
    "    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])\n",
    "    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])\n",
    "    fc1 = tf.nn.relu(fc1)\n",
    "    \n",
    "    # Apply Dropout\n",
    "    fc1 = tf.nn.dropout(fc1, keep_prob)\n",
    "\n",
    "    # Output, class prediction\n",
    "    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create model\n",
    "def conv_net_SNN(x, weights, biases, dropout_prob, is_training):\n",
    "    # Reshape input picture\n",
    "    x = tf.reshape(x, shape=[-1, 28, 28, 1])\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv1 = conv2d_SNN(x, weights['wc1'], biases['bc1'],)\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv1 = maxpool2d(conv1, k=2)\n",
    "\n",
    "    # Convolution Layer\n",
    "    conv2 = conv2d_SNN(conv1, weights['wc2'], biases['bc2'])\n",
    "    # Max Pooling (down-sampling)\n",
    "    conv2 = maxpool2d(conv2, k=2)\n",
    "\n",
    "    # Fully connected layer\n",
    "    # Reshape conv2 output to fit fully connected layer input\n",
    "    fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])\n",
    "    fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])\n",
    "    fc1 = selu(fc1)\n",
    "    \n",
    "    # Apply Dropout\n",
    "    fc1 = dropout_selu(fc1, dropout_prob,training=is_training)\n",
    "\n",
    "    # Output, class prediction\n",
    "    out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# RELU: Store layers weight & bias\n",
    "## Improved with MSRA initialization\n",
    "\n",
    "weights = {\n",
    "    # 5x5 conv, 1 input, 32 outputs\n",
    "    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32],stddev=np.sqrt(2/25)) ),\n",
    "    # 5x5 conv, 32 inputs, 64 outputs\n",
    "    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64],stddev=np.sqrt(2/(25*32)))),\n",
    "    # fully connected, 7*7*64 inputs, 1024 outputs\n",
    "    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024],stddev=np.sqrt(2/(7*7*64)))),\n",
    "    # 1024 inputs, 10 outputs (class prediction)\n",
    "    'out': tf.Variable(tf.random_normal([1024, n_classes],stddev=np.sqrt(2/(1024))))\n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'bc1': tf.Variable(tf.random_normal([32],stddev=0)),\n",
    "    'bc2': tf.Variable(tf.random_normal([64],stddev=0)),\n",
    "    'bd1': tf.Variable(tf.random_normal([1024],stddev=0)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes],stddev=0))\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (4) Initialization with STDDEV of sqrt(1/n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# SNN: Store layers weight & bias\n",
    "weights2 = {\n",
    "    # 5x5 conv, 1 input, 32 outputs\n",
    "    'wc1': tf.Variable(tf.random_normal([5, 5, 1, 32],stddev=np.sqrt(1/25)) ),\n",
    "    # 5x5 conv, 32 inputs, 64 outputs\n",
    "    'wc2': tf.Variable(tf.random_normal([5, 5, 32, 64],stddev=np.sqrt(1/(25*32)))),\n",
    "    # fully connected, 7*7*64 inputs, 1024 outputs\n",
    "    'wd1': tf.Variable(tf.random_normal([7*7*64, 1024],stddev=np.sqrt(1/(7*7*64)))),\n",
    "    # 1024 inputs, 10 outputs (class prediction)\n",
    "    'out': tf.Variable(tf.random_normal([1024, n_classes],stddev=np.sqrt(1/(1024))))\n",
    "}\n",
    "\n",
    "biases2 = {\n",
    "    'bc1': tf.Variable(tf.random_normal([32],stddev=0)),\n",
    "    'bc2': tf.Variable(tf.random_normal([64],stddev=0)),\n",
    "    'bd1': tf.Variable(tf.random_normal([1024],stddev=0)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes],stddev=0))\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-7-8c4c5b6016f3>:23: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From <ipython-input-2-9c56e27e3d92>:5: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-3-829569ce6bf2>:12: scalar (from tensorflow.python.framework.tensor_shape) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.TensorShape([]).\n",
      "WARNING:tensorflow:From <ipython-input-11-70217e69e778>:6: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Construct model\n",
    "pred_ReLU = conv_net_ReLU(x, weights, biases, keep_prob)\n",
    "pred_SNN = conv_net_SNN(x, weights2, biases2, dropout_prob,is_training)\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost_ReLU = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred_ReLU, labels=y))\n",
    "cost_SNN = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred_SNN, labels=y))\n",
    "\n",
    "optimizer_ReLU = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost_ReLU)\n",
    "optimizer_SNN = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cost_SNN)\n",
    "\n",
    "# Evaluate ReLU model\n",
    "correct_pred_ReLU = tf.equal(tf.argmax(pred_ReLU, 1), tf.argmax(y, 1))\n",
    "accuracy_ReLU = tf.reduce_mean(tf.cast(correct_pred_ReLU, tf.float32))\n",
    "\n",
    "# Evaluate SNN model\n",
    "correct_pred_SNN = tf.equal(tf.argmax(pred_SNN, 1), tf.argmax(y, 1))\n",
    "accuracy_SNN = tf.reduce_mean(tf.cast(correct_pred_SNN, tf.float32))\n",
    "\n",
    "\n",
    "# Initializing the variables\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "training_loss_protocol_ReLU = []\n",
    "training_loss_protocol_SNN = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RELU: Nbr of updates: 1, Minibatch Loss= 2.529600, Training Accuracy= 0.14062\n",
      "SNN: Nbr of updates: 1, Minibatch Loss= 2.043923, Training Accuracy= 0.51562\n",
      "RELU: Nbr of updates: 2, Minibatch Loss= 2.308014, Training Accuracy= 0.26562\n",
      "SNN: Nbr of updates: 2, Minibatch Loss= 1.727961, Training Accuracy= 0.51562\n",
      "RELU: Nbr of updates: 3, Minibatch Loss= 2.068156, Training Accuracy= 0.38281\n",
      "SNN: Nbr of updates: 3, Minibatch Loss= 1.357726, Training Accuracy= 0.64844\n",
      "RELU: Nbr of updates: 4, Minibatch Loss= 1.670266, Training Accuracy= 0.62500\n",
      "SNN: Nbr of updates: 4, Minibatch Loss= 0.805312, Training Accuracy= 0.77344\n",
      "RELU: Nbr of updates: 5, Minibatch Loss= 1.599183, Training Accuracy= 0.67969\n",
      "SNN: Nbr of updates: 5, Minibatch Loss= 0.634932, Training Accuracy= 0.85156\n",
      "RELU: Nbr of updates: 6, Minibatch Loss= 1.444664, Training Accuracy= 0.71094\n",
      "SNN: Nbr of updates: 6, Minibatch Loss= 0.472679, Training Accuracy= 0.85156\n",
      "RELU: Nbr of updates: 7, Minibatch Loss= 1.424880, Training Accuracy= 0.75000\n",
      "SNN: Nbr of updates: 7, Minibatch Loss= 0.443895, Training Accuracy= 0.86719\n",
      "RELU: Nbr of updates: 8, Minibatch Loss= 1.238384, Training Accuracy= 0.70312\n",
      "SNN: Nbr of updates: 8, Minibatch Loss= 0.433366, Training Accuracy= 0.87500\n",
      "RELU: Nbr of updates: 9, Minibatch Loss= 1.223281, Training Accuracy= 0.74219\n",
      "SNN: Nbr of updates: 9, Minibatch Loss= 0.446125, Training Accuracy= 0.91406\n",
      "RELU: Nbr of updates: 10, Minibatch Loss= 1.126184, Training Accuracy= 0.82812\n",
      "SNN: Nbr of updates: 10, Minibatch Loss= 0.353353, Training Accuracy= 0.92188\n",
      "RELU: Nbr of updates: 11, Minibatch Loss= 0.945392, Training Accuracy= 0.83594\n",
      "SNN: Nbr of updates: 11, Minibatch Loss= 0.323721, Training Accuracy= 0.91406\n",
      "RELU: Nbr of updates: 12, Minibatch Loss= 1.061059, Training Accuracy= 0.72656\n",
      "SNN: Nbr of updates: 12, Minibatch Loss= 0.319183, Training Accuracy= 0.91406\n",
      "RELU: Nbr of updates: 13, Minibatch Loss= 1.053779, Training Accuracy= 0.71094\n",
      "SNN: Nbr of updates: 13, Minibatch Loss= 0.341918, Training Accuracy= 0.90625\n",
      "RELU: Nbr of updates: 14, Minibatch Loss= 0.861647, Training Accuracy= 0.82031\n",
      "SNN: Nbr of updates: 14, Minibatch Loss= 0.247850, Training Accuracy= 0.94531\n",
      "RELU: Nbr of updates: 15, Minibatch Loss= 0.859886, Training Accuracy= 0.85938\n",
      "SNN: Nbr of updates: 15, Minibatch Loss= 0.300610, Training Accuracy= 0.91406\n",
      "RELU: Nbr of updates: 16, Minibatch Loss= 0.803375, Training Accuracy= 0.82031\n",
      "SNN: Nbr of updates: 16, Minibatch Loss= 0.233539, Training Accuracy= 0.96094\n",
      "RELU: Nbr of updates: 17, Minibatch Loss= 0.770378, Training Accuracy= 0.85156\n",
      "SNN: Nbr of updates: 17, Minibatch Loss= 0.292139, Training Accuracy= 0.90625\n",
      "RELU: Nbr of updates: 18, Minibatch Loss= 0.703464, Training Accuracy= 0.81250\n",
      "SNN: Nbr of updates: 18, Minibatch Loss= 0.214945, Training Accuracy= 0.96094\n",
      "RELU: Nbr of updates: 19, Minibatch Loss= 0.667796, Training Accuracy= 0.90625\n",
      "SNN: Nbr of updates: 19, Minibatch Loss= 0.267386, Training Accuracy= 0.92969\n",
      "RELU: Nbr of updates: 20, Minibatch Loss= 0.739539, Training Accuracy= 0.82812\n",
      "SNN: Nbr of updates: 20, Minibatch Loss= 0.301252, Training Accuracy= 0.92188\n",
      "RELU: Nbr of updates: 21, Minibatch Loss= 0.712775, Training Accuracy= 0.82031\n",
      "SNN: Nbr of updates: 21, Minibatch Loss= 0.229995, Training Accuracy= 0.94531\n",
      "RELU: Nbr of updates: 22, Minibatch Loss= 0.668107, Training Accuracy= 0.83594\n",
      "SNN: Nbr of updates: 22, Minibatch Loss= 0.302777, Training Accuracy= 0.92188\n",
      "RELU: Nbr of updates: 23, Minibatch Loss= 0.662650, Training Accuracy= 0.80469\n",
      "SNN: Nbr of updates: 23, Minibatch Loss= 0.277247, Training Accuracy= 0.91406\n",
      "RELU: Nbr of updates: 24, Minibatch Loss= 0.751235, Training Accuracy= 0.82031\n",
      "SNN: Nbr of updates: 24, Minibatch Loss= 0.312389, Training Accuracy= 0.89844\n",
      "RELU: Nbr of updates: 25, Minibatch Loss= 0.610983, Training Accuracy= 0.85938\n",
      "SNN: Nbr of updates: 25, Minibatch Loss= 0.230509, Training Accuracy= 0.93750\n",
      "RELU: Nbr of updates: 26, Minibatch Loss= 0.598711, Training Accuracy= 0.85156\n",
      "SNN: Nbr of updates: 26, Minibatch Loss= 0.257897, Training Accuracy= 0.92969\n",
      "RELU: Nbr of updates: 27, Minibatch Loss= 0.649527, Training Accuracy= 0.82031\n",
      "SNN: Nbr of updates: 27, Minibatch Loss= 0.225033, Training Accuracy= 0.92969\n",
      "RELU: Nbr of updates: 28, Minibatch Loss= 0.513970, Training Accuracy= 0.92188\n",
      "SNN: Nbr of updates: 28, Minibatch Loss= 0.170374, Training Accuracy= 0.94531\n",
      "RELU: Nbr of updates: 29, Minibatch Loss= 0.548257, Training Accuracy= 0.83594\n",
      "SNN: Nbr of updates: 29, Minibatch Loss= 0.211176, Training Accuracy= 0.92188\n",
      "RELU: Nbr of updates: 30, Minibatch Loss= 0.541867, Training Accuracy= 0.88281\n",
      "SNN: Nbr of updates: 30, Minibatch Loss= 0.221660, Training Accuracy= 0.96094\n",
      "RELU: Nbr of updates: 31, Minibatch Loss= 0.485644, Training Accuracy= 0.89844\n",
      "SNN: Nbr of updates: 31, Minibatch Loss= 0.212724, Training Accuracy= 0.92188\n",
      "RELU: Nbr of updates: 32, Minibatch Loss= 0.450281, Training Accuracy= 0.85938\n",
      "SNN: Nbr of updates: 32, Minibatch Loss= 0.169182, Training Accuracy= 0.96094\n",
      "RELU: Nbr of updates: 33, Minibatch Loss= 0.633430, Training Accuracy= 0.75000\n",
      "SNN: Nbr of updates: 33, Minibatch Loss= 0.212616, Training Accuracy= 0.96875\n",
      "RELU: Nbr of updates: 34, Minibatch Loss= 0.477125, Training Accuracy= 0.89062\n",
      "SNN: Nbr of updates: 34, Minibatch Loss= 0.202256, Training Accuracy= 0.96094\n",
      "RELU: Nbr of updates: 35, Minibatch Loss= 0.549391, Training Accuracy= 0.82812\n",
      "SNN: Nbr of updates: 35, Minibatch Loss= 0.310254, Training Accuracy= 0.89844\n",
      "RELU: Nbr of updates: 36, Minibatch Loss= 0.368259, Training Accuracy= 0.90625\n",
      "SNN: Nbr of updates: 36, Minibatch Loss= 0.153444, Training Accuracy= 0.93750\n",
      "RELU: Nbr of updates: 37, Minibatch Loss= 0.492477, Training Accuracy= 0.86719\n",
      "SNN: Nbr of updates: 37, Minibatch Loss= 0.211344, Training Accuracy= 0.93750\n",
      "RELU: Nbr of updates: 38, Minibatch Loss= 0.556713, Training Accuracy= 0.87500\n",
      "SNN: Nbr of updates: 38, Minibatch Loss= 0.203756, Training Accuracy= 0.95312\n",
      "RELU: Nbr of updates: 39, Minibatch Loss= 0.442008, Training Accuracy= 0.88281\n",
      "SNN: Nbr of updates: 39, Minibatch Loss= 0.194691, Training Accuracy= 0.94531\n",
      "RELU: Nbr of updates: 40, Minibatch Loss= 0.427592, Training Accuracy= 0.90625\n",
      "SNN: Nbr of updates: 40, Minibatch Loss= 0.226508, Training Accuracy= 0.91406\n",
      "RELU: Nbr of updates: 41, Minibatch Loss= 0.598306, Training Accuracy= 0.82812\n",
      "SNN: Nbr of updates: 41, Minibatch Loss= 0.207904, Training Accuracy= 0.93750\n",
      "RELU: Nbr of updates: 42, Minibatch Loss= 0.551276, Training Accuracy= 0.85938\n",
      "SNN: Nbr of updates: 42, Minibatch Loss= 0.198079, Training Accuracy= 0.94531\n",
      "RELU: Nbr of updates: 43, Minibatch Loss= 0.404157, Training Accuracy= 0.89844\n",
      "SNN: Nbr of updates: 43, Minibatch Loss= 0.127414, Training Accuracy= 0.98438\n",
      "RELU: Nbr of updates: 44, Minibatch Loss= 0.368151, Training Accuracy= 0.93750\n",
      "SNN: Nbr of updates: 44, Minibatch Loss= 0.153246, Training Accuracy= 0.96875\n",
      "RELU: Nbr of updates: 45, Minibatch Loss= 0.461047, Training Accuracy= 0.88281\n",
      "SNN: Nbr of updates: 45, Minibatch Loss= 0.242575, Training Accuracy= 0.92969\n",
      "RELU: Nbr of updates: 46, Minibatch Loss= 0.456105, Training Accuracy= 0.89844\n",
      "SNN: Nbr of updates: 46, Minibatch Loss= 0.231932, Training Accuracy= 0.93750\n",
      "RELU: Nbr of updates: 47, Minibatch Loss= 0.414762, Training Accuracy= 0.89062\n",
      "SNN: Nbr of updates: 47, Minibatch Loss= 0.234552, Training Accuracy= 0.95312\n",
      "RELU: Nbr of updates: 48, Minibatch Loss= 0.371672, Training Accuracy= 0.90625\n",
      "SNN: Nbr of updates: 48, Minibatch Loss= 0.172536, Training Accuracy= 0.96094\n",
      "RELU: Nbr of updates: 49, Minibatch Loss= 0.396432, Training Accuracy= 0.91406\n",
      "SNN: Nbr of updates: 49, Minibatch Loss= 0.162846, Training Accuracy= 0.95312\n",
      "RELU: Nbr of updates: 50, Minibatch Loss= 0.281247, Training Accuracy= 0.96875\n",
      "SNN: Nbr of updates: 50, Minibatch Loss= 0.125570, Training Accuracy= 0.97656\n",
      "Optimization Finished!\n",
      "\n",
      "ReLU: Testing Accuracy: 0.8339844\n",
      "SNN: Testing Accuracy: 0.9140625\n"
     ]
    }
   ],
   "source": [
    "# Launch the graph\n",
    "gpu_options = tf.GPUOptions(allow_growth=True)\n",
    "with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:\n",
    "    sess.run(init)\n",
    "    step = 0\n",
    "    # Keep training until reach max iterations\n",
    "    while step < training_iters:\n",
    "        batch_x, batch_y = mnist.train.next_batch(batch_size)\n",
    "        batch_x_norm = scaler.transform(batch_x)\n",
    "        # Run optimization op (backprop)\n",
    "        sess.run(optimizer_ReLU, feed_dict={x: batch_x, y: batch_y,\n",
    "                                       keep_prob: keep_prob_ReLU})\n",
    "        sess.run(optimizer_SNN, feed_dict={x: batch_x_norm, y: batch_y,\n",
    "                                       dropout_prob: dropout_prob_SNN,is_training:True})\n",
    "        \n",
    "        \n",
    "        if step % display_step == 0:\n",
    "            #batch_x, batch_y = mnist.test.next_batch(batch_size)\n",
    "            #batch_x_norm = scaler.transform(batch_x)\n",
    "            # Calculate batch loss and accuracy\n",
    "            loss_ReLU, acc_ReLU = sess.run([cost_ReLU, accuracy_ReLU], feed_dict={x: batch_x,\n",
    "                                                              y: batch_y,\n",
    "                                                              keep_prob: 1.0})\n",
    "            training_loss_protocol_ReLU.append(loss_ReLU)\n",
    "            \n",
    "            loss_SNN, acc_SNN = sess.run([cost_SNN, accuracy_SNN], feed_dict={x: batch_x_norm,\n",
    "                                                              y: batch_y,\n",
    "                                                              dropout_prob: 0.0, is_training:False})\n",
    "            training_loss_protocol_SNN.append(loss_SNN)\n",
    "            \n",
    "            print( \"RELU: Nbr of updates: \" + str(step+1) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss_ReLU) + \", Training Accuracy= \" + \\\n",
    "                  \"{:.5f}\".format(acc_ReLU))\n",
    "            \n",
    "            print( \"SNN: Nbr of updates: \" + str(step+1) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss_SNN) + \", Training Accuracy= \" + \\\n",
    "                  \"{:.5f}\".format(acc_SNN))\n",
    "        step += 1\n",
    "    print(\"Optimization Finished!\\n\")\n",
    "\n",
    "    # Calculate accuracy for 256 mnist test images\n",
    "    print(\"ReLU: Testing Accuracy:\", sess.run(accuracy_ReLU, feed_dict={x: mnist.test.images[:512],\n",
    "                                      y: mnist.test.labels[:512],\n",
    "                                      keep_prob: 1.0}))\n",
    "    print(\"SNN: Testing Accuracy:\", sess.run(accuracy_SNN, feed_dict={x: scaler.transform(mnist.test.images[:512]),\n",
    "                                      y: mnist.test.labels[:512],\n",
    "                                      dropout_prob: 0.0, is_training:False}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABUOklEQVR4nO3dd3iUVfbA8e+dSSUJgVRKKIEAIYQeekdBqthQUERs2HV1XVdFd137qr9d195QFMVeQUGQLp3Qey+hBkJCCaTe3x83A+mZJNOSnM/z5Jlk5p13zgyak3vvec9VWmuEEEIIT2NxdwBCCCFEcSRBCSGE8EiSoIQQQngkSVBCCCE8kiQoIYQQHsnL3QFURlhYmG7atKm7wxBCCFEJiYmJJ7TW4YXvr9IJqmnTpqxevdrdYQghhKgEpdT+4u6XKT4hhBAeSRKUEEIIjyQJSgghhEeq0mtQQoiaJysri6SkJC5cuODuUEQ5+fn5ERUVhbe3t13HS4ISQlQpSUlJBAUF0bRpU5RS7g5H2ElrzcmTJ0lKSiI6Otqu58gUnxCiSrlw4QKhoaGSnKoYpRShoaHlGvlKghJCVDmSnKqm8v67SYISQgjhkWpsgtJa88+fN/HzukPuDkUIUcUEBgY6/TWsVisdOnQgPj6ekSNHkpqaWurxzzzzDK+99lqB+yZMmMB3331X4L7SYv/ss8+Ij4+nTZs2xMXFXTzfhAkTaNiwIRkZGQCcOHECWxefffv2oZTizTffvHie+++/nylTptj5TktWYxNURnYu246e4S9fr+PLlQfcHY4QQhTg7+/PunXr2LRpEyEhIbz99ttOfb2ZM2fy+uuvM3v2bDZv3syaNWsIDg6++LjVauXjjz8u9rkRERH873//IzMz06Ex1dgE5edtZcqtXenXMpwnftjIR4v3uDskIUQVtm7dOrp37067du24+uqrOXXqFABvvPEGcXFxtGvXjjFjxgCwcOFCOnToQIcOHejYsSNnzpwp9dw9evTg0CEz27N7926GDBlC586d6dOnD9u2bXNI/C+99BKvvfYaDRo0AExJ+J133nnx8b/85S/897//JTs7u8hzw8PDueyyy/j0008dEotNjS4z9/ex8sHNCTz01Vqe/3Ur6Zk5PDAwRhZghagi/jV9M1sOn3boOeMa1OafI9uU+3njx4/nzTffpF+/fvzjH//gX//6F6+//jovv/wye/fuxdfX9+I03Wuvvcbbb79Nr169OHv2LH5+fiWeNycnh7lz53L77bcDMHHiRN577z1atGjBihUruPfee5k3b16F3mt+mzZtonPnziU+3rhxY3r37s3UqVMZOXJkkccff/xxhg4dym233VbpWGxq7AjKxsfLwptjO3JNp4b8Z84OXp65Da21u8MSQlQhaWlppKam0q9fPwBuueUWFi1aBEC7du246aab+Pzzz/HyMmOCXr168cgjj/DGG2+Qmpp68f78zp8/T4cOHQgNDSUlJYVBgwZx9uxZli5dyujRo+nQoQN33XUXR44cKTGu4v7Yrswf4E8++SSvvvoqubm5RR6Ljo6ma9euTJs2rcLnL6xGj6BsvKwWXruuPQE+Xry/aA9nM7J5blQ8FouMpITwZBUZ6bjar7/+yqJFi/jll1947rnn2Lx5M48//jjDhw/nt99+o3v37vzxxx/ExsYWeJ5tDSotLY0RI0bw9ttvM2HCBOrUqcO6devseu3Q0NCLU40AKSkphIWFATBp0iR+/fVXwExPtmnThsTERAYOHFji+WJiYujQoQPffPNNsY8/+eSTXHfddfTt29eu+MpS40dQNhaL4tlRbbirXzO+WHGAR79dT3ZO0b8ShBCisODgYOrWrcvixYsBmDp1Kv369SM3N5eDBw8yYMAAXnnlFVJTUzl79iy7d++mbdu2/P3vfychIaHUdaTg4GDeeOMNXnvtNfz9/YmOjubbb78FTDXy+vXrS3xu//79+frrry8WL0yZMoUBAwYA8MILL7Bu3bqLye6JJ57gscce4+jRowBkZGTwxhtvFDnnpEmTilQL2sTGxhIXF8eMGTPK+MTsIyOofJRSPD4kliBfL16bvQMU/N/o9rImJYQoID09naioqIs/P/LII3z66afcfffdpKen06xZMz755BNycnIYN24caWlpaK15+OGHqVOnDk8//TTz58/HarUSFxfH0KFDS329jh070r59e7766iu++OIL7rnnHp5//nmysrIYM2YM7du3B+D555/n9ddfv/i8pKQkEhMT6dy5M1arlebNm/Pee+8V+xrDhg3j2LFjXH755WitUUoVu57Upk0bOnXqxJo1a4o9z6RJk+jYsWNZH6FdVFVeb0lISNDO2rDwf3/s5L9/7OCBgTH8dXArp7yGEKL8tm7dSuvWrd0dhqig4v79lFKJWuuEwsfKCKoED14Ww+HU87w5bxcN6vgztmtjd4ckhBA1iiSoEiileP7qeI6evsBTP22iXrAfA1pFuDssIYSoMaRIohTeVgtv39SJ2HpB3PfFGjYmpbk7JCGEqDEkQZUh0NeLTyZ0oW4tH26dsoqDKenuDkkIIWoESVB2iKjtx6e3dSEzO4cJn6wkNd2x/aaEEEIUJQnKTjERQXw4PoGDKeeZ+FkiF7Jy3B2SEEJUazU3QeXmwvqvYc9Cu5/SrVko/3d9e1buS+HjJXudGJwQwpO5YruNF154gTZt2tCuXTs6dOjAihUrAHPxbULCpYrs1atX079/fwAWLFiAUorp06dffHzEiBEsWLDA6fE6Qw1OUNmw8N/w26OQk2X300a2b0CfFmFMWbKPzGzpNCGEcLxly5YxY8YM1qxZw4YNG/jjjz9o1KjRxcePHz/OzJkzi31uVFQUL7zwgqtCdaqam6C8fOCKF+DEDlj1UbmeekefZhw/k8Ev6w87KTghRFXjyO02jhw5QlhYGL6+vgCEhYVd3AYD4G9/+xvPP/98sXG0b9+e4OBg5syZ44y36VJV8joopdRIYGRMTEzlTtRyCDQbAAtegrbXQ0CoXU/r2yKMVpFBfLR4D9d2aiitkIRwl5mPw9GNjj1nvbYw9OVyP82R220MHjyYZ599lpYtW3L55Zdzww03XOyUDmZ/qB9//JH58+cTFBRUJJannnqKp556ikGDBpX7fXiSKjmC0lpP11pPzL/bY4UoBUNegoyzsODFcjxNcWffZmw7eobFO09ULgYhRJXn6O02AgMDSUxM5IMPPiA8PJwbbrihyBbqTz31VImjqD59+gBcbF5bVVXJEZRDRbSGhNtg9WRIuB0i4+x62pXtG/DKrG18uHgPfVuGOzlIIUSxKjDScbWKbrdhtVrp378//fv3p23btnz66adMmDDh4uMDBw7k6aefZvny5cW+7qRJk3jhhReK3WuqqqiSIyiHG/Ak+NaG358AO5vn+nhZmNCrKYt3nmDrEcfu6CmEqFocvd3G9u3b2blz58Wf161bR5MmTYq87qRJk3jllVeKjWnw4MGcOnWq1O04PF3VTa2OVCsE+j8Bs/4O22dC7DC7nnZT1ya8NW8XHy7ew3+u7+DcGIUQHsPZ222cPXuWBx544OL0X0xMDB988EGROIYNG0Z4eMkzOJMmTWLUqFGOe+MuJttt2ORkwbu9IDcL7l0OXr52Pe2ZXzbzxYr9LH5sIPWC/cp+ghCiUmS7jaqtPNttyBSfjdUbrngRUvbAivftftrtvaPJydVMWbrPebEJIUQNJAkqvxaXQ4vBsOhVOJts11MahdRiaHx9vlixn7MZ2U4OUAghag5JUIVd8SJkpcO85+x+yh19ojlzIZtvVh10YmBCCJuqvDRRk5X3300SVGFhLaDrRFjzGRzZYNdTOjauS5emdZn8516yc6T9kRDO5Ofnx8mTJyVJVTFaa06ePFnkouTSSBVfcfr9HdZ9AUteh+s+tuspd/ZpxsSpiczafJQR7RqU/QQhRIVERUWRlJREcrJ90/DCc/j5+RWofiyLJKji+NeB9mNh9cdw7gQEhJX5lMtbRxIdFsCHi/YwvG19aX8khJN4e3sTHR3t7jCEC8gUX0k6T4CcTFg3za7DLRbF7b2jWZ+Uxh2frmbFHpmCEEKIypAEVZKI1tCoOyROsbu7xJgujXj48pasPZjKDR8sZ9TbS/hl/WFZlxJCiAqQBFWazhMgZTfss6/hopfVwkOXt2Dp4wN54ep4zlzI5sEv19Lv1QV8tHiPlKELIUQ5SIIqTZurwC/YjKLKwc/byk3dmjD3kX58OD6BhnX8ef7XrfT+9zw2JKU6I1IhhKh2JEGVxtvfFEtsnW6KJcrJYlEMiovkm7t78NN9vQjy82L8xyvZfvRM2U8WQogaThJUWcpZLFGSDo3q8MXt3fH1sjBu8gr2njjnmPiEEKKakgRVlgoUS5SkcWgtvrijGzm5mnEfreBQ6nnHxCiEENWQJCh7lLNYojQxEUF8dltXTl/IYtxHKzh+5kLl4xNCiGpIEpQ9KlgsUZL4hsFMubULx05fYPzklaSmZzrkvEIIUZ1IgrKHrVhiyy8VKpYoTucmIXw4PoE9J85xy8crOXMhyyHnFUKI6kISlL06TzCbGVayWCK/XjFhvHNjJzYfPs3o95bx7PQtfLZsH4t2JHPgZDo5udKJQghRc8mOuuUx+Qo4lwwPJIIDe+3N2nSUN+buZN/Jc6Rn5ly839uqaBRSi0FxkTw+JFb6+wkhqqWSdtSVZrHl0XkC/HS3KZaI7uuw0w6Jr8eQ+HporUk+k8HeE+fYd/Ic+06ms+lQGu8v3ENMeCCjExo57DWFEMLTSYIqjzZXway/m2IJByYoG6UUEbX9iKjtR7dmoQDk5GrGfricf03fQo/moUTVreXw1xVCCE8ka1Dl4YRiibJYLYr/G90erTWPfrueXFmXEkLUEJKgyqvjzaZYYtsMl71ko5Ba/HNkG5bvSeGTpftc9rpCCOFOkqDKKyIOfILg6EaXvuzohCgubx3Bv2dtY+cx6eUnhKj+JEGVl8UC9eLh6CaXvqxSipeuaUegrxePfLOeLNljSghRzUmCqojIeDi2CXJdmyTCg3x54ap4Nh5K4615u1z62kII4WqSoCqiXjxknoXUfS5/6aFt63NNx4a8NX8X6w+muvz1hRDCVSRBVURkW3Pr4mk+m39e2YaIIF8e/mYdF7Jyyn6CEEJUQXIdVEVEtAZlMYUScVe6/OWD/b15bXR7bvpoBWM+WE6HRnWIqutPVN1aNAoxt8H+3i6PSwghHEkSVEX41ILQGLMO5Sa9YsJ4anhrvktM4rvEJM5mZBd4vLafF6MTGvHE0Fi8rDJQFkJUPZKgKioyHpJc2AewGHf0acYdfZqhtSY1PYukU+c5eCqdpFPpbEhKY/Kfe9mdfJa3buxEoK/8Uwshqhb5rVVR9drC5h/gfCr413FrKEop6gb4UDfAh7ZRwRfv79n8AE//vInr3l3KJ7d2oX6wvxujFEKI8pG5n4qql1co4cZpvrLc2K0xn0zoQtKp84x6awmbDqW5OyQhhLCbJKiKiow3t26q5LNX35bhfH9PT7ytFka/t4w/thxzd0hCCGEXSVAVFVQPaoXCMde2PKqIVvWC+PG+nrSIDOTOqav5+M+9VOV9wIQQNYMkqIpSykzzefgIyiYiyI+vJ/ZgcFwkz87YwhRpOiuE8HCSoCojMh6Ob4Wc7LKP9QD+PlbevakzfVuG8585Ozh1LtPdIQkhRIkkQVVGvbaQkwEnd7o7ErtZLIqnhrfmXEY2b0o/PyGEB5MEVRn13NvyqKJaRgZxQ5dGTF2+j/0nz7k7HCGEKJZcB1UZYS3B6pNXKDHa3dGUy8OXt+TndYd5ZdZ23r6pU5nHH0xJ5+WZ29Bogv19qFPLmzr+3ua2lg/NwwOJiQh0QeRCiJpCElRlWL0hvJXLNy90hIjaftzZpxn/m7uT2w+colPjuiUee/pCFrdNWcXh1PPUr+NPanoWaeczycq5VAloUfDTfb1oF1XHBdELIWoCj0lQSqkA4B0gE1igtf7CzSHZJ7It7PrD3VFUyMS+zZi28gAv/rqVb+/ugVKqyDHZObk8MG0te0+c47Pbu9KzeRgAWmvSM3NIPZ/FybMZjPtoBW/N28UH4xNc/TaEENWUU9eglFIfK6WOK6U2Fbp/iFJqu1Jql1Lq8by7rwG+01rfCbi+RXhF1WsL547Dmap3AWyArxcPX96S1ftP8fvm4uN/8bdtLNyRzHNXxV9MTmDaKwX4etGwjj/touowoVc0s7ccY/tR2Y5eCOEYzi6SmAIMyX+HUsoKvA0MBeKAsUqpOCAKOJh3WNXZ5KheXkeJKnDBbnGuT4giJiKQf8/aVmQb+S9XHuDjJXu5rVc0Y7s2LvU8t/ZsSi0fK+8ukMpAIYRjODVBaa0XASmF7u4K7NJa79FaZwJfAaOAJEySKjUupdREpdRqpdTq5ORkZ4RdPlWk5VFJvKwWnhgay94T55i24sDF+5ftPsnTP22if6twnhwWW+Z56gb4MK57E35Zf1gqA4UQDuGOMvOGXBopgUlMDYEfgGuVUu8C00t6stb6A611gtY6ITw83LmR2qNWCNSO8uimsWUZGBtB92Yh/G/uTk5fyGLfiXPc80Ui0WEBvDG2o937Sd3ROxovq4X3Fu52csRCiJrAHQmq6Eo8aK31Oa31rVrre6pMgYRNvfgqO4ICs540aVgcKecyee337dz+6SoUMPmWLtT2s39n3ojaflyfEMV3iUkcTbvgvICFEDWCOxJUEtAo389RwGE3xOE4kfFwYgdkVd1fym2jgrmqQwM+W7afAynpvDeuM41Da5X7PHf1bU6uhg8W7XFClEKImsQdCWoV0EIpFa2U8gHGAL+4IQ7HqdcWdA4kb3V3JJXy6BWtaBYewMvXtKNbs9AKnaNRSC2u6tCQaSv3c/JshoMjFELUJM4uM/8SWAa0UkolKaVu11pnA/cDvwNbgW+01pudGYfTVdGWR4VF1a3FvL/259rOUWUfXIp7+jcnIzuXT5bsc0xgQogayakX6mqtx5Zw/2/Ab858bZeqGw3eAVW6UMKRYiICGRpfj0+X7WNiv2blWscSQggbaRbrCBYLRMZVyZZHznJv/xjOXMhm6rL97g5FCFFFVckEpZQaqZT6IC0tzd2hXGLbvFB2qgUgvmEw/VuFM/nPvZzPrDrXXQshPEeVTFBa6+la64nBwcHuDuWSyHjISIO0g2UfW0PcPyCGlHOZfLnyQNkHCyFEIR7TLLbKu1gosRHqlN4WqKZIaBpCt+gQXp61jcl/7sXbqvCyWvC2WvC2KrytFlpGBvH3Ia2oU8vH3eEKITyMJChHiYgDlJnmix3u7mg8xr9GteHTpfvJzM4lOzeX7BxNZk4u2Tm5ZGTn8u3qg8zfdpz/XN+enjFhZZ9QCFFjSIJyFN9ACGlWZZvGOktsvdq8dE3bEh/fdCiNB79ay40freDOPtE8ekUrfL2sLoxQCOGpylyDUkq9opSqrZTyVkrNVUqdUEqNc0VwVU4Vb3nkDvENg/n1gT7c3L0JHy7ey6i3lsiWHUIIwL4iicFa69PACEybopbA35waVVUV2RZO7YULp90dSZXi72Pluavi+XhCAifOZjDyrT/5+M+95OZKRaQQNZk9Ccp2leUw4EutdeHtM4SNrVDiUKJ746iiBsZGMusvfekTE8azM7ZwzbtL+XXDEbIL7VMlhKgZ7ElQ05VS24AEYK5SKhxwa1dUj7wOCiC6L/gGw7qq1Yzdk4QF+vLRLQn8+9q2nDyXwX3T1tD3lfm8t3A3qemZ7g5PCOFCSttxYalSqi5wWmudo5SqBdTWWh91enRlSEhI0KtXr3Z3GAXN/Dus/hge2QoBUpVWGTm5mrlbj/HJkn0s23MSf28r13RqyK29mhITEeTu8IQQDqKUStRaJxS+354iidFAdl5yegr4HGjghBirh863Qk6mjKIcwGpRDG5Tjy8ndmfmQ30Y2b4+3yYmcfl/FvHkjxvJkTUqIao1e6b4ntZan1FK9QauAD4F3nVuWFVYRCw07gmrP4FcWTtxlNb1a/PKde1Z9vhAbu8dzbQVB3jwy7VkZle9zzgnV1fJuIVwNXsSlK2R2nDgXa31z4Bc9l+ahNtMNd/eBe6OpNoJDfTl6RFxPDW8Nb9uPMLEqaurVK+/85k5XPX2Em6dshJ7pteFqMnsSVCHlFLvA9cDvymlfO18Xs0VdyXUCjWjKOEUd/RpxsvXtGXhjmRu+XglZy5kVfhcMzceoe0zvzPxs9X8seWY06oGtdY89dMmNh5KY8muk6zcKwWxQpTGnkRzPWZzwSFa61QgBLkOqnRevtDhJtj2K5w+4u5oqq0xXRvzxpiOrDlwihs/XEHKufJX+S3YfpwHv1pLZG0/1hxI5Y7PVtP9pXm89NtWdh137AXDX606yPdrkrirXzNCA3x4b+Fuh55fiOqmzASltU4HdgNXKKXuByK01rOdHllV13mC2QZ+7efujqRaG9m+AR+M78yOY2e44f1lHE2z/wqIFXtOcvfnibSICOL7e3qy7ImBfDg+gU6N6zD5z71c/p9FXPPOEr5ZdbDSBRmbDqXxz18206dFGI9dEcuEnk2Zvz2ZrUfkom4hSmJPFd9DwBdARN7X50qpB5wdWJUX2hyaDYDEKZBbddZIqqKBsZFMubUrh1PPM/r9pXa1StqQlMrtn66mYR1/pt7elWB/b7ytFgbFRfLB+ASWPXEZk4a15syFbB77fgOj31vK7uSzFYovLT2Le75IJDTAh/+N6YjVohjfoykBPlbel1GUECWyZ4rvdqCb1vofWut/AN2BO50bVjWRcBucToKdc9wdSbXXo3ko0+7sztkL2Qz93yL+/t0GjqSdL/bY7UfPMP7jldSp5c3nd3QjNNC3yDHhQb7c2bcZsx/uy//GdGB38jmG/W8xHy3eU67RVG6u5q/fruNo2gXevqkTIQGmvii4ljdjuzZm+oYjHExJr9ibFqKasydBKS5V8pH3vXJOOPbx2E4ShbUaCoGR5sJd4XTtG9Vh7l/7M6FnND+sTaL/qwt4eeY20s5fKqDYd+Ic4yavwMdq4Ys7ulE/2L/UcyqlGNWhIXMe7kufFmE8/+tWbnh/GXtPnLMrpvcW7eaPrceZNKw1nRrXLfDY7X2isSj4aPGe8r9ZIWoAexLUJ8AKpdQzSqlngOXAZKdGVQaP3FG3OFZv6DQeds6GVNlV1hVCAnz4x8g45v21P8Pa1uf9Rbvp+8p8Ply0h30nznHTRyvIzsnl8zu60SQ0wO7zRtT248PxCfzn+vbsOHaGof9bVGZD26W7T/Da79sZ3q4+t/RsWuTx+sH+XNWhIV+vPsjJsxkVebtCVGv2tjrqBPTGjJwWaa3XOjswe3hkq6PCUg/C/9pB70fgsqfdHU2Ns/lwGq/M2s7CHckoBQE+Xnx5Z3faRlX8j5ujaRd44ocNzN+eTKvIIJqFBxAa6ENogC9hgT6EBPgS4Gvl0W/XE+zvzc/39ybQt/it13YdP8Og/y7igQExPDK4VYVjEqIqK6nVUYkJSikVUtoJPaGreZVIUADTboBDa+CRLWZUJVxu6a4TfLZsP3f0iSahaan/adtFa813iUl8l5jEyXOZnDybwan0gtdi+Xtb+fn+XrSMLL1v4MTPVrNibwpLHx9IQAmJTIjqrKQEVdr/DYmA5tJ6ky2Tqbzvmzk0wuos4TbYMctcF9XmKndHUyP1jAlz6JbySilGJzRidEKji/dl5+RyKj2Lk+cySDmbSaOQWjQKqVXmue7u35zZW47x5coD3NGn/P9bHUxJ57XZ23lgYIw00RXVSokJSmsd7cpAqrWYyyG4ESR+IgmqGvOyWggP8iU8qGhVYGk6Na5Lt+gQJv+5l/E9muLjZX+jltX7Upg4NZGUc5loDW+M7VjesIXwWNKyyBUsVuh0C+xZAKf2uTsa4YHu6d+cI2kX+HndIbuf8+PaJG78cAXB/t4Ma1uPmZuOcMKDiy1un7KKN+fudHcYogqRBOUq7Uab2y2/uDcO4ZH6tQyndf3avL9oT5lb3efmal77fTsPf72ezk3q8uO9PXlkUCuycjRfrzrooojL53DqeeZuO86ny/bLNinCbpKgXKVuU6jfHrb87O5IhAdSSnF3v2bsOn6WT5buK7Hs/HxmDg98uZa35u/ihoRGfHpbV+rU8iEmIpCezUOZtuKARyaARTuSAThxNoMVe066ORpRVdjT6iikmC8pRauIuFFwaDWkJbk7EuGBhretT6vIIJ6bsYXOz/9Bn1fmcd+0NXy4aA8r96aw/+Q5xnywjN82HWHSsNa8fG3bAutV47o34VDqeRZsP+7Gd1G8BduTiQjypZaPlekbpIGysI89Na1rgEbAKUwFXx3giFLqOHCn1jrReeFVM61HwdxnYet06H6Pu6MRHsbLauHn+3ux/mAq65NSWX8wjXUHUvk13y90f28r74/rzOA29Yo8f1BcJBFBvkxdvp/LWke6MvRSZeXksmTXCYa3q096Zg6zNh3h2VFt8LZWnwmcv327nlo+ViYNjytXkYsonT0Jahbwo9b6dwCl1GBgCPAN8A7QzXnhFU8pNRIYGRMT4+qXrpywGIhoY6b5JEGJYvh5W+nWLJRuzUIv3pd8JoMNSalsP3aGgbERxNarXexzva0WxnRtzJvzdnLgZDqNQ8sucXeFtQdSOZORTf9W4ViU4pf1h1my6wT9W0W4OzSHOH76At8mmlmR7cfO8N64ztSpJXu6OoI9qT7BlpwA8rba6Ku1Xg6Ur57WQapMq6PixI2CA8vhzFF3RyKqiPAgXy5rHcm9/WNKTE42Y7s2wqIUX6zc76LoyrZg+3GsFkXPmDD6tQonyNeLGdVomm9B3vrafQOas2Z/Kle/s9TuXo2idPYkqBSl1N+VUk3yvh4DTimlrIBzth6tzuJGAdpM8wnhYPWD/bm8dQTfrk7iQpZnbPOycEcynRvXpbafN75eVga1ieT3zUfJyPaM+CprwfbjRNb25dHBrZh2ZzfSzmdx9TtLWC7FIJVmT4K6EYgCfgJ+Bhrn3WfF7LYryiMiFsJaSjWfcJqbuzcl5VwmMze5f5Ry/MwFNh8+Tb9W4RfvG9muAWcuZLN4xwk3RuYYWTm5LN55gv4tI1BKkdA0hJ/u7UVogA83T17Bt6s9s+y/qrBnR90TWusHtNYdtdYdtNb3a62TtdaZWutdrgiy2okbBfuXwLmq/z+o8Dw9m4cSHRbA58vd30F/UV4S6tfyUoLqFRNGnVrezNhw2C0xzd9+nKd/2lTm9Wb2WLP/FGcuZDMg9tL7axxaix/u7UW36FD+9t0G/j1rm0Neqyayp8y8pVLqA6XUbKXUPNuXK4KrtuJGgc6FbTPcHYmohiwWxU3dGpO4/xRbDpe8pfzSXSf4y1dreXb6Fib/uZdZm46y6VAaqemZ2LPLgT0W7kgmLNCXuPqX1s58vCwMaVOPOVuOlTkNeeJsBuM/XsmmQ47Z+23HsTPc98Uapi7fz8KdyZU+34IdyXhZFL0K9XkM9vfmk1u7MLZrY95dsJvJf+6t9GvVRPZU8X0LvAd8RMGNC0VFRcZD3Wgzzdd5grujEdXQdZ2jePX37Xy+Yj8vXt22wGPnMrJ5eeY2pi7fT51a3mRk5XK+UKII8LHSsXFd3r6xE8G1KnbZY06uZvHOZC6LjcRiKbjH6Yh2Dfhq1UEWbD/OkPj6xT5fa82TP2xk0Y5kWtcLIr5h5YqiTl/I4u6pidTy8aKWj5Wpy/YzoJKVhPO3HSehaV2C/Ip+Rt5WCy9eHc/WI6f5Zf1h7uwr/bXLy54Ela21ftfpkdQkSplR1LK3ID0FalV++wch8qtTy4cr2zfgp7WHeGJo7MVfoMt2n+Sx79eTdOo8t/eO5tHBrfDztnAqPYukU+kcOnWeQ6nnOZiSzpcrD3Ln1NVMvb0rvl7WcsewPimV1PSsAutPNt2bhRAa4MP09UdKTFA/rDnE7C3H8PGysGJv5Xb3yc3V/PWb9exPSWfaHd1YsusEb87fxcGUdLs6zhfnaNoFth09w+NDY0s8RinF4DaRvDJrO4dTz9OgTuk7OIuC7CmSmK6UulcpVT9/NwmnR1bdxY2C3GzYPtPdkYhqalz3JqRn5vDj2kOkZ2bzz583MfbD5ViV4pu7evD0iDj8fawopQgJ8KFdVB2Gtq3PHX2a8a9R8bw6uh0r96bw12/WV2gNZeH2ZCwK+hSzzYmX1cLQtvWYu+0Y5zKyizx+OPU8z/yymS5N63Jbr2g2HUor9jh7vbtwN3O2HGPSsNZ0axbKjd2aYFGKz5dXvBzf1rGjrFHY4DhzUfUfW49V+LVqKnsS1C3A34ClmD2iEoEqsEugh2vQEYIbSzWfcJr2jerQLiqYDxfvYcjri/ls+X5u7dWUmQ/1pYsdmzaO6tCQx4fGMmPDEf49a1u5X3/BjmTaN6pD3YDiL1od0a4BF7JymbutYGum3FzNY99tIEdrXhvdnu7NQsjO1aw9kFruGMD0AXxt9nZGdWjArb2aAlAv2I/BcZF8vfpghcvxF2xPpn6wHy0jA0s9LiYikGZhAczeLAmqvOyp4osu5ksmUytLKYi7EnbPgwuOWQAWorBx3ZtwMOU8AF/d2Z1/jmyDv4/903V39W3Gzd2b8P6iPXy6dJ/dz0s5l8mGpNQC1XuFdWkaQkSQLzPWF6zm+3zFfv7cdYJJw1vTJDSAzk3qYlGwcl/5p/kOpqTz4FdraRUZxEvXtEWpS2thN/doQmp6FtPXl7+aMDM7lz/zumHkP2dJBrWJZPmek6SdzyrzWHFJiQlKKTUw7/aa4r5cF2I1FjcKcrNgx+9lHytEBVzbKYr3b+7MrL/0KdA+yV5KKZ65sg2D4iJ5ZvpmZm2yrwPK4p3JaE2p7YysFsXwdvVZsCOZMxfML+69J87x4m9b6dcynBu7NgYgyM+bNg2CWbm3fBe+XsjK4e7PE8nJ1bw3rjO1fAouufdoFkqLiECmVmCaL3H/Kc7mtW+yx+C4emTnao9s5OvJShtB9cu7HVnM1wgnx1UzNEyAoAYyzSecxmpRXNGmXpFfzuU9xxtjOtI+qg4PfbWWxP2nynzOwu3J1K3lTdsyKu9GtGtAZnYuc7YcIydX89dv1uFjtfDva9sVGJl0aRrC2gOpdnef0Foz6cdNbD58mtdv6EDTsIAixyilGN+jCRuS0lh3MNWu89os2H4cb2vR8vKSdGxUh7BAX2ZvkWm+8igxQWmt/5l3e2sxX7e5LsSilFIjlVIfpKVV8akxiwVaj4Rdf0DGWXdHI0SJ/H2sTL4lgfrBftzx6apSe83l5moW7UymT4twrJbSp786Na5Dwzr+zNhwhPcX7WbNgVSeuyqeesF+BY7rGh1CRnau3ddDzd5yjO/XJPHQZS1K7ex+dacoAn29+Kwc05dgLvbt0jSEQF/7Er/FohgUF8GCbcerTYsnV7DnQl1fpdSNSqknlVL/sH25IriSVOlmsYXFjYLsC7BztrsjEaJUoYG+TLm1K0opbvpwORuSUos9bsuR05w4m2nX9JdSZppv0Y5k/jtnB8Pa1uPK9g2KHNelaV0Au8vNf1xziPAgXx68rEWpxwX6enFNp4bM2HCkxE0iCzuUep4dx86W+xqqQXGRnMvMYdlu6dFnL3uq+H4GRgHZwLl8X8IRGneHgAiZ5hNVQtOwAKbebpLUde8t4+tVRdsp2dZZ+rSwb31mRLv6ZOdqgv19eP6qtsUWHYQG+tIiIpCVdiSocxnZzN9+nKHx9cocwQHc3L0JmTm5fG1n3zzb+7N3/cmmZ/MwavlYZZqvHOxJUFFa6xu01q9orf/P9uX0yGoKi9VM8237FZa+Bbky/BeerU2DYKY/0Jtu0SH8/fuNPPHDhgKl2gt3JBPfsDbhQfbtxtO2YTC39YrmzbEdCSmhJB2gS3QIiftOlbml/bxtx8nIzmVY2+IvAC6sRWQQPZqF8sXyA2WeG0x5ecM6/sRElF5eXpift5X+rcL5Y8sx6c1nJ3sS1FKlVNuyDxMVNuBJiLkcZk+CyYPg+FZ3RyREqUICfJhya1fuG9CcL1ce5Pr3l3Eo9Txp57NYcyCV/i3tn/5SSvGPkXH0aF56lWG36BDOZGSz9UjJ/QUBftt4hLBAX7uu9bIZ36MJh1LPM29b6VV2Gdk5eZsthttVXl7YoLhIjp/JYH0J06P2yMjOYe7WY6w5cIojaeftSqpVlT0rfL2BCUqpvUAGZtt3rbVu59TIapKAMBj7JWz6HmY+Bu/1gb5/g94Pg5fszCk8k9Wi+NsVsbSLqsOj36xnxBuLGdWhITm5utj2RpVlSzgr96aU2JcvPdNM743u3Miu6T2bQXGR1Kvtx2fL9jEoruSiitX7TpGemVPhHn4DW0VitShmbzlGx8Z1K3SOjxbv5dXft1/82WpRRAT5Uj/Yj/rB/rSLCuaOPs3K9f49lT0jqKFAC2Awl0rMRzozqBpJKWh7Hdy30hROLHgRPugPh9a4OzIhSnVFm3r8fH8vwgJ9mbJ0H0F+XnRsVMfhr9Ogjj9Rdf1LXYeat+04F7Lsn96z8bJauLFbYxbvPMGe5JIraudvO46P1ULPmPJfUwYQXMubbtEhzKngOlRWTi6fL99P16YhfDKhCy9cHc89/ZrTo3koft5WNh9O46WZ2/jrN+vIzqn6+8mWOIJSStXWWp8GzrgwHhEQBtdNNslqxsPw0WXQcRzEDIImvSCglP8xMs7CnvmwfRbsnmueN/Ap18Uuaqxm4YH8dF8vXvhtKw3r+ONltedv3/LrGh3Cwu3JaK2LnWKzTe91jS5/u9AxXRvx5rydfPTnXp4bFV/sCGTBjmS6NQup1HVlg+MieWb6FvYkn6VZePnWsWZvPsaRtAs8NyqeAbHFj+LemreT12bvIEfDf69v77R/C1co7VOehhktJQIaM7VnowFpd+RMrYZCk54w55+w/itY85m5P6INNO0NTXuZhJV1HnbMMl97F0NOBvgGQ626sPw96PUQ+Aa5972IGiHA16vI1h6O1i06hB/WHGJ38rkiRQrpmdnM23ac6zpHVWh6KyLIjyvbN2TaigP8seUYw9vVZ2T7BnRsVAelFAdT0tl1/CxjujSq1Hu4PC9BzdlyjLv6lS9BTVm6l8YhtUpMTgD3D2yB1WK5uFHi62M64F1Fk1SJCUprPSLvNtp14YgC/IJh5Osw9BU4vAb2/Wm+1k6Fle8XPDakGXS5A1oNgcY94PA6mHw5bPgGutzujuiFcLiu0WYGYeXelCIJav625ApN7+X34jXxDIgNZ/r6w3yx4gCfLNlHVF1/RrRrQG7eJo6lJQd7RNWtRZsGtZm95Rh39Wtu9/M2HUpj1b5TPDW8dZkJ+J7+zfGyKF74bSs5uZo3xnbEx6vqJSm7xqlKqbqYdaiLl3drrRc5KyhRiJePuV6qcXfo+yjkZMHhtSZZWbyg5RAIa2HWsWyiEqBeO1g1GRJuK/iYEFVU09BahAX6snLvSW7s1rjAY2Z6z4du0RVbHwLw9bIyol0DRrRrwOkLWczZfIzpGw7z0eI9ZOdqGoX406yYtknlNTiuHq/P3UHymQy7y/E/XboPf28roxPsG8Hd2dcUSjw7Ywv3TVvD2zd2qnJJqswEpZS6A3gIiALWAd2BZcBAp0YmSmb1hkZdzVdJlDIjqukPwoHl0KSH6+ITwkmUUnSLDmHVvoL9AM9n5jBv23Gu6dTQYdVrtf28ubZzFNd2jiLlXCZzthylaWhAhcrLCxsUF8l//9jB3K3HGNO1cZnHnzybwc/rD3N9QhTB/vbvcHxb72i8rIp//LyZez5P5J1xnSq0+aS72JNOHwK6APu11gOAjkCyU6MSjtH2OrMeteojd0cihMN0jQ7hUOp5kk6lX7xv/vbjnM/KYXglpvdKExLgww1dGleoI3xxWtcPIqquv93VfF+tOkhmdi639Gha7tca36Mpz18Vz9xtx7nn8zVkVaHqPnsS1AWt9QUwffm01tuAVs4NSziETwB0uNG0UTorbf5F9ZD/eiibXzceITTAp0LVe+6glGJQXCSLd50oc6fgrJxcpi7bT58WYbSIrFjB07juTXj+qnjmbTvOUz9uQuuqcXGvPQkqSSlVB/gJmKOU+hko/w5fwj263G72nFo71d2RCOEQreoFUdvPi1V5Gxiez8xh3tbjXBFfr0qVVA+Oq0dmdm6Z287P3nyMo6cvVGj0lN+47k14YGAMX68+yJvzdlXqXK5iz466V2utU7XWzwBPA5OBq5wcl3CUsBYQ3Q9WfyJ9/kS1YLUoujQNudjZfIGTp/ecpVt0CJe3juSlmdv4aPGeEo+zp7TcXo8Mask1nRrynzk7+NaO5rjZOblMW3GA9eXcL8tRSk1QSimLUmqT7Wet9UKt9S9a60znh1ZqXNVjPyhX6XIHpB2ULT1EtdE1OoQ9yedIPpPBrxuPEBLgQ7cqMr1nY7Eo3rmpE0Pj6/H8r1t5e37RUY2ttHx8jyYOKf5QSvHyNe3oHRPGEz9sZNGOkssJdhw7wzXvLuXJHzfy4m/u6Q9aaoLSWucC65VSZZeZuFC12g/KFVoNg6D6Uiwhqo0ueclo8c5k5m07zhVtqtb0no2Pl4U3x3bkqg4NePX37fxnzo4C60NTlu6jlo/9peX2vuY74zoRExHIvV+sYcvhgs13s3NyeXv+Lka88SdJp87TKyaUxP2nOHMhy2Ex2Muef9H6wGal1Fyl1C+2L2cHJhzI6gWdbzU796aUPJUgRFUR3yAYf28r/5u7k/TMqje9l5+X1cL/Xd+B6xOieGPuTl6euQ2tNSfPZvDL+sNc06lhuUrL7VHbz5tPbu1CkJ8Xt05ZyaHU8wDsPHaGa99dyqu/b2dQXCRzHu7LAwNbkJ2rWeqGjRbtuVD3X06PQjhfp/Gw6BVY/TEMft7d0QhRKT5eFjo1qcOSXScJCfChe7OqNb1XmNVipt58vCy8v2gPGdm5hAX6VLi03B71g/355NYujH53Gbd+spIr2zfgjbm7CPTz4u0bOzG8nUn6QX7eBPp6sXBHMle0qeeUWEpiT4IaprX+e/47lFL/BhY6JyThFLXrQ+wIWPs5DJgE3v7ujkiISunSNIQlu05yRZvIKjm9V5jFonhuVDx+XlY++nMvVouqVGm5PWLr1eb9mztzyycreW32Doa1rcezo+IJC7zU3cLHy0LP5qGlNul1Fnv+VQcVc99QRwciXKDLHXD+FGz+0d2RCFFp/VqGoxRc1aGhu0NxGKUUk4a35v4BMeTkam7v7fxWqD1jwphya1c+Gp/AOzd1LpCcbPq1CudQ6nn2nDjn9HjyK227jXuAe4FmSqkN+R4KApY4OzDhBE17Q1grUyzR4UZ3RyNEpXRsXJfEpwaVuk18VaSU4tErWnFb72iXvbdeMWGlPt63hdmAcuH2ZJqXc4uQyihtBDUNszHhL3m3tq/OWutxLohNOJqtP9+hRNkIUVQL1S055edJ761RSC2ahwewsJSydGcoMUFprdO01vu01mO11vvzfZW8naXwfO1vMP35vp0AJ3e7OxohRBXRr2UEy/ec5EKW6y74r/ori6J8/ILh5h8h8yxMHmy27RBCiDL0axVORnbuxQ4eriAJqiaK6gy3/Q7etWDKCNg9v+LnykyHX/8K/xcrDWmFqMa6RYfg62Vh4XbXTfNJgqqpwlrA7bOhThP4YjRs+r785ziyAT7ob4ouzhyB7b85PEwhhGfw87bSvVkoC3e47g9RSVA1We36cOtvENUFvrsdVrxf9nMAcnNh6Vvw0WVwIc1MGQY3hh2/OzdeIYRb9WsZzu7kcxxMSS/7YAeQBFXT+deBm38w/fpmPgZzn4WsCyUff/oIfH4NzJ4EMYPgnqXQfCC0vMJMFWadd1noQgjX6tfKlJsv2umaaT57OkmI6s7bH67/DH59GBb/Hyz+DwQ3gtDm5iukOYTGQMZp+O1vJgmNeB06TzCl6wCthsCqD2HvIpOshBDVTrOwABrW8Wfh9mRu6tbE6a8nCUoYVi8Y+Qa0HGLWllJ2mzL0Dd9CRr5tTeq1g2snQ3jLgs9v2gd8AmH7TElQQlRTSin6tQrnl3WHyczOxcfLuZNwkqDEJUpB7HDzZaM1pJ+Ek7sgPQViLgOvoq1Q8PKF5gPMOpTWl0ZWoqgdv4PFy3yWomzZmeDlORet1nT9WoYzbcUB1hw4RfdmoU59LVmDEqVTCgLCoHF3iB1WfHKyaTkEzhyGoxtKPqam2z0fvhwDP91rik1E6c4cg383hR2y2aan6Nk8FC+LcklXCUlQwnFaXAEo2D7L3ZG4RsZZOLjS/uNT9sJ3t4KXP5w9alpOidLtXwJZ5+DQandHIvIE+XnTuUldl1wPJQlKOE5gOEQlwI6Z9h2/8kP4+X7nxmSPnX/A623h2ObyPW/OP2DyIJj/kpnWLE3GWfjqRnPchBlmim/b9IrHXFMkrTK3KXvdG4cooF+rcLYcOc3x06VU/DpAlUxQSqmRSqkP0tLSyj5YuFbLIaZ90ukjpR+XngJ/PANrp8LxbS4JrVgndsF3t0HqAUj81P7n5WSZbUt8g2Hhy/Dbo5BbQo8yreGneyB5G4z+BBp2MkUlW2eUndhquoMrzO0pSVCepF9LW7n5Cae+TpVMUFrr6VrricHBwe4ORRTWcoi53VnGmsGytyDzHCgrrPu8fK/x22OwblrF4svvwmn4aixYrNCkl+mmkZNl33N3z4fzKXD1e9DzQdNN4/vbITuj6LGLX4Otv8CgZ801YwCtR5hKyWQ3JmdPl3Uejqw338sIyqPE1a9NeJAvi5y8DlUlE5TwYJFtzDVUO0pZh0pPMV0r2lwFrYbC+q/sTwxH1sPK9+HP/1Yuztxc+GGiKaW//jPocR+kn4A9C+x7/sZvwa8OxFwOg5+DQc+ZEdUXoyHjzKXjts+CeS9A2+uhR77pzNgRgIKtMs1XosNrITcbGnU3/zb5P1fhVkop+rYIZ/HOZHJynTcLIAlKOJZSZhRVWlcJ2+ip72PQ8WY4l1z2iMvG1o7pxA5I3lHxOBe8aNbKhrwE0X1MVwy/OrDhm7Kfm3kOtv1qEqyt/LnXg3DVu7DvT9OA99wJE9/3d0D9dnDlGwVL74PqmRZTkqBKZitAaXe9uZVRlEfp1yqcU+lZbDzkvKUWSVDC8VoNgezzpqtEYflHT5FxZgQSGAlr7ZjmO5tsRi6thpmft82oWHxbfoZFr0KHcdB1ornPywfaXG3OmXG29Odvn2kqy9qOLnh/hxthzDRI3m62MvlqrCnLv+EL062jsNYjTEn+qf0Vex/V3cGVpotJVIL5WdahPEqfmDCUwqnVfJKghOM16Q3eAcVP8+UfPYHpYNF+jLl49cyx0s+bOAVyMuHyZ6BBp4olqGOb4cd7oGECjPhPwVFNu+shK73sruwbv4OgBtC4Z9HHWg2B8T+ZKalT++CGqVCnUfHniR1hbrf9Wv73Ud1pbQokGnWDutHmPhlBeZS6AT68dHVbhrat57TXkAQlHM/br2BXCZvCoyebDuNA58CGr0s+Z06WKURoPhDCW5luF4cS4fRh++NKT4Evx4JvENzwedGLjht1N13ZS4sjPQV2/QHx14ClhP99GneHiQvNnltNikliNqHNIaJNxUeC1VnKHpPkG3UFv9pQK1RGUB5oTNfGtIwMctr5JUEJ52g1FE4fKthVovDoySa8pflLee3nJZddb/nZXNza7R7zc+uR5tbe0UdurrlI9swRk5xq1y96jMUCba8z62clbb649RfIzSo6vVdYSPSlqanStB4B+5ea6Utxie36p0bdzG3daJO0RI0iCUo4R4vBFOgqUdLoyabjODixHZJK6Biw4n0IaWbWrADCWpoO6/YmqO2/mQq9IS9Doy4lH9fuBjOa2/RD8Y9v/A5CW0D99va9blliRwBaNnss7OAK8K0N4bHm55BoSNnn1pCE60mCEs4RGAENO19ahypp9GTT5mqzBf3aqUUfO5QISSuh612XptWUMr/c9y2G86fKjmf5O2b6rtMtpR8XEQv12sLGYqr50g6ZKr22ox3XDLdeW7OrsUzzFXRwpRmB2v6960bD6STTOFbUGJKghPO0GgKH18DxraWPnsCsC8VdZUYumYV261zxPvgEmSq5/GJHmOtkds4pPY7D60xPt24TTVFGWdrdYJLiyd0F79/8A6DNNKCjKGWmK/csMBcOl+TMMfjwMlj9ieNe21NdOG2KWWzTe2BGzzrXdPwQNYYkKOE8LYea269uKn30ZNNxHGSeMes8NmeOmaTV8SazWJ5fw84QWK/sa4mWv2P2quo03r64468FVNFrojZ+Bw06muIGR4odYaoTd5WQaDPT4csbTMPUmY/B0Y2OfX1PcygR0KZAwiYkr5JPCiVqFElQwnlsXSVSdpc+erJp0tP8pZz/mqjVH5uiBNv1SvlZLGYLkF1zS74o+PRh08Ko483gZ2drrNoNILqvqeazFW2c2AlH1pVdHFERjbpCQETxiTY3B36404wCR70N/iHm4t+S3m91cHAloMylADZSal4jSYISzqOUqeZDlT16sh3f4SazrpSyx/S1W/2xKbgoadQSO8JcNFtSi6KVH5pf8t3uKl/s7a43f63btsTY+J15H22uKd957GGxmkS7cw5kFeoOPecfZn1qyMtmhHnVO6Z/35x/Oj4OT3FwBUTEFRwxB0aYa+tkBFWjSIISztX/Cbj1t7JHTzbtx4KymGawm3+Cc8eh290lH9+0j+koXlyRQWY6JH5irpmyTRHZq/VI8PK7NIra+K1piVRcebojxI6EzLOwd+Gl+1Z9ZIpLut4F3fM+g5jLoPu9ph9hWWtvVVFurqnkzD+9B+aPl7pNZQQFcO6k6YRSeI20GpIEJZyrVkjpF6sWFtzQXIy7bppZOwpreakDeHG8fKDlYNN+KCe74GPrvzQVfj3uK3/cfsFm9LfpB/MLM2U3xDuwOKKw6L6mrNo2zbdzDvz2N9PXcMhLBY+97J/mAt+f7q1+10+d2A4ZaQULJGxComUEBab7/7zn4c3O8M14OLTG3RE5jSQo4Xk6jjMX+R5ZZ9aeyirpjh0O6Scv7R0E5i/x5e9C/Q7QuEfF4mh7velmMOMvYPGGuCsrdh57ePlAyyvM9VCH18G3EyAyHq6dbKYA8/P2g2s/hAtp8MsD1WtPKdu/YeERFJgR1Kl95t+2JjuUCLUbQu+HYfcC+HAAfHol7J5X8n8LuTnmMgl7LsnwIHbU3ArhYq2GgX9d84uo/diyj4+5HKy+ZpqvaS9z364/4OROuObDil+zFHO5iePYJmg13HzvTLEjzFTilBFmNHXj1+AbWPyxkW1MT8LfnzDTmAm3OTc2Vzm40rQ1CmlW9LGQaMi+YDqK1G7g+tg8xaE1JoFf/k+TpBKnmNmGqVdDvXamkOd8CqQlXfo6fdhcgF63KTywtuQ2XR6makQpahYvXxj5Box6s+Rf0Pn5BkGz/iZB2f6CXP62aegad1Ul4sjrcA6OvfapJLZEi4abvin7l3C3u6HZAJj1pKkyrA5sDWKL+6PiYiVfDW55dPY4pB00l1iAKSTp9SA8tB6ufNNUd855Gpa+ZZK9ytuMs/fD0HmCGYEeWevOd1AuMoISnqm802mxw2Hn7+YaIYvVVPVd9s9L+zVVVI/7zfSIbYsPZ/INhNFTICjSdJgoi8Vi9qB6t6cpPb99TuXfrzudOwknd5lKzuKE5Cs1b9rbdXHZrP3cbJrY7W7HdRIpL9t6ky1B2Xj5muv8Oowz0921QouOktJTYM1U2Dqj6PM9lIygRPXQahigTG++5e+YtkmdJ1T+vKHNzWaD3n6VP5c9YoeV75dH7fomviPrim8TVVG5OaV3tnCGwg1iCwtuZEYE7iiUOLIefnkQZj0OP99v/w7QjnYo0VS5ltQL0mKBwPDip/BqhZgp8Cq0vYskKFE9BIabbS42fGU6QLQfa/6HrAliR5imqhu/q/y5cnNg/dfwdlf4dxPTBWTPQtcUYhxcARYv062jOFZvs7eWq0vNc7JNcqoVAj0fNFV0X44te2NLZziUaK4R8wmo2PNjR5hKySoyJSwJSlQfsSPMHHtOJnS/x93RuI5SpgT+wFKzIF4RuTkmsb/dDX6caK4B63qX2QrksyvhnR6warJpWeUsB1eaRX6fWiUfE9LM9SOoFe+ZEerQf8Pg52DE67B7LkwZXvK2LM6gteltWVICt0fscHNbRZoTS4IS1Yftf74WV0BYC/fG4mrxeR0uStompCS5OWbk9U5301LJ6gPXT4W7FsPQl+GRLabFktUbfn0E/q+1KcpIPVj+GC+kFe2UYZOTZX75Fldenl/daNeOoE7tg/kvmP+mbF1EEm6FMdMgeTtMHuS6C2ZP7TVl4pVZPwqOMpdeVJFpPklQovoIiYZR75hfrDVNaHPzl/Wm7+1/zsndZmT0/e1mau36z+DuP02Bim0Nw9vfXJd21yKzQ3DMZaaLxeTB5dv6IicL3usDr8eb69MKJ6pjmyArvewEFRINF1LLvp7nxE74+mZTFJBxxv4489MaZjxi1nyG/1/BwohWQ2HCDHPuyYNK3sfMkUoqkCiv1iPMet/pI5WPyckkQYnqpeNNxV9DUxPEX2emouz9i37+i+aC6NGfwt1LIG5UydfHKGXW+EZ/YkYPZw4X7Dpflm0zIHW/aYo763F4s5PZOsRWbHBwpbktqUDCxt6msas/NvH9cj+81sp03di3pHxraRu/NVN5l/3DrH0VFpVgKid9g8y1a2s/L9rNxJEOJYKXP0S0rtx5YkeY2yqwSaYkKCGqi/hrAGVfsUTqQdj8o6l0bHNV+S7cjBlk/ghY+YH9z1n5odmY8e7FcMt00wlhxl/grQRY/5VZ66rd0ExBlcaebTe0Nq2vWgyG22ZD22thyy8wZRi80REWvlr2Wt25kyaRNkyALneUfFxoc5Ok6rWFn++Dd7qZIpPcnNLPXxGHEk31ntW7cucJjzX/flVgHUoSlBDVRe0Gpu/hpu/KHimseM/cltaItyQWC3S501TdHV5X9vFHN5kNI7vcYa5Ri+4Lt8+GG781HTN+vAu2/ARRXco+V92m5ra0EVTydpPAWg6Bxt3MBayPboer3zcJcP7z8Ho7+GEiHNtS/Dl+f9KsmV35RtFWU4UFRpjpz+unmgutf5xoik02fOu4RJWTBUc2QMNOlT+XbTfqvYvgfGrlz+dEkqCEqE7ir4UTO0rf1PDCaVjzmRk5FTd1ZY8ON5przVZ9WPaxqz40VYEdx126TynT5HfiQrP21binfW2tfAIgMLL0EdSOmea25ZCCz2s/xqwbPbTeVHlunQHv9oBpYy5NMYLZX2zDV9DrL6allD0sFrN2d/ef5v1YfeCHO0zxycbvKt8/8PhWyD7vuAts7d2N2s2qZIJSSo1USn2Qlpbm7lCE8CxxV5mCh9KKJdZ8BhmnTZeMivKvY/bM2vid6VBQkvOnTPl629HFX5dmsZi1r9tmQqshRR8vTlmVfNtnmXL14IYlPL8pXPECPLzJbAdzcLkpdPhkGGz7DWY8DKEx0Pdv9sWTn+393P2nWduzeJkilBXvlv9c+dn2JXPECArMaDUgwuOn+apkgtJaT9daTwwOtnOHVCFqioBQ059v0w/FT/PlZJvpvSa9Kv/Lrsudpnlr/h2QC1s3zVTndb2zcq+VX0gpCercSUhambdRZhlqhUD/x+HhzXDFS+acX401xRwjK9k9xGIxI9S7l5g1n93zKn4uMAnKv+6lIpHKurgb9R8ll/57gCqZoIQQpYi/FtIOFJy2stnyk2k2WpnRk029eJPoVn1U/FpLbq4pjmjUveTWPBVRN9pUERa37f3O2aBzC07vlcUnAHrca6b+Rr0DV751qSt+ZVkspnQ+aXXlpvkOrzXTe47sAVjcJpkeRhKUENVN7HCz5rOpUDWf1maH3tCY8v0CL03XO82Io7i1jN1zzVqRI0dPkK+Sb3/Rx3bMhMB65mLU8vLyMZcpdLq5UuEVEdXFXLuVUsELejPPwfEt0MBB03s20X3AJ8ijp/kkQQlR3fjVNiXWm38seF3O/qXmL/Hu9zpuP6DYERBUv/iS85UfmIKG1g7e6LFuCaXm2Zmwa55Zy/Kk/Y5s1Ym2ZrjldWS9GRU6ugO5ly+0GGTW3ZxRFu8AHvSvKIRwmLbXwblk2Lf40n3L3gL/EPuq5exl9YbOt5rRUv4LhE/uNqOqzrc6fguQkBIu1t3/J2SegZZ2rD+5UlgrU05f0QTl6AKJ/FqPMLtGFzcd7AEkQQlRHbUYbKZvbNN8J3aZi1e73FF6M9aK6DwBLN5mLcpm9cfm+iFHbHlSWK1Q8wu/8Ahq+yzTaaFZP8e/ZmVYLCa5VDhBrYHgxuZ6K0eLGWRK4j10mk8SlBDVkbe/WYvaMh2yM8wOw1Yfx68HgdlgMW4UrP3CbEGRec7sTdX6SrNflaMpZUrF84+gtDbrT836m/fuaaK6wLHNFesGfyjROaMnMNPB0f0K7kbtQSRBCVFdtb0OMtJgw9em3Lvd9c75Kxyg60TzWhu/MT3sLqSZ+5wlJLrgCOr4Fkg9YP+1VK4W1cWsIx0u53br506YIhRnJSgwf8ic2mc+Qw8jCUqI6qpZf7Pm9Ntj5nolR5SWl6RRV3Nx7MoPzVdkW9Nc1lnqRpsqPtvi/vZiukd4koYJ5ra803yO6mBeGttu1Fs9b5pPEpQQ1ZXV20y9ZZ83aw0Rsc57LaXMiOn4FrN1Rtc7HXvNTmEh0ZCbZbqxA+yYZbYbCarnvNesjIBQ06C1vNtyXNzivYNTwgLMFG3T3rDoVfjjX87dlLKcJEEJUZ11HGfWnno/7PzXir8W/OqAX7BpbeRMF7fd2ANnk80vfk+r3issqosZQZVnrefwGtOJwjfQeXEBXPeJmRL+8z/wVlfY8rNHrElJghKiOotKgMcPOq4zQml8asFV75ovR1cKFpa/1Hzn74D23PUnm6gucPaY6eRhD62dWyCRX2A4XP2e6cruXxe+GQ9TrzYbP7qRJCghqrvK9JQrr9hhZtHd2Wo3NKXtp/aa9afaDc0amCeLKuc6VOp+SD/p+A4SpWncHSYugKGvmPWvd3rAH8+4bdpPEpQQouqxWKFuE7P30+75pjjCmWtejhAZb1pQ2bsOdfECXScWSBTH6gXd7oIHVpvKzz//C7/+1bUx5JEEJYSomupG53XjPmdf93J3s3qbQg67E9QaswGivXtSOVpgBFz1jrm4e/OP5tIBF5MEJYSomkKizaZ73gHQtI+7o7FPVILprZedUfaxh9Y4Zov3yupwo7lMYfNPLn9pSVBCiKrJVsnXfIBr19kqI6oL5GTA0U2lH5eTDUfWuX56rzgNOpl+guumufylJUEJIaqmkGbm1lMvzi2OvZ3Nj200Gz26ooKvLEpBh7Fm5+GTFdwypIIkQQkhqqbmA2DQc+b6naqidgNTcVhWglryhpm6bD7QNXGVpd0N5oLh9V+59GUlQQkhqiYvX+j1oGc2hy1NVELpCeroRtj8A3S/BwLCXBdXaWo3MK2z1n9VuZ2By0kSlBBCuFJUF3ON09njxT8+73nTjaPnA66Nqyztb4S0A7B/icteUhKUEEK40sV1qGLKzQ+sMH0Fe/0F/Ou4MqqyxQ43e4yt/9JlLykJSgghXKl+e7B4FZ3m0xrmPQcBEeZCWU/jUwvaXGX69Lmos4QkKCGEcCVvf6jXtmiC2rMA9i2Gvo+CT4BbQitThxsh8yxsne6Sl5MEJYQQrhbVxVyIa9vPSmuY+ywEN4LOE9waWqka9zC7GbvomihJUEII4WpRXUyLpuNbzc/bfjVba/R/3FQneiqloP1Y2LsI0pKc/nKSoIQQwtXydzbPzTGVe6EtoN0Y98Zlj/ZjAO2Sa6IkQQkhhKvVjYZaoaaSb+N3kLwVBjxpOol7urpNoUkvU83n5E0NJUEJIYSrKWWm+Q4sgwUvmqKJuKvcHZX92o+Fk7vKv4V9OUmCEkIId4hKgJTdcGofDPwHWKrQr+O4UeDlD+udWyxRhT4RIYSoRmwX7DbqDi0GuTeW8vKrDa1HwqbvIeuC015GEpQQQrhDVFdoNRyGvuz5uwEXp8NYs4nhjplOe4kqsCInhBDVkE8tGOv6PZYcJrofhLWEM8ec9hKSoIQQQpSfxQr3rnDq2plM8QkhhKgYJxd2SIISQgjhkSRBCSGE8EiSoIQQQngkSVBCCCE8kiQoIYQQHkkSlBBCCI8kCUoIIYRHkgQlhBDCIynt5P08nEkplQzsr+RpwoATDginOpHPpCD5PIqSz6Qo+UyKsvczaaK1Di98Z5VOUI6glFqttU5wdxyeRD6TguTzKEo+k6LkMymqsp+JTPEJIYTwSJKghBBCeCRJUPCBuwPwQPKZFCSfR1HymRQln0lRlfpMavwalBBCCM8kIyghhBAeSRKUEEIIj1RjE5RSaohSartSapdS6nF3x+MOSqmPlVLHlVKb8t0XopSao5TamXdb150xuppSqpFSar5SaqtSarNS6qG8+2vk56KU8lNKrVRKrc/7PP6Vd3+N/DzyU0pZlVJrlVIz8n6u0Z+JUmqfUmqjUmqdUmp13n2V+kxqZIJSSlmBt4GhQBwwVikV596o3GIKMKTQfY8Dc7XWLYC5eT/XJNnAX7XWrYHuwH15/23U1M8lAxiotW4PdACGKKW6U3M/j/weArbm+1k+Exigte6Q79qnSn0mNTJBAV2BXVrrPVrrTOArYJSbY3I5rfUiIKXQ3aOAT/O+/xS4ypUxuZvW+ojWek3e92cwv4AaUkM/F22czfvRO+9LU0M/DxulVBQwHPgo3901+jMpQaU+k5qaoBoCB/P9nJR3n4BIrfURML+sgQg3x+M2SqmmQEdgBTX4c8mbyloHHAfmaK1r9OeR53XgMSA33301/TPRwGylVKJSamLefZX6TLwcHGBVoYq5T+rtxUVKqUDge+AvWuvTShX3n0zNoLXOAToopeoAPyql4t0cklsppUYAx7XWiUqp/m4Ox5P00lofVkpFAHOUUtsqe8KaOoJKAhrl+zkKOOymWDzNMaVUfYC82+NujsfllFLemOT0hdb6h7y7a/znorVOBRZg1i1r8ufRC7hSKbUPszwwUCn1OTX7M0FrfTjv9jjwI2YppVKfSU1NUKuAFkqpaKWUDzAG+MXNMXmKX4Bb8r6/BfjZjbG4nDJDpcnAVq31f/I9VCM/F6VUeN7ICaWUP3A5sI0a+nkAaK2f0FpHaa2bYn53zNNaj6MGfyZKqQClVJDte2AwsIlKfiY1tpOEUmoYZh7ZCnystX7BvRG5nlLqS6A/piX+MeCfwE/AN0Bj4AAwWmtduJCi2lJK9QYWAxu5tL7wJGYdqsZ9LkqpdpjFbSvmD9pvtNbPKqVCqYGfR2F5U3yPaq1H1OTPRCnVDDNqArN0NE1r/UJlP5Mam6CEEEJ4tpo6xSeEEMLDSYISQgjhkSRBCSGE8EiSoIQQQngkSVBCCCE8kiQoUaMopZbm3TZVSt3o4HM/WdxrOfg16iulZjvoXGfLeLyOUupeR7yWEBUhCUrUKFrrnnnfNgXKlaDyuuCXpkCCyvdajjQE+N0J5y1OHUASlHAbSVCiRsk3angZ6JO3d83DeQ1RX1VKrVJKbVBK3ZV3fP+8/aGmYS7eRSn1U15DzM22pphKqZcB/7zzfZH/tZTxqlJqU95+OTfkO/cCpdR3SqltSqkv8jpZoJR6WSm1JS+W1/K9hSHAzLznzsj3vt5SSk3I+36fUurfyuzjtFIpFZN3f7RSalnee3wu33MDlVJzlVJr8uKzdfZ/GWie955ezTv2b/k+I9veUAFKqV+V2TNqk+39CVFZNbVZrBCPk9cBACAv0aRprbsopXyBJfmm0roC8VrrvXk/36a1Tslr/bNKKfW91vpxpdT9WusOxbzWNZi9lNpjunasUkotynusI9AG0wtyCdBLKbUFuBqI1VrrfK2GrEArrfWWvIacpTmtte6qlBqP6ZgyAvgf8K7W+jOl1H35jr0AXJ3XFDcMWK6U+iXvM4q3vSel1GCgRd7noYBflFJ9gXDgsNZ6eN5xwWXEJoRdZAQlhDEYGK/MthIrgFDML2OAlfmSE8CDSqn1wHJM0+EWlK438KXWOkdrfQxYCHTJd+4krXUusA4z9XgakzQ+UkpdA6TnHdstLzZ7fJnvtkfe973y3T8137EKeFEptQH4A7P1TGQx5xyc97UWWAPEYt77RuDyvFFbH611mp0xClEqGUEJYSjgAa11gfWdvF5r5wr9fDnQQ2udrpRaAPjZce6SZOT7Pgfw0lpnK6W6ApdhmpHeDwzE7AA9K+/YbAr+gVk4Bm3H9zY3YUZBnbXWWcp06S7uPSngJa31+0UeUKozMAx4SSk1W2v9bDHPF6JcZAQlaqozQFC+n38H7lFmqw2UUi3zujIXFgycyktOsZht4W2ybM8vZBFwQ946VzjQF1hZUmDK7EUVrLX+DfgLZnoQTMKam/f9fiBOKeWbN6V2WaHT3JDvdlne90swCQ9MUsr/no7nJacBQJO8+4v7jG7Liw+lVEOlVIRSqgGQrrX+HHgN6FTSexOiPGQEJWqqDUB23lTdFMz6TFNgTV6hQjLFb089C7g7bzpsO2aaz+YDYINSao3WOn8C+BEzzbYeM4J5TGt9NC/BFScI+Fkp5YcZtTycl9guaK1PA2itDyqlvsl7Hzsx0275+SqlVmD+CB2bd99DwDSl1EOY/a5svgCmK6VWY6YZt+W9xkml1BKl1CZgptb6b0qp1sCyvFqOs8A4IAZ4VSmVC2QB95TwvoQoF+lmLkQVoJQaB0RprV+249h9QILW+oTTAxPCiWQEJUQVkDd9JkSNIiMoIYQQHkmKJIQQQngkSVBCCCE8kiQoIYQQHkkSlBBCCI8kCUoIIYRH+n+paZ6ghG4p/gAAAABJRU5ErkJggg==\n"
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.plot( training_loss_protocol_ReLU, label='Loss ReLU-CNN')\n",
    "ax.plot( training_loss_protocol_SNN, label='Loss SNN')\n",
    "ax.set_yscale('log')  # log scale\n",
    "ax.set_xlabel('iterations/updates')\n",
    "ax.set_ylabel('training loss')\n",
    "fig.tight_layout()\n",
    "ax.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}