BERT_COLING_NER.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_COLING_NER.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "558GInfR5b4X",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "outputId": "3095e1f3-4fdc-4820-be06-39b9823bcf5c"
      },
      "source": [
        "\n",
        "LOAD_FROM_LOCAL = True\n",
        "\n",
        "BASE_URL   = 'https://lthelsinkioffens.slack.com/files/UTPM2EMJS/F016E18GDA5/'\n",
        "TRAIN_PATH = 'NERadx.train'\n",
        "DEV_PATH   = 'NERadx.dev'\n",
        "TEST_PATH  = 'NERadx.test'\n",
        "\n",
        "if LOAD_FROM_LOCAL:\n",
        "  from google.colab import files\n",
        "  uploaded = files.upload()"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-e9a3b234-d41a-4b48-a4a4-b3e7d44c03e8\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-e9a3b234-d41a-4b48-a4a4-b3e7d44c03e8\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving NERadx.dev to NERadx.dev\n",
            "Saving NERadx.test to NERadx.test\n",
            "Saving NERadx.train to NERadx.train\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JmtdQgHB5i5-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Hyperparameters\n",
        "\n",
        "SEED = 12345 #@param {type:\"raw\"}\n",
        "\n",
        "BERT_MODEL = 'english_base_uncased' #@param [\"multilingual\", \"english_base_cased\", \"english_large_cased\", \"english_base_uncased\", \"english_large_uncased\", \"finnish_cased\", \"finnish_uncased\", \"dutch\", \"chinese\", \"german\", \"arabic\", \"greek\", \"turkish\"]\n",
        "#DO_PREPROCESSING = False #@param {type:\"boolean\"}\n",
        "#DO_BALANCING = False #@param {type:\"boolean\"}\n",
        "\n",
        "EPOCHS = 3 #@param [\"2\", \"3\", \"4\"] {type:\"raw\"}\n",
        "MAX_LEN =  48  #@param [\"32\", \"48\", \"64\", \"128\", \"256\", \"512\"] {type:\"raw\"}\n",
        "BATCH_SIZE = 96 #@param [\"128\", \"96\", \"64\", \"32\", \"16\", \"8\"] {type:\"raw\"}\n",
        "LEARN_RATE = 2e-5 #@param [\"3e-4\", \"1e-4\", \"5e-5\", \"3e-5\", \"2e-5\"] {type:\"raw\"}\n",
        "EPSILON = 1e-8 #@param [\"1e-6\", \"1e-7\", \"1e-8\"] {type:\"raw\"}\n",
        "nb_warmup_steps = 0 #@param {type:\"raw\"}\n",
        "\n",
        "#@markdown ---\n",
        "CROSS_VALIDATION = True #@param {type:\"boolean\"}\n",
        "NUM_FOLDS  = 5 #@param [\"3\", \"5\", \"10\"] {type:\"raw\"}\n",
        "\n",
        "CHANGE_SPLIT = True #@param {type:\"boolean\"}\n",
        "PCTG_TRAIN = 0.75 #@param {type:\"slider\", min:0, max:1, step:0.05}\n",
        "PCTG_DEV   = 0.15 #@param {type:\"slider\", min:0, max:1, step:0.05}\n",
        "PCTG_TEST  = 0.1 #@param {type:\"slider\", min:0, max:1, step:0.05}"
      ],
      "execution_count": 85,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mh3eEj4U5lT-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install transformers"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fzI68OBw5ob3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys, io, os, re, csv, json, string, time, datetime, random, unicodedata, itertools, collections, torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from transformers import BertTokenizer, BertForSequenceClassification, AdamW, BertConfig, get_linear_schedule_with_warmup\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler, random_split\n",
        "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n",
        "from sklearn.metrics import classification_report, confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
        "from sklearn.utils import resample\n",
        "\n",
        "% matplotlib inline\n",
        "random.seed(SEED)\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.cuda.manual_seed_all(SEED)\n",
        "pd.set_option('precision', 4)"
      ],
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xE8tbYWY5qtS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "models = {\n",
        "    \"english_base_cased\": ('bert-base-cased',False),\n",
        "    \"english_large_cased\": ('bert-large-cased',False),\n",
        "    \"english_base_uncased\": ('bert-base-uncased',True),\n",
        "    \"english_large_uncased\": ('bert-large-uncased',True),\n",
        "    \"multilingual\": ('bert-base-multilingual-cased',False),\n",
        "    \"finnish_cased\": ('TurkuNLP/bert-base-finnish-cased-v1',False),\n",
        "    \"finnish_uncased\": ('TurkuNLP/bert-base-finnish-uncased-v1',False),\n",
        "    \"dutch\": ('wietsedv/bert-base-dutch-cased',False),\n",
        "    \"chinese\": ('bert-base-chinese',False),\n",
        "    \"german\": ('bert-base-german-cased',False),\n",
        "    \"arabic\": ('asafaya/bert-base-arabic',False),\n",
        "    \"greek\": ('nlpaueb/bert-base-greek-uncased-v1',False),\n",
        "    \"turkish\": ('dbmdz/bert-base-turkish-cased',False)\n",
        "}\n",
        "\n",
        "datasets_olid = {\n",
        "    \"arabic\": \"https://github.com/mapama247/TFM/raw/master/ar20a.tsv\",\n",
        "    \"danish\":  \"https://github.com/mapama247/TFM/raw/master/da20a.tsv\",\n",
        "    \"english\": \"https://github.com/mapama247/TFM/raw/master/en19a.tsv\",\n",
        "    \"greek\":   \"https://github.com/mapama247/TFM/raw/master/gr20a.tsv\",\n",
        "    \"turkish\": \"https://github.com/mapama247/TFM/raw/master/tr20a.tsv\"\n",
        "}"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uczQsMri5s25",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "f7edad2f-1e5c-49c6-8351-4411724551e0"
      },
      "source": [
        "if tf.test.gpu_device_name() == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(tf.test.gpu_device_name()))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('Using', torch.cuda.get_device_name(0), ':)')\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n",
            "There are 1 GPU(s) available.\n",
            "Using Tesla T4 :)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fYf-VuI_5u_y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def prepare_data(sentences, labels, random_sampling=False):\n",
        "  input_ids=[]\n",
        "  attention_masks = []\n",
        "  for sentence in sentences:\n",
        "    encoded_dict = tokenizer.encode_plus(sentence, add_special_tokens=True, max_length=MAX_LEN, pad_to_max_length=True, return_attention_mask=True, return_tensors='pt')\n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "  \n",
        "  # Convert numpy arrays to pytorch tensors\n",
        "  inputs = torch.cat(input_ids, dim=0)\n",
        "  masks =  torch.cat(attention_masks, dim=0)\n",
        "  labels = torch.tensor(labels)\n",
        "\n",
        "  # Create the DataLoader for the given set (iterator to save memory)\n",
        "  data = TensorDataset(inputs, masks, labels)\n",
        "  if random_sampling: # train data\n",
        "      sampler = RandomSampler(data)\n",
        "  else: # dev and test data\n",
        "      sampler = SequentialSampler(data)\n",
        "  dataloader = DataLoader(data, sampler=sampler, batch_size=BATCH_SIZE)\n",
        "  \n",
        "  return dataloader"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cSde3yBM5xQq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train(model, optimizer, scheduler, train_dataloader, dev_dataloader, epochs, verbose=True):\n",
        "  total_t0 = time.time() # Measure the total training time for the whole run.\n",
        "\n",
        "  training_stats = [] # training loss, validation loss, validation accuracy and timings.\n",
        "\n",
        "  for epoch_i in range(0, epochs):\n",
        "      if verbose: print('\\n======== Epoch {:} / {:} ========'.format(epoch_i+1, epochs))\n",
        "      else: print(\"Epoch\",epoch_i+1,\"of\",epochs,\"...\")\n",
        "      \n",
        "      t0 = time.time()     # Measure how long the training epoch takes.\n",
        "      total_train_loss = 0 # Reset the total loss for this epoch.\n",
        "      model.train()        # Put the model into training mode.\n",
        "\n",
        "      # For each batch of training data...\n",
        "      for step, batch in enumerate(train_dataloader):\n",
        "          \n",
        "          if verbose==True and step%40==0 and not step==0: # Progress update every 40 batches.\n",
        "              elapsed = format_time(time.time()-t0) # Calculate elapsed time in minutes.\n",
        "              print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "          # Unpack this training batch from our dataloader and copy each tensor to the GPU using the to() method.\n",
        "          b_input_ids = batch[0].to(device)\n",
        "          b_input_mask = batch[1].to(device)\n",
        "          b_labels = batch[2].to(device)\n",
        "\n",
        "          # Always clear any previously calculated gradients before performing a backward pass.\n",
        "          model.zero_grad()\n",
        "\n",
        "          # Perform a forward pass (evaluate the model on this training batch).\n",
        "          loss, logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "\n",
        "          # Accumulate the training loss over all of the batches so that we can calculate the average loss at the end.\n",
        "          total_train_loss += loss.item()\n",
        "\n",
        "          # Perform a backward pass to calculate the gradients.\n",
        "          loss.backward()\n",
        "\n",
        "          # Clip the norm of the gradients to 1.0 in order to prevent the \"exploding gradients\" problem.\n",
        "          torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "          optimizer.step() # Update parameters and take a step using the computed gradient.\n",
        "          scheduler.step() # Update the learning rate.\n",
        "\n",
        "      avg_train_loss = total_train_loss / len(train_dataloader) # Calculate the average loss over the training data.\n",
        "      if verbose: print(\"\\n  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "      \n",
        "      # Measure how long this epoch took.\n",
        "      training_time = format_time(time.time() - t0)\n",
        "      if verbose: print(\"  Training epcoh took: {:}\".format(training_time))\n",
        "\n",
        "      ########## VALIDATION ##########\n",
        "      t0 = time.time()\n",
        "      model.eval() # Put the model in evaluation mode--the dropout layers behave differently during evaluation.\n",
        "      eval_loss, eval_accuracy, nb_eval_examples = 0, 0, 0      # Tracking variables \n",
        "\n",
        "      # Tracking variables \n",
        "      total_eval_accuracy, total_eval_loss, nb_eval_steps = 0, 0, 0\n",
        "\n",
        "      # Evaluate data for one epoch\n",
        "      for batch in dev_dataloader:\n",
        "          # Unpack the inputs from the dataloader object after adding batch to GPU\n",
        "          b_input_ids = batch[0].to(device)\n",
        "          b_input_mask = batch[1].to(device)\n",
        "          b_labels = batch[2].to(device)\n",
        "          \n",
        "          # Telling the model not to compute or store gradients, saving memory and speeding up validation\n",
        "          with torch.no_grad():        \n",
        "              # Forward pass, calculate logit predictions (output values prior to applying an activation function)\n",
        "              (loss, logits) = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "\n",
        "          logits = logits.detach().cpu().numpy() # Move logits to CPU\n",
        "          label_ids = b_labels.to('cpu').numpy() # Move labels to CPU\n",
        "\n",
        "          total_eval_loss += loss.item() # Accumulate the validation loss.\n",
        "          total_eval_accuracy += flat_accuracy(logits, label_ids) # Calculate the accuracy for this batch of test sentences, and accumulate it over all batches.\n",
        "\n",
        "          nb_eval_steps += 1 # Track the number of batches\n",
        "\n",
        "      # Report the final accuracy for this validation run.\n",
        "      avg_val_accuracy = total_eval_accuracy / len(dev_dataloader)\n",
        "      if verbose: print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
        "\n",
        "      # Report the average loss over all of the batches.\n",
        "      avg_val_loss = total_eval_loss / len(dev_dataloader)\n",
        "      if verbose: print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
        "\n",
        "      # Measure how long the validation run took.\n",
        "      validation_time = format_time(time.time() - t0)\n",
        "      if verbose: print(\"  Validation took: {:}\".format(validation_time))  \n",
        "\n",
        "      # Record all statistics from this epoch.\n",
        "      training_stats.append(\n",
        "          {\n",
        "              'epoch': epoch_i + 1,\n",
        "              'Train Loss': avg_train_loss,\n",
        "              'Valid Loss': avg_val_loss,\n",
        "              'Valid Acc': avg_val_accuracy,\n",
        "              'Train Time': training_time,\n",
        "              'Valid Time': validation_time\n",
        "          }\n",
        "      )\n",
        "\n",
        "  print(\"\\nTraining complete!\")\n",
        "  print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))\n",
        "\n",
        "  df_stats = pd.DataFrame(data=training_stats)\n",
        "  df_stats = df_stats.set_index('epoch')\n",
        "  return df_stats"
      ],
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M-EYsCgq50sR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict(model, dataloader):\n",
        "  print('Predicting labels for test sentences...')\n",
        "\n",
        "  model.eval() # Put model in evaluation mode\n",
        "\n",
        "  predictions = [] # Tracking variable\n",
        "  true_labels = [] # Tracking variable\n",
        "\n",
        "  for batch in dataloader:\n",
        "    batch = tuple(t.to(device) for t in batch) # Add batch to GPU\n",
        "    \n",
        "    b_input_ids, b_input_mask, b_labels = batch # Unpack the inputs from the dataloader\n",
        "    \n",
        "    with torch.no_grad(): # do not compute or store gradients to save memory and speed up prediction\n",
        "      outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask) # Forward pass, calculate logit predictions\n",
        "    \n",
        "    logits = outputs[0] # retrieve the model outputs prior to activation\n",
        "    logits = logits.detach().cpu().numpy() # Move logits to CPU\n",
        "    label_ids = b_labels.to('cpu').numpy() # Move labels to CPU\n",
        "    \n",
        "    predictions.append(logits)    # Store predictions\n",
        "    true_labels.append(label_ids) # Store true labels\n",
        "\n",
        "  print('COMPLETED.\\n')\n",
        "  return predictions, true_labels"
      ],
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "raNBC3DN522k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(predictions, true_labels, avg='macro', verbose=True):\n",
        "  avgs = ['micro', 'macro', 'weighted', 'samples']\n",
        "  if avg not in avgs:\n",
        "    raise ValueError(\"Invalid average type (avg). Expected one of: %s\" % avgs)\n",
        "\n",
        "  # Combine the predictions for each batch into a single list.\n",
        "  flat_predictions = [item for sublist in predictions for item in sublist]\n",
        "  flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "\n",
        "  # Combine the correct labels for each batch into a single list.\n",
        "  flat_true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Compute the results.\n",
        "  precision = precision_score(flat_true_labels, flat_predictions, average=avg)\n",
        "  recall    = recall_score(flat_true_labels, flat_predictions, average=avg)\n",
        "  f1        = f1_score(flat_true_labels, flat_predictions, average=avg)\n",
        "  acc       = accuracy_score(flat_true_labels,flat_predictions)\n",
        "\n",
        "  # Report the results.\n",
        "  if verbose:\n",
        "    print('Accuracy:        %.4f' % acc)\n",
        "    print(avg+' Precision: %.4f' % f1)\n",
        "    print(avg+' Recall:    %.4f' % f1)\n",
        "    print(avg+' F1 score:  %.4f' % f1, \"\\n\")\n",
        "    print(confusion_matrix(flat_true_labels,flat_predictions))\n",
        "    #print(classification_report(flat_true_labels, flat_predictions, digits=2, zero_division='warn'))\n",
        "\n",
        "  return f1, acc"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nyFSZLgf549R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def plot_lengths_distribution(lengths, title=\"Length distribution of tokenized sentences\"):\n",
        "  sns.set(style=\"darkgrid\")\n",
        "  sns.set(font_scale=1.5)\n",
        "  plt.rcParams[\"figure.figsize\"]=(10,5)\n",
        "  lengths = [min(length,MAX_LEN) for length in lengths]\n",
        "  ax = sns.distplot(lengths,kde=False,rug=False, hist_kws={\"rwidth\":5,'edgecolor':'black', 'alpha':1.0}) \n",
        "  plt.title(\"Sequence length distribution\")\n",
        "  plt.xlabel(\"Sequence Length\")\n",
        "  plt.ylabel(\"Counts\")\n",
        "\n",
        "  num_truncated = lengths.count(MAX_LEN)\n",
        "  num_sentences = len(lengths)\n",
        "  print(\"{:.1%} of the training examples ({:,} of them) have more than {:,} tokens\".format(float(num_truncated)/float(num_sentences),num_truncated,MAX_LEN))\n",
        "\n",
        "def plot_value_counts(df, title=\"Label distribution\"):\n",
        "  emotions = ['TRUST','ANGRY','ANTICIP.','DISGUST','FEAR','JOY','SADNESS','SURPRISE']\n",
        "  df2 = df.replace([0,1,2,3,4,5,6,7], emotions)\n",
        "  df2.label.value_counts(normalize=False).sort_index().plot(kind='bar')\n",
        "  plt.xticks(rotation=25)\n",
        "  plt.title(title)\n",
        "  plt.show()\n",
        "\n",
        "def plot_loss(df_stats, plot_train=True, plot_valid=True):\n",
        "  sns.set(style='darkgrid') # Use plot styling from seaborn.\n",
        "  sns.set(font_scale=1.5) # Increase the plot size.\n",
        "  plt.rcParams[\"figure.figsize\"] = (12,6) # Increase the font size.\n",
        "  if plot_train: plt.plot(df_stats['Train Loss'], 'b-o', label=\"Training\")\n",
        "  if plot_valid: plt.plot(df_stats['Valid Loss'], 'g-o', label=\"Validation\")\n",
        "  plt.title(\"Training & Validation Loss\")\n",
        "  plt.xlabel(\"Epoch\")\n",
        "  plt.ylabel(\"Loss\")\n",
        "  plt.legend()\n",
        "  plt.xticks([1, 2, 3, 4])\n",
        "  plt.show()"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y7tGljE558AV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "def flat_accuracy(preds, labels):\n",
        "    \"\"\"Function to calculate the accuracy of our predictions vs labels\"\"\"\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "\n",
        "def format_time(elapsed):\n",
        "    \"\"\"Function for formatting elapsed times: Takes a time in seconds and returns a string hh:mm:ss\"\"\"\n",
        "    elapsed_rounded = int(round((elapsed))) # Round to the nearest second.\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded)) # Format as hh:mm:ss\n",
        "\n",
        "def print_model_params(my_model):\n",
        "  \"\"\"Function to print all the model's parameters as a list of tuples: (name,dimensions)\"\"\"\n",
        "  params = list(my_model.named_parameters())\n",
        "  print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "  print('==== Embedding Layer ====\\n')\n",
        "  for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "  print('\\n==== First Transformer ====\\n')\n",
        "  for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "  print('\\n==== Output Layer ====\\n')\n",
        "  for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26TrJF805-Fu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 418
        },
        "outputId": "245b98e4-c6e9-4712-8908-391a053445ed"
      },
      "source": [
        "df_train = pd.read_csv(\"NERadx.train\", delimiter='\\t', header=None, names=['label','sentence']).sample(frac=1, random_state=SEED)\n",
        "df_dev = pd.read_csv(\"NERadx.dev\"  , delimiter='\\t', header=None, names=['label','sentence']).sample(frac=1, random_state=SEED)\n",
        "df_test  = pd.read_csv(\"NERadx.test\" , delimiter='\\t', header=None, names=['label','sentence']).sample(frac=1, random_state=SEED)\n",
        "\n",
        "df_train[\"label\"].replace({8: 0}, inplace=True)\n",
        "df_dev[\"label\"].replace({8: 0}, inplace=True)\n",
        "df_test[\"label\"].replace({8: 0}, inplace=True)\n",
        "\n",
        "df_data = pd.concat([df_train, df_dev, df_test]).sample(frac=1, random_state=SEED)\n",
        "NUM_CLASSES = len(df_data.groupby('label'))\n",
        "\n",
        "if CHANGE_SPLIT:\n",
        "  if PCTG_TRAIN+PCTG_DEV+PCTG_TEST!=1:\n",
        "    raise SystemError('Check train-dev-test percentages! The sum of them should be 1.')\n",
        "  else:\n",
        "    print('The entire dataset was split using',PCTG_TRAIN,'for training,',PCTG_DEV,'for validation and',PCTG_TEST,'for testing.\\n')\n",
        "    df_train, df_dev, df_test = np.split(df_data, [int(PCTG_TRAIN*len(df_data)), int((1-PCTG_TEST)*len(df_data))])\n",
        "else:\n",
        "  print('The original train-dev-test sets are used.\\n')\n",
        "\n",
        "X_train = df_train.sentence.values\n",
        "y_train = df_train.label.values\n",
        "X_dev = df_dev.sentence.values\n",
        "y_dev = df_dev.label.values\n",
        "X_test = df_test.sentence.values\n",
        "y_test = df_test.label.values\n",
        "\n",
        "print('Number of sentences train set: {:,}'.format(len(X_train)))\n",
        "print('Number of sentences dev set:   {:,}'.format(len(X_dev)))\n",
        "print('Number of sentences test set:  {:,}'.format(len(X_test)))\n",
        "\n",
        "plot_value_counts(df_data, title=\"Value counts labels full dataset\")\n",
        "#plot_value_counts(df_train, title=\"Value counts labels train set\")\n",
        "#plot_value_counts(df_dev, title=\"Value counts labels dev set\")\n",
        "#plot_value_counts(df_test, title=\"Value counts labels test set\")"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The entire dataset was split using 0.75 for training, 0.15 for validation and 0.1 for testing.\n",
            "\n",
            "Number of sentences train set: 18,123\n",
            "Number of sentences dev set:   3,624\n",
            "Number of sentences test set:  2,417\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAE3CAYAAACXcVduAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd7gU1fnHP4A0BRuiYi9JvmpijcaOqFHsBbv+UDQYjR2xl9gSa+wtUbGgiWKJ2IKi2HsL2F8rsaEiVlREyu+P9wx3WPbC7mWHuxffz/PcZ/fOnD1zZubMees502ry5MkEQRAEQa1p3dwNCIIgCGZPQsAEQRAEhRACJgiCICiEEDBBEARBIYSACYIgCAohBEwQBEFQCCFgWhiSlpI0WdLJzd2WANK9uHYmfj9S0sO1a9GUemveTyS1lnSypHclTZBU9RwHST1Su/rUuq1FXcug6czR3A2YHZF0C7AjsKqZDW+kTCvgXWA+oJuZ/TALm/izQtJSQB9gcGP3I6iIvYCTgKuBR4GJzduc2iLpMOArM7u2uduSpyX337BgimFA+tx7OmU2BJYCbgrhUjhL4QPjKs3cjpbOJsDXQF8zu87MbmjuBtWYw/CBvN5Yihbaf0PAFMNQ4ANgD0ntGimTCZ8BjewPgnpjYVzDj+U/gooIF1kBmNmk5Jc/EdgGuDW/X9LcwA7AK2b2nKTOwNG4hrgs0BkXULcCp5rZ99M7nqQewEPA3qXmfWrHXmbWqmT7L4E/A78HugAfA7cAJ5vZd5Wcp6RVgeOA7sC8wGfA48AJZvZOrlxf4ABgeWA88Ew6r8dzZZYC3gNOMbOTS45zMq7BLW1mI/PnlY57Jn495wZeAA43s2dSuT7ANamqayRl3x8xsx6SWgOHAPsASwOTgVHpPPY3s58quRYl7d0F2APXOBcCvk31/dnMXmrkN6sBfwN+h1+ju4AjzeyzknLtgf6p/mWBccBjqe7/VtC2PYGDgF8BbYFPgaeAw8xsdCO/6YH3r+z/TMBcZ2Z9JI0ERppZj0Z+N02/nBkkLQ6cC/QEWgGP4NZHubIV3YvcOS1ZElta2sxGStoU+AOwBtAN+BF4FvirmT1ScsxfAycD6wALAF8CrwN/M7N7cuVmeC9n1H9ncKmanbBgiuMafLAq5ybbFehIg/WyKNAXeB44DTgceBE4Cri91g2T9Nt0rO7AP4ADgbvxgfZ+SW0rqGMr4GlgY+A64GDgSmAJ4De5cmel7T/hwuhcYAXgIUlb1OB07gMWA04FzkjHvicJbfBYwenp+xVA7/T317TteOB8YCQu5I/Er/naQPsmtukgYFI63oH4+a8PPJEEeymLAcPwmNxRwL9TGx+SNGdWKN2Xe3Fh+xTQDxeuK6S6V59eoyT1xu/VOFy5OAy4ARCw4HR++npqzxvA5zRcw39M73hFIGle/J72Aq4HjgG+xwXZXGV+Uum96I2f2xs0nF9vIBO6fYD5gYF4Xz8fV5iGSVo/174uwIP4s3UV8CfgvFTPmrlyld7LGfXfuiYsmIIws/ckPQT0lNTNzEbldu+Na6mZD/tdYPESbflSSacBJ0j6nZk9W8PmXY1r6WuY2bfZRknD8MFtD+Daxn6cBr1rcH/8qmb2UW73qckqQJLwAfsJYCMzG5+2XwW8BlwmaVkzm5lg8YtmdkCuba8BNwO7A/8ws3cl3Y8Lt6fKxA22B143s21Kth8zE23arNQKlDQQGI4PJAeUlF8W6GdmF+TKv4oPTIfgAw/4YNkj1X9fruxlwCu4BdRjOu3aHtfgNzKzCbntf57eyZjZp8ANyRLt2Myxl6PwmMQ+ZpZp85dJugA4tEz5iu6Fmd0g6S/Ap42c375l6vk78CpwLG55AKyLC+tdzOzm6ZxHRfeygv5b14QFUywDgDbAntkGScsBawF3mtnnAGY2PhMukuaQNJ+kBYAH0s/WpEZIWhFYCfgX0F7SAtkf7jr4Dth0BtX0xE3/c0uEC+AuwvR1W9yFcXYmXNL+j3EBtSSw6kye0vkl/z+YPstZCuX4GlhU0noz2Y4pZAORpFaS5k7XdjRglL+X3wCXlWy7LG3fPrft/3AN+4WS+9YOuB9YT1LH6TTta2BOYMuUxdgS2Q536w0s2X5WucJNuBdlyQsXSZ2SpTIRd/fm6/k6fW6eXOGNMbP3skUQFkyx/Bv4CrdYsgdgn/R5db6gpAOA/YFfM63gn6+GbVo+fZ6S/sqx0AzqyAbvGfn8l06fr5bZl21bBnfXNZV38/+Y2Rg3nOhS4e+PAwYDj0n6GHgYuAe4NS8UqyHFpk7DNdRSt817ZX7ybumxzOxHSe/i1ydjedy1WjZWklgAj9+V43TcdTMYGCPpEWAIMChvydY5ywDPlVq9ZjZK0lelhZtwL8oiaVncLdUTj/vlmRKzMbNHkoXUB0/yeQ5XFAeZ2Wu538zsvWwRhIApEDMbJ+lfwAGS1sG1nd7Ah3jsAABJh+OxiaHARXjAfTwem7mWGVua08vqKb3HmeZ6Lu4DLseXMzheEVRzDlOYjnutIg3dzJ5Kg0dPPHV8Q9y9doKk9czsi0rqyZC0BO43/wYf2Ay3CicDFwCdqqmvhFbAy3iMrjEaHbDM7C1JK+Bxs42BDfCYxCmSuucTM6qksXvXrONLre6FpE6pnrnS717GXY2TcPfYRvnyZraXpHOAzfF4T3/geEmHmdklqdhM3cuWQgiY4hmA+3n3xoOEC+OZJ5NyZXrjQebN89slbVbhMbJBcP4y+5Yp+f+t9DnRzB4oLVwhb6bPVXCh2BiZdfFroHTwWqGkTDXnUC3TTas1s7HAbekvsyYvxbOGzqnyWNvjA9c2ZvZQfkdyq/xY5jfLSGqXt2JShtEyuBsl4y2gK/BgSf+pGDP7EfhP+iMlWtyDD3QHNqVO/N4Vcd/K8S7wS0lt8sqFpG5Ma1lUey8a6ycbA4swddwnq+cv5X5gZq/gsZRzUmLCM8CZki5Nad7V3MsWmxYeMZiCMbMX8YDiLvgDPJkS9xjuy51MTuuWNAeVB5rfAybgKcdTSFbTWiVl/4t3/P0lTTMApBhQucEiz1A846Z/erBL68jO4078vI7MZ6al3+wN/C+1h+Si+QTYKB8fSG3cbgbtmRFj0+c055X83qW82Fj5CsgGvdK08H1x5aIcczNt4P+AtH1wbtvAVEdZrVfSdF2bBZxrxpvAcpIWzR2rPU0XWNPjDtyFu2fJ9qPLlK32Xoyl/HVorJ5NKYnjSJo/S3LJMLOv8Gd0TqBD2lzNvWy0/9Y7YcHMGgYAFwObAQ+b2bsl+2/FU2yHSPo3PrDsjqf2zhAzGyufF9JX0o14HOGX+CD+ErByruzklK76IPCSpKvxeMicwC/w9M9jmU4WmZl9L+kPqd2vpKywt3GNrCee/XSHmVlyFRwFPCppED7H54+4ZrlHiYvrEuAv6ToMxrXG/XGBuEYl16IRXsNdGgdI+h6Pi31mZg8Cr0t6GtcwP8bnOPwRd1He1IRjDcHTZq+XdAnublwX2AK34so9c+8AJ0n6DT6P57d4rO4N3GWacSE+V+ocSRvh9/AbPDV8Yzz9eMPptG1oilM8hvv258VjBZPxlN+mcgmeev9Ayqxqh1vl052/1UTOxp+NK+Xp9q/i8ZW1caUnT7X34mngDyl783XcBXYXnvzyCXCufL7Wh7j13ht3c62Yq2NPoJ+k2/Fn4ifcFdkTuDm3akc193J6/beuCQtm1vBPvMPAtNYLuBvmONylcCGu+Q1lWi1tevTDBdmmeGbVmsDWuPU0FWk9o1XxNOltcOF3Am7tXIvPyZguZnYnsB4+WP0BH2T2xx++l3PljsYH7A54uu2R+MC5kZn9p6Tas/BrsTLu694q1V1arirSQ70r/gBfANxIQ2ruucA8eDrw5ekcngXWNrMRTTjWO7jv/T38np6Ja54b4NemHB/ig8oyeHrqDnif6ZHPXkqZhlvi6bhd8SSN83Hr+F1cSZkel+Pa+H54llp/4H1g4zL3omLM7AlcULXG798BeN86rql1TudYX+JxjcH483EWrhxtiMdX8mWrvRfH43OgDkztvxHomiyQnrgScjANc7m2oMECzHg4/W1FQ3/+NXAEuee5mns5g/5b17SaPLnFuveCIAiCOiYsmCAIgqAQQsAEQRAEhRACJgiCICiEn1MWWXs8E2kUs9mLkoIgCAqkDZ5d+Rzl53E1ys9JwKxBw4J0QRAEQXWsj6dsV8zPScCMAvjyy++YNKl2mXNdunRizJixMy7YjLSENkK0s9ZEO2vLz7WdrVu3Yr755oI0hlbDz0nATASYNGlyTQVMVme90xLaCNHOWhPtrC0/83ZWHVqIIH8QBEFQCCFggiAIgkIIARMEQRAUQgiYIAiCoBBCwARBEASFEAImCIIgKIQQMEEQBEEhhIAJgiAICuHnNNGyYjrP3ZEO7Su/NF27dp5hmXE/TuDbb36YYbkgCILZhRAwZejQfg627n9HTeu869xt+bamNQZBENQ34SILgiAICiEETBAEQVAITXKRSToKOAsYYWarlOxbBzgbWA34BhgEHGtm35eUaw+cCvQG5gNGAMeb2bAyx6uoziAIgqB+qNqCkbQwcALwXZl9qwDDgA7A4cBVwH64QCjlWqAfcANwKDAJGCJp7ZmoMwiCIKgTmmLBnAk8jwuneUv2nQ6MAXqY2VgASSOBKyVtZGYPpm2/A3YF+pnZBWnbQOAV3DLqXm2dQRAEQX1RlQWTBMP/4ZZE6b65gU2AgZkgSAwExgI757btCPyEWyMAmNk4YACwnqRuTagzCIIgqCMqFjCSWgEXA9eZ2fAyRVbELaLn8xvNbDwwHFg1t3lV4I0SoQHwLNAKyOI61dQZBEEQ1BHVuMj2BFYAtmtkf7f0We61mqOAtUvKftRIOYBFmlBnRXTp0qnan9SMSiZkzo7HroZoZ22JdtaWaGd1VCRgJHXGYy9nmllj72XumD5/LLNvXG5/Vraxcvm6qqmzIsaMGTvD14kWdXNGj26eqZZdu3ZutmNXQ7SztkQ7a8vPtZ2tW7dqsmJeqYvsBGA8cN50ymTroLQvs69Dbn9WtrFy+bqqqTMIgiCoI2ZowaSA+2HAicBCkrJdHYB2kpYCvqbBjdWttI607ePc/6OmU45c2WrqDIIgCOqISiyYhYB2ePrwe7m/NYHl0/ej8RTjCcDq+R9LaocH7fOJAcOB5SSV2l1rps8R6bOaOoMgCII6ohIB8x6wfZm/V4GR6ftAM/saeADoXSI4egOdgFty224F2gJ9sw1pZv/ewBNm9jFAlXUGQRAEdcQMXWRpkB9cul3SYcAEM8vvOx54EnhY0lXAYkB/YIiZPZCr8xlJtwBnJxfcO8BewJJAn5JDVVRnEARBUF/UdLFLM3sR+D2e9XU+sC9wJbBTmeJ7Ahemz4twi2YLM3tiJuoMgiAI6oQmvw/GzHo0sv1xYN0Kfj8OODL9zahsRXUGQRAE9UMs1x8EQRAUQgiYIAiCoBBCwARBEASFEAImCIIgKIQQMEEQBEEhhIAJgiAICiEETBAEQVAITZ4HEzQ/nefuSIf2ld3CSl9BMO7HCXz7TSxSHQTBzBMCpgXTof0cbN3/jprWede521L/b7wIgqAlEC6yIAiCoBBCwARBEASFEAImCIIgKIQQMEEQBEEhhIAJgiAICiEETBAEQVAIIWCCIAiCQggBEwRBEBRCCJggCIKgEELABEEQBIUQAiYIgiAohBAwQRAEQSGEgAmCIAgKIQRMEARBUAghYIIgCIJCCAETBEEQFEIImCAIgqAQQsAEQRAEhRACJgiCICiEEDBBEARBIYSACYIgCAohBEwQBEFQCCFggiAIgkIIARMEQRAUwhzN3YAgCKqj89wd6dC+ske3a9fOMywz7scJfPvNDzPbrCCYhhAwQdDC6NB+Drbuf0fN6rvr3G35tma1BUEDMxQwklYHjgdWAxYEvgaGA6ea2ZMlZdcBzk5lvwEGAcea2fcl5doDpwK9gfmAEcDxZjaszPErqjMIgiCoLyqJwSyLC6IrgYOAc3BB86ikTbJCklYBhgEdgMOBq4D9cIFQyrVAP+AG4FBgEjBE0tr5QlXWGQRBENQRM7RgzGwQJQO6pMuBd3HhcH/afDowBuhhZmNTuZHAlZI2MrMH07bfAbsC/czsgrRtIPAKcBbQPXeoiuoMgiAI6o8mZZEl99RoYF4ASXMDmwADM0GQGAiMBXbObdsR+Am3RrL6xgEDgPUkdWtCnUEQBEGdUXGQX1JnoD3QBdgL+A0eRwFYMdX1fP43ZjZe0nBg1dzmVYE3SoQGwLNAK2AVYFSVdQZBEAR1RjVZZNcAO6Tv44G/4y4sgG7pc1SZ340C8rGVbsBHjZQDWKQJdVZMly6dmvKzmlBJymg90JztjGvUPDT3+TT38Ssl2lkd1QiYU4B/AIvh2V/tgbbAj0DHVObHMr8bl9tP+t5YOXJlq6mzYsaMGcukSZOnW6aomzN6dG2TQVtKOyula9fOzXbsamjudhZx35v7fOK+145at7N161ZNVswrFjBm9jLwMoCkG3DX1bV4TCWbpdW+zE875PaTvjdWjlzZauoMgiAI6oymBvl/Au4AeknqSIMbq1uZ4t2Aj3P/j5pOOXJlq6kzCIIgqDNmZi2yjnhQvjOeYjwBWD1fQFI7PGg/PLd5OLCcpFKba830OSJ9VlNnEARBUGfMUMBI6lpm29zATsAHZvaZmX0NPAD0LhEcvYFOwC25bbfisZu+ufraA3sDT5jZxwBV1hkEQRDUGZXEYAZJGgc8CXwCLI4Lg8XwCZMZx6cyD0u6Ku3vDwwxsweyQmb2jKRbgLPTnJd38LTnJYE+JceuqM4gCILZmWoWOIX6WeS0khbfAOwJHIKvG/YV8DTQ28weyQqZ2YuSfo/Pxj8fXzfsSuDYMnXuCZyWPucDXgK2MLMn8oWqrDMIgmC2pNYLnMKsWeS0kqVirgaurqQyM3scWLeCcuOAI9NfTeoMgiAI6ot44VgQBEFQCCFggiAIgkIIARMEQRAUQgiYIAiCoBBCwARBEASFEAImCIIgKIQQMEEQBEEhhIAJgiAICiEETBAEQVAI1bxwLAiCoCJaytpZLaWdLZUQMEEQ1JyWsnZWS2lnSyVcZEEQBEEhhAUTBIlq3CWVuEog3CXBz5sQMEGQCHdJENSWcJEFQRAEhRACJgiCICiEEDBBEARBIYSACYIgCAohBEwQBEFQCJFFFhROpP8Gwc+TEDBB4UT6bxD8PAkXWRAEQVAIIWCCIAiCQggBEwRBEBRCCJggCIKgEELABEEQBIUQAiYIgiAohBAwQRAEQSGEgAmCIAgKIQRMEARBUAghYIIgCIJCCAETBEEQFEIImCAIgqAQQsAEQRAEhRACJgiCICiEEDBBEARBIczwfTCS1gD6ABsCSwJjgCeBE8zs7ZKy6wBnA6sB3wCDgGPN7PuScu2BU4HewHzACOB4MxtW5vgV1RkEQRDUF5VYMEcDvYAHgEOBK4AewH8lLZ8VkrQKMAzoABwOXAXshwuEUq4F+gE3pDonAUMkrZ0vVGWdQRAEQR1RyRstzwN2N7Px2QZJg4CXceHTJ20+HbduepjZ2FRuJHClpI3M7MG07XfArkA/M7sgbRsIvAKcBXTPHbuiOoMgCIL6Y4YWjJk9mRcuadtbwKvA8gCS5gY2AQZmgiAxEBgL7JzbtiPwE26NZPWNAwYA60nq1oQ6gyAIgjqjSUF+Sa2AhYDP06YVcWvo+Xy5JJiGA6vmNq8KvFEiNACeBVoBqzShziAIgqDOqMRFVo49gEWB49P/3dLnqDJlRwH52Eo34KNGygEs0oQ6K6ZLl05N+VlN6Nq1c7MduxqinbWlJbSzJbQRop21puh2Vi1gJC0HXAo8DlyfNndMnz+W+cm43P6sbGPl8nVVU2fFjBkzlkmTJk+3TFEXffTob2taX7Qz2lkrWkIbIdpZayppZ+vWrZqsmFflIpO0MHAP8CWwk5lNSrt+SJ/ty/ysQ25/Vraxcvm6qqkzCIIgqDMqtmAkzQMMAeYB1jWzT3K7MzdWt2l+6Ns+LinbWDlyZaupMwiCIKgzKrJgJHUA7gJ+BWxlZlZS5BVgArB6ye/a4UH74bnNw4HlJJXaXGumzxFNqDMIgiCoM2YoYCS1wSc2ro27xZ4uLWNmX+MTMXuXCI7eQCfglty2W4G2QN/cMdoDewNPmNnHTagzCIIgqDMqcZGdC2yDWzDzS/q/3L6xZjY4fT8eX0LmYUlXAYsB/YEhZvZA9gMze0bSLcDZac7LO8Be+DI0fUqOXVGdQRAEQf1RiYssm5eyNZ41lv+7ICtkZi8Cv8ezvs4H9gWuBHYqU+eewIXp8yLcotnCzJ7IF6qyziAIgqCOmKEFY2Y9Kq3MzB4H1q2g3DjgyPRXkzqDIAiC+iKW6w+CIAgKIQRMEARBUAghYIIgCIJCCAETBEEQFEIImCAIgqAQQsAEQRAEhRACJgiCICiEEDBBEARBIYSACYIgCAohBEwQBEFQCCFggiAIgkIIARMEQRAUQgiYIAiCoBBCwARBEASFEAImCIIgKIQQMEEQBEEhhIAJgiAICiEETBAEQVAIIWCCIAiCQggBEwRBEBRCCJggCIKgEELABEEQBIUQAiYIgiAohBAwQRAEQSGEgAmCIAgKIQRMEARBUAghYIIgCIJCCAETBEEQFEIImCAIgqAQQsAEQRAEhRACJgiCICiEEDBBEARBIYSACYIgCAohBEwQBEFQCCFggiAIgkKYo5JCkroBhwJrAqsDnYANzezhMmW3AU4GVgA+AwYAfzWzCSXl5gXOBrYH5gSeAQ43s+FNrTMIgiCoHyq1YAQcDSwGvNRoIWlzYDDwBXBw+v5n4PyScq2Be4BdgYuBo4CFgIclLduUOoMgCIL6oiILBngBWMDMxkjaDri9kXJ/A/4L9DSziQCSvgGOlXSRmb2Vyu0IrANsb2aDU7mbgTeBk4A9m1BnEARBUEdUZMGY2bdmNmZ6ZSStgLuw/pEJgsRl6Tg75LbtCHwM3JE7xmjgZmA7SW2bUGcQBEFQR9QyyL9q+nw+v9HMPgY+zO3Pyr5gZpNL6ngW6Az8ogl1BkEQBHVEpS6ySuiWPkeV2TcKWKSk7IONlCOVfb3KOiuiS5dO1f6kZnTt2rnZjl0N0c7a0hLa2RLaCNHOWlN0O2spYDqmzx/L7BuHZ4rlyzZWLl9XNXVWxJgxY5k0qdRwmpqiLvro0d/WtL5oZ7SzVrSENkK0s9ZU0s7WrVs1WTGvpYvsh/TZvsy+Drn9WdnGyuXrqqbOIAiCoI6opYDJ3Fjdyuzrhgf182UbK0eubDV1BkEQBHVELQVMNkFy9fxGSYvg82eGl5T9raRWJXWsCYwF3m5CnUEQBEEdUTMBY2avAm8Af5TUJrfrT8Ak4LbctlvxAP222QZJCwA7AXeY2U9NqDMIgiCoIyoO8ks6IX1dPn32lrQe8JWZXZK2HQncCdwnaRDwG+AgfB7Lm7nqbgWeBgZK+hvwOXAALvBOLjl0pXUGQRAEdUQ1Fsxp6W/39P8+6f8jsgJmdjfQC+iCLwHTC/gLcEi+ojRpcgt8YuUhwDnAaHx9s7dLylZUZxAEQVBfVGzBmFlpvKSxcoPx9cJmVO5LoG/6q0mdQRAEQf0Qy/UHQRAEhRACJgiCICiEEDBBEARBIYSACYIgCAohBEwQBEFQCCFggiAIgkIIARMEQRAUQgiYIAiCoBBCwARBEASFEAImCIIgKIQQMEEQBEEhhIAJgiAICiEETBAEQVAIIWCCIAiCQggBEwRBEBRCCJggCIKgEELABEEQBIUQAiYIgiAohBAwQRAEQSGEgAmCIAgKIQRMEARBUAghYIIgCIJCCAETBEEQFEIImCAIgqAQQsAEQRAEhRACJgiCICiEEDBBEARBIYSACYIgCAohBEwQBEFQCCFggiAIgkIIARMEQRAUQgiYIAiCoBBCwARBEASFEAImCIIgKIQQMEEQBEEhhIAJgiAICmGO5m5AJUhqD5wK9AbmA0YAx5vZsGZtWBAEQdAoLcWCuRboB9wAHApMAoZIWrs5GxUEQRA0Tt1bMJJ+B+wK9DOzC9K2gcArwFlA92ZsXhAEQdAILcGC2RH4Cbgq22Bm44ABwHqSujVXw4IgCILGqXsLBlgVeMPMxpZsfxZoBawCjKqgnjYArVu3quigC87XsYomVkalx66GaGdt+bm2syW0EaKdtaaSdubKtKm2/laTJ0+u9jezFEmvAB+ZWc+S7SsArwJ9zWxABVWtBzxWQBODIAh+DqwPPF7ND1qCBdMR+LHM9nG5/ZXwHH6BRgETa9CuIAiCnwNtgG74GFoVLUHA/AC0L7O9Q25/JfxIldI3CIIgAOCdpvyoJQT5R+HSs5Rs28ezsC1BEARBhbQEATMcWE5Sp5Lta6bPEbO4PUEQBEEFtAQBcyvQFuibbUgz+/cGnjCzsGCCIAjqkLrPIgOQdDOwHXA+7gvcC1gD2NDMnmjOtgVBEATlaQlBfoA9gdPS53zAS8AWIVyCIAjqlxZhwQRBEAQtj5YQgwmCIAhaICFggiAIgkIIAVNnSFqkudsQzN5I6iZpy+ZuR1Adkmq/wNmMj7m5pPWaeuwQMHWCpFaSbgcGSqr9qnbBNEjaStIlaV27nwWS5gReBHZL/1e9gOHsQJrqUPdI6iqpP4CZzbKAuaTekl4D7sHnHDYpISwETB0gqTW+ptqawKtm9kNzaCuVImleSRtIWlLSXGlbi+lLkv5P0uvAncDiwPcFH69j7ntz39dFgXmA5wHM7Ge1Lp+kVSSdAVwqaae0re76rqRfSLoRX8nkUElLzaLj7ivpf8Bl+CT3rYDrzOynptTXUtKUZ0uS9jjJzCZJ6gLMC7wGs1ZbqRRJvwJOAjbBF8D7CXhM0p5mVumacM2GpD8BhwPLADem7/81s08KOt5u+MvyJiWBdr6ZjS7iWI0cv6uZjU4D6OTUp9rg6/i9msq0qse+VmskrQGcAawLvAVMAMYCt5jZpOZsW570gsXT8GfsJeAwYBjwYYHH7AgcDxwMdE6bNwceBcbNTP8IATOLSJprK+A4QPgbOj/P7Ss8kHQAACAASURBVOuain6ZbauXB1/SL4G/AVvjS/Och69mvRWwAzASOLKe2pwhqQMND09HfFWI/wD7mdl3BR3zMOBoYG7gZWABYFtgBUkHFLX6RHb9JS0IPAh0krQP8LiZjU/FVsHv3XioT0Wm1kgScDXwGbAvrsS9YWaFWq7VIGkB4FygN3A/3s4HgQ/MbEJBx2wFnAMcAnwOXIgvCnwisKaZ3ZeU4CZbuXVnGs6upAd5MrAksAvwhKQ9JM2R9nXDNcvX6nCg/iMuXC7DX3lwSXp99d64q2V/SXPVWZuRtAnwCd7+S3Gt7HJgY2D+VKZmLitJi0l6HBfAt6bjbQOsBfwdv4bbFBX3yK6/mX2Gv078Y9wNeIGkRVOx1vjq5C8V0YZ6I1lvJ+LP1/HAjWb2opl9L6lNHbgsMyYCC+Lu2qPNbICZvZcXLpLa1eJAkrpIWjr1ly+AvwCbm9mfgSuBB4B+MPMu1LBgCkTSL3DN9VM8tvJfYF9JA3EN93pgJ0kH4A/A18CE5h6oJW0HzGNm16VNNwObAqtmbxaV1NrMPpD0ArA60AMPCDYr6RXavwHexN9fcRTwNPB2GlQm4QHufsDhNb7WP+AW3m+Aq8xsykKskq7FLb6Nzezv6frNtGtGUmdgP2BpXEN/BbjbzK6XlGnCJwG/l/QHoDuuwc9FspZnF9IAvC7wDf68jcPdzr8GHjazp/MCpTnjTymLb3HgLjP7yMy+TOPCurhbdXiu7JL4eLGUpD5JgWjKMdfAXW7LAV9LOtPMTpfULrNwzewzSXcAW0g62MwuTkpwk6yomMlfAJLWBE4HNgC+wrXlr4BjzOyKXLmj8TjAe7iL7Ct8oP6uufzCkroCt+AP5QpZzEDS3/DOuYmZPSSpLa513Yo/FOuYWZPeGVELJP0W11RXTZv+C/Qxs69Kyi2AuwW2A5Yzs0+beLx2Zja+1NqUtCFwBx5vOSl7OCUthwu8t8xstaYcs+T4c+HCsz8wJv0tg7vlXgD2MbOXU9ktgGNw12xX4CkzW3dm21AvpPjlcfg9nROPVxxkZs+kbLEngUnADmb2vqR58WSHicASgJnZ/2ZBO9vh9yxbuHdx3NI8IcVhFwSuAdYwswVTzPMU3A39v1T2xmpdu0lA/Q23qEfgCsZoYCB+7pNTuTZmNlHS0sB1wNJmtvjMnHO4yGqIpCXkr3h+Km3aF9ge2An4FjgpBfEAMLOz8Js+EtdA5wV+Sp1tlpjuktpJ+pWkeVKbRgOXAJ3wBzbjLvzB7Z/+nxMPmm4HDAHenxXtLUVSe0l/Be7D16n7By7cj8GveVauFUCKe/0baIffn2qP90tJtwKnS5ozxTxa5e7XS8Bg4I+SFs9pfhvgQeUrsvbM5D3eG9gfuBjvY5vhg+VfcQ31Gkk9AMzsP8Dv8UHmG2DOzGVWRy6iqsjaLakXLlC3Am7HB+QjgBFJuP8IDAJWAu5KmVn3Anfjg+29wKuS+kmaL193jdu7FK549MX7x7G4u/nETJlMlskdQEdJ7+CC4Bd4P+0OXN3EuOEBuOJ1JL6e44FmdpyZvZFXjjKLzszeA24DFk2JKkhqkrcrXGQ1IKfFfgasADwDHGpmr+TKdMH98JsBz2bbzexFYFdJ3+ADxda4BdEKj9kU1eauwJ/xbJVJwDKSdjOz21P7ngQOlHRLsgKewAXJHyRdn9r5PXCYmV1UVDsrYB1gH9zdeCkwspw5nwmCdJ+ew8/lT5IuNrOvqzjen4Be+L3uiD+sk3MCbIykQcDOwA7yuQQnAOul3+8q6W0ze6Cak8y7KSStCpyMZ/mcklxBWblTgTfw63G4pBFm9iWuuJyTLM9TcYH3r+Z2xzaVdM3nx11HhvflVzMXLkwlKC7F+2p/YEN83HscGIAnYGyMZ25NAi6s5TXJrAI8Dvg7PNnkLjP7tqRc1jeH4gpqd7wPDQO+rcajIWl+M/sifV8bOBC4wswuz5VpDbQpTT/OuW6H4m66E3GrqUkusrBgZgI1zLrPBpdxeIB3JWCpVKZtKvNk+myvXN59LuB7XvrcSVLbIlxkklpLaitpfzy7qSeuTV2Fa35zpaIf4ubzSqkMqYPdDbyLW2THAD0y4VJyTh2SG6Lm2qCknjlNsyPQBxfER5vZ27lBuG3p8XNB8E9wIb4g8H8VHLNV7j49j2faHIEnN/xZ0txpwMsUthfw4Pp5uOX3FZ4d1Be3nIZK+mvWfzSdoL+kBSU9Qgq6JtbA3a6X5IVLOrefzOyfwE24Vr992pUd4xw8g2zH5C5sMVaMpC0kbZK7Xrvh8b+LzOyZXHxwDphKqfjOzC7BY2NrA0uZ2XZm9hczOwx3QY0HtpS0cA3auZKkvVMbsjjP7nhs7F954aKSOThmNhK3xlsDC+SVn9KyZY67iKQ3gItyZTvi3oapVp43s0l54ZJTkDJr6nXc/b2cpM1SmaqTU0LAVImkjpJOlPQocLukl/Ega8ZZ+E3dUdK8uZu4PR7EfzHvAks+z1Zm9gbesdbFrYqaTwBLnWd13A/8OP6AnmJm55nZGWZ2Q67c47imf2Dy9wM8hFs3k4EHzOzNkrqRtBg+uP41ba+lNrg/bnmsm+r+AXctjib3Wu2k7f9U7vi5wfQpPA30EJWZ1S1pH0mbZ3XkBoofcNfbQ7iQ7UvDuU5In5/gLobvcBfWrrgWeDWwI+7COxZ3Y3Wy6Qeb2+PP6V6S5kvt/1Xa90bJOeW/n5Y+t5LUIcWB2qbrcjnuml2/3DWqN+Szyl/HFZzf4tcE3N0HJa9NzykZc5a4gL4zz8waK88gy56v93B3WTdgTA0E7nXAWZJWS+1YgoYYWf685svFXjrkdv0Htx4OS+2elP9MY9BC6Xu+rd/jVm3P3LFexZ+P3SQtK2l+SVtL2kvSyZIukLRBSbuy63IvPmfoz+n/qpXeEDAVkrTyo/CZtX8E3sYH4IHARElzJHN4NPBPXHtcQdLcks7DTfk5gQUzjTfV24qG+3BxKrM9NHSoGrS9dfpsg/vhJ+KurRfSID1FO8l1rv/hbpb1aBiIvseFxw+kzq9pfbNfABvhA9tStWh/jvtxC2rnTPvGNfUVgcMkrSipL3CKpIGSXpZ0laStSisys/dx3/wvcQ0WSXNKOkPSd8DZQOfctcse5Mn4BNNFcNfLmbgQPkopjpV4ChfGGwOtcvdylJmdgAultYAn5Vl7jfEhnh6+PLBt6jfZNV+1tHBOa38deAxP1lgi7c7acDpu0Wwtae6S86sbJB0iaRQet8riLAOsYf7KG/j8jTOSZdMrWZRXSHoKuE0+M71zqi/r43OY2cTcPVkXV7w+pWFCalPam3krzsbnW+2Q/v8QeBhYV9LZkvaTdCFwldyF+iZwvaStYYr1cA/wK0k7prpbSeokT2Y5CXhU7iKcQnJl34hbyZkF9Sn+zPfClwj6EI/zXIMLjkNwZenEXFXZ+Q/HrZi1JP0u9a2qrJjIIquQpD3/GQ8QXw28U+q7V0MWxi9xv/AbwLJ4APx+/GFfGn8r57GW0iZt6iyku/BMsu3MbNhMtHdjPD7QK/3fCtdqngeuMbPDcxptY3WsgLuS3gV2NLMf5e6pAbgQWcbMvihzDmul69OkWevyVOOvzdOKp9SdOvcJeLByCzN7NG0fhvvW87yPC+7FcIG4LTAsPSStk+b4i3Qu8+DKQh98kLkWd229bCWT8VLbPsLTjR9K287BLZPBZtYv19beeF/Z1cxuzranPjIHPkfmRKC/mT04neuxFC7ss+WEtsb74eHAxeUsoCQYz8Gt6xWSQM0f/1Z80OluZo83duzmQB43eBhXhK7FlYhXsrhCrlwHPAB+YUkV2coMndLf38zsqPSbbDJqKzyTbHU8NrMQ8H9m9iwVIqknbs2+Zb5iQhZvQdKLqdgeZva6pJVwz8HuuSo+pcFyWhG3NP5gZnfLU4qvTfWvgwvBndPvO+MK7lpm9k2uvizWez7uBVnDzD5M2w/Cx5+F8GfjHlxR6oLPg1kC2MjMnknls2dkPdzSf9zMNqr02mSEBVOGUm1Onvl1Bj6f4njziVpfp32Z1pJ3d72Fu0iWw2/2GrimsCH+QKwE3CGfCJgdI5tEdRk+uE1xP1XZ9p3S4NYd2E7Szqltk3Gtex5SdtX0hEviHXxg2xx/EDEPGN+R9h+WPqfSaszs6ZkQLn1wgbZH2lQ6b+Fm3BWwWxJ24IJhG3zA7Y/PVl8ntbkvfr798IBu3jJ8F3djrQxsiQuuLYG/Jp9+uZneHfCsv7Vy287AXRGHSjpAHmSdiFsxzwEHKK1HZg2ZOhPM7N9mtur0hEviA/w+rAZsgSsv76W2LpGu21RusnSOc+JCaZ7SCvH+uH29CBdNPYlwAr7awGDgVDN7tFS4gMc8zexi/Drsjwvr3+P3fw08qP4a0E+eeguwiaTLgH/hkwr/had2H1yJcJG7p06T9CneF4cCL0has0TQX4Vbx9ultr6E99PVcJfphnicdkczWxkX9l3xJBLM7DncW7AyLmzvxgXMGXg8Zb60rzTOOAa3zOdNx8m2X2JmfwJ2MbPDzOx+M3vYzG6jwa2/aaovP0frOfwV9fkwQMWEgCkhDfpZCmfW6bvjft9TcoIli6H8lP7fVNKOOU3+zPQ52cy+Sn7h1mZ2H37j38XNz1NTPdlEpyFmtruZfVBFm+eUdHpy7ZyLpzYOwlNmj8gV/RAfnNuXcW3l68vO7Uc8LvQBPkhmwnQoPqCeIJ8PMtNLWeQekidwgbCJPHZQmrL9Du6C3Ak/T8zsAzO728wuMLPz08P8qXna5+24X3p93G2BfKHOv+MP+I24S+A63BoYYSXrqmnqGE2WJjpe0jzJJfcwPmC8h8dljk5l3sNdqN1JcbWmkAauh/H7+SfcLXQrbulumTTnKUkGuT64GO5DfyV3TzMB97GZ3UEdIHch/1seqwDvW9fj13TBkrLbSHpbKYgOU56ZK8zsr2b2oJl9Zj558Q3cgmyDK0ng/X+1VO/X+NyYldNzOb02dpR0Lp6wsQ9+Xw/D3XfzA+dLUu4nA/DY0FbyuGTW1uFmdrOZPZKer+/S9sG44rCkGpKHHsBjNwvhCtISZpbNR1so1d3Gpna3g7sT7wP2U4MrObNef5LHn/KrGGTZrgumtkxxzZvZj2Z2Y1KaqyYETI7k2rkP960DTEgP7Rp4lsl7WdncTe0j6Rk8IHZzbv8LuAbbK/lNye27H3dzvAIcJ2mVJrZ3QfkEyC/xbKhLcFfNSDN7DQ+I/zaZ8uBaypu4ptK2TJVTnVviTXwQ3hqPA2Bmo4CLgJ0sTTZsYvs3lLSbPEbSOdX9Fm7BdcdTumFqK2Y8Pvi0w9OAO6a6psSZlHzsqfyXuBugFbCtfOWBh3DXZUczux53N2xILggraS5Jx0p6lWStJMvgM3yQ2R8f8C/FlYVe+CA2BF+XbR/8+boXF4rZ4NlURqbz7onfh6tTvSfgGvxUwW35wp6b4yvh1qUfPNdvxuP3eg2YEuu7Hre8tkzns6c80H8TLjRfL1dX3qOQyDK2vkh1P4735Z3NbFczG1JB+8CTC/bGhcDmwMlmdo2ZHY5bAGvQ0E+y+TfX4plrWYxtioUjj9nm3b+L4f30c2C0pINxZeV13B06MNWJ+QoRj+HPepZI0NqmzpL8J+6O3zY7Zs7DMjH9TZZPwjwCT5i4pcx5zxQhYJgqsP0q/uBuIWlB81S+CXjHmIibvPnf/Q73/76JWyyTJR2SK3JK+k0+1TfrDJ/jJu+cZjacKpC0nNyH/gnuFnocdxGdYGbPZh0RH6hH4r5f8IH0SdyFMF2NWtJ2khYwn9h1D+7L7pddKzMblMzrqrOQJB0sKdPuz8BdAfvnilyMC5AdVX5Jldfwh2EPklCwhkybibnrvKB84cm1cbfLRbg11gt3V2Ra2Zm4O2U9+asIzsTjLIfg/uc3s/OUv0/lZdzdeC/usvoDcL95Vt1JuHv0KuAA89UNVjJPk20ySbAOxZWcg83McE12bmCQpDMl9ZC0De7qOxq////I2j4zx68V8oypTJvO2nQGrsn3ks/PAn8Wb8cF6Fu46/gJPPa3g5k9na83N7hmHoW5kmJ1TPrdA7myo5MraXrtXBSYJzc2vI4HxpcAxtrUEx6fwa2k0hUa/oErI9tK6mJTZ7RNSP2po6SV8ay/tngS0Re4Z2MO3Eo/Gvi7pPys+lNwAbJZqq/0GXkaF0JTXFs5IdxRnkq9HZ6QsDl+fZ9LddWsr/wsBUzSiP6olEaY3DCtzfPT78I1quNT2Tb4he+MuxzyjABWM7PeuKn8OA2DOWY2FB+Mekn6TaqvVW4w/CgnDCpp92KSHsYH2KXxeM5zuO92fDJ/W+ceiqdw4dBd0tppkBqCm+5HS1om1TtHyXEWwgfIbNB/Fe+oF5bpyBWTBONQfNAYig+QO+LXNT8ADMcH9u74gDJVyrb53I+BuEm/Qa7+TvJMso3ksZzz8WyttvjclS3wmMwd6dpnbsmr8YDrObhg2Tl9746vVzYqd+zv8UHd8OVghpnZmNw9/QR3oRyMW5RZOnUteBu4AR+wVjeze3Hr5QH8Wt6D37fDcBdpbysTu2gOkltyBPCUpF+TMpXkiSZf4y6lrXBBDy5wBqTv7+D3eX/z+N40sbEkUBZIQnZXfCLqAFwxPN3MPq9EM5e0pXwNt2G4a/UOeUxtDK4ItcJXaWidq2+JdD4GLjySAP0Cvw+/TeeGGlbO2FTSoXji0CD8OWiLL4R7KR5L2g53g16NeyimKIXmcbuX8Hlz2diSH88/xvvD6pKWT9vmxse1z3FhOQBPLuhrPrO/5q/c+NkJGHla5pv4rPpblJZCyA2cj+E3Zn9JHZOr5XV8mY9eSimP6Tc/Jg01myB1A7CIpN65Q56HazYrpXJVawepMy6MWxEj8UFyRzMbgGvMvwF6ZsIrJzAn4w/Fp7hWC+7LvxzX6s+UtGRO4+8sz+C5FB9ob0pt/srMrqzW0io5h1a4tbUKrpEdY2a3mdnz5vGAF1O5fMr2vPiDV05DexNfb2zbXP3z46mY1+Ma4G+Ag/BBt5VXY5/b1BlvmcZ8dvr9zbir4ywze8umnoyWCeKXcOH2UbnzNLNvzexSa1givyakAeAePNsoCwY/hA9Ka+KWWV+gq5kdbT7fo16e8S3xwWxp/P4cCFMlmpyLu3C3ljRPukfD8YF+fuDLXD8tt8zOvLg34QE8K2pH3EW1cuYGm96zJ3fVvoEnsExI9XyX2n2hPKHkBTzxoA/QOVkgvfHEki+Z+p0tWeLLpbjC2ku+4vh4PCPsCvyZPAS3WlbE7+37+AoLL5jZ1+ZrpF2JZ0JmQf0sNnwa3sc3TeeXzZPJstmy9dV+m/Z/nc7rclyR3t7MVjCzWxq7LjNLvXS+WYZ5Wt8/07+dgcsl/SW3/wu8k7XBzWtwi+F+/AFeX7l8+tLq0+fOufquA9Y3s381pb3yuNC9wKXmQct+eJrxe6n+e1P7diK58PJWEi4wh+Juv+WSaX8hDfGaIZLOkXQa7ku+BReIp5vZ201pcyOsjQ9+p5nZteaxkSxmku+HWbbVI7jrYeMk9LKyWbB6FB6wXVieDDDZPBV3a9yn3Ms8eHsNLig74WnDpLo2lM+VyFaAvhAfULrgL4GbkAayvOWUJTN8gLvIFk91TbNaQIFkcYhdcprpOPPkhPvM7M7U9jlK+kGzkHtGbsIt/G/wge88SSdkCpuZfYRndG1PGhDxQfsqPOa0cTawpnudWUBd5BMWP8JdnbvjrsmlzeyESrRySZvSMCZsgS+SehBudVyPP/drmq8UcBNuCdwg6S3cDTYWd4X9XdL18smzmXX8P3w8WT3VDa6YfI6nJ3+ED/j/S+e6GD7GtM31q3Z4PGrhVGdW9224wrlz1hdywgUaJqROeTWDuWvxWDM72VKaf5HM1gJG/nrU1dP3fCrtVbhWMRj3zx4m6ZrcA/sYLlAOgSmT8q4jLVhJQzAy06g6yvPF++Nadf+0vVV6yKdapqHCtpfGhbaRx4W+tIYFFrNzOhvXujdI7ZqSVZJccHfgD8ERaf9Y3JW3B55JswcuoHqkY/3azAZV2+Z0zOUl7ZKufb5/ZVk8w/LnZ2nCm6Q1JN2Ca4tLpbKX4JZCn1zZ7Ny64tbQc8CPOcFzr3nWy/O5Yz+Hz47uK5+h/zyuyX2BL1rZMQ3El+DxsilzanJa4RxyF2UbkhCkIf4zy+Ib6d4NxvvvXvk2pnZm12GWv/ZBvtLABnL/fpYQkq1u8CIeF2iLD9Ln4FbmgNz9Pgd3926uhln4T+LP6D648M8UjUUkbY+7Sk+Qr5rxsnmG1tAK25sN4K/j8R6A5yytsJ3cYvfg93vFtP9l3KW1JfAInp24Gb66wrW4W+suTT07/mJcOBwtXwHkXlx47om7vW5P5/os/nz0BRbNnnO8P36GJ9tkbc+smFNxhfDQ1OaJyROxKe4mHIK7VqdgTXz9cVOYLSdaJq3pMHzgfR9YO2m8+QlEN+ETj47BB7ErcUtgF/OJUXviHeYwM7tIPqlrN9xv+VGq+794x/k1PkhPxn2qNzP17O1K2jwn7md9Pu8uSm3dFtfurjKzQ1Xm/Qzy92gb7k99PzfQTE5uwavwQf4XZvapGibctcc1nUWShdQk5AHV43Arak7cajrCGib4XYQLikVs6gUJl8Wv81q4BdEGd//dl/YPxh/CvmZ2YzqX5XEhvg4e8H2mgvZthSsJ8+ED3JXAC5abqCafiT8m7d/PzL6TZ6ktjg8iu+PC+lJgWfP3+8wy1DBJMJtM93szW2RGv5sF7VoOv/fr4c/A4niAeiieePJ8Krc+LhBGmNl28snLF+ID/L5m9lxSMtbCs7yeSgJ9N/ze9caF1Dr4ILwengCwf7J4Z9TO+QHhb4n8sGTfbriLex8zu04Nr1nYD7cwjjJPEUY+u/6f+ATZS3J1tMEFz3W4pbZnau9uwAW4FT0GX/PvD+WUOEk74F6EvXFX3dG4GwxcCF5uZg8ol/wiXyF6F9zt9TYuELfDFeL9zOzhGV2bopgtLZg0+C6L++k74OZsplFkWsvfcMGwZnKj7IYPjPfJg4RDcM3pqFTnuFSuDy60zsdTmi/BtYeh+DtRBiUTvhrhUm1caIIaMkKye3geHhCfsr5UpgGlQfR2XOvNltufMtfFzL5pqnCRL6FzBi4Av8WD6vvgPvZ8jKIt/oCtn36XWV/j8YdiGzyo/j0euMxiI9nEsn9KegkfBG7Hg6Bn4isTVMLjePzpfeBIM3uoRLjMYQ3B5q2BTeUrMvTHr/0F6bf/TNdrlgoXmCpTagyewLGwpHVmdTsyktU5FFfMVsAVhbNwF/Hf8WVybkjaNPg9uBufv7GWmf0dz8BrgwfTd8c9BIvi179dcvcMw63Q0/F7PxAP3q9vZsvPSLgkq/pmPFngXuBFScdo6qV9nkztOygpXxPk0wuyeUe358o+jScA9E3PLjDFwr4TH+wn4unw/8G9AoPx/r0vaW2w1LbSWNIjqe5r0jV8Hp83ty+eTDBUvmRVl9xvDsX76fK4YNkYj/Gs1JzCBWZDAZMbuEbis4F74hrDZZK6WcP8iOfxgWM3SSuZB7q2pmFy3B54Z1wodfxMgxyIa7Pr4msN7Qt0M7ODbAapj41h1cWFsiy1NmlfJoSylMgdlFaELem8D+BL1/RTWvywKW3NyF3n9fFB4kr8vROXmc9QH50spKyP/Sd9bpTaneXkf4BroEPMJ0hmPu9fp3LP4A/NGbiF1h5PqVzUfHZyRW8lNF+n6Vb8Ie1epkh2HU/D78Hp+IN+Iu4uXcjMds4ssjpgCdzia5a3MqbB9z7cmuiNx/PONp/wOBhXMPrgMYUTJCkJyHvwpJNsMuqN+PM0En/uNsCtks1IQW18gL8RP+fXcBfuxmaWvXepsTauIl966VXcnXk27oYbid/fA3LFP8T73m/xeUyDcZfVWPwle/mX6X2Uyq5EmoOUjpcpbUPxQX4NXNj+HuhnZk8mAfQQ0EM++790fa8vcG8DeF88OLn9MsXnCvxZuCx5PTCfWHq+mf0qtVXmMZaKM1SLYrYTMLkB53XcXfMhbk63Af4lf9tkxnm4pbOppPbmi8z1wn2sJ+Jaxmg8+ynva//OfCmRe83srmpupGY+LpStsJp/V3dr89Tdy/AHc+2svTkrZjTuk13DSpZ4r6Lt7SVlfv/sOu+HP4Tnmdn71hAjKU2AuA+3ALZSw8TSLA6Tv35n4Vbn5sktiZl9b2bH426TnubLrDclpTJ718YhalghmnSMSUlz/RBXLIS7wuY2swOsiUvf1Bp57G1PXEt+As+aaw4+xLX6b4DHzGxk1q+SNTjZ/LUBF+KurGwNrkfwvrC5pF8lrf8j/HwuxzPAFsaf3U1S3/0JH9DnN7Pd0nNaCafiAuAc3M16vpllGWY/AHtnFkjqz4/iVu7puCt1d2AbMxuhqZM9JuPWzlP4bPk5c9szRppngo0xX13gy5zC92/cUj8o/Z+fOT8JT9F/lYYVRLLg/Sd4XPh03OJ/OhvPcsLt3QqvzSxhthMwOebC/Z2/MX/x1y646f13paWuzVM8n8QFybJp2xjcLD4OD9x1BVZTbt2watxfGfIg8RH4gHBr3ppKAuItXLisjQuKg3HNcJCk5ZPmfCM++evArM6S9lyIa7RbJX/zFL99Kne/zUSqMe6+uCbn8gDXLvMrQpNz47WjIStsfGqfSA9WZsXkrxGeudMBtz5rneY7BteS12TaxTHz7GZmrZMga3YtME+6l/PiCtSBNgsCtmp4/05+kP0UHyjnxxNE8m3MW8en4/dyW0lLJMXgblwpOTLV28Z8uZ+DcQEzPtW7Mg3rx31hJa+/LtPOHZVbQgYXLF8C85onx4yTz7t5H7foFyWtTpH4H/4MTsaF0aDkPSj3zL+Px1q7U8YiLhE2pd6EobiA2jyz7DR1QswnuKt2E7yvMg3H6wAAC7lJREFUZs9KazMbb2Yn4p6ML3FBVTcTaUuZ7QRM7kb+F08DzFL6RuCunA74IJktdX4hvux599xvvzOzf+Am/gu4a+mzmWmX1TYudGyuzuy8W6cB9EY8QJjlvs90x8t1/iG4++Lw3PZhuBvkXPly6Ufj6zJlS6TcJF+2BDM7H/el7yNfin2+9HDNJX//+J9w4XMTcE7pQ12jh+hOPG1zf029wGLeKvtmml/VFxeb2ZYlbpuaI18G6WPg7GSVZEHl/JpX9+JvBl2gzO/bmKfFX4MP5KunXcNwLX1XSQtZw+rSWR/ZA0813qVSyzG5hfsDl+TqegyPYWwkad1UdFKyWhbChfRb2TklZeIhPCb4f5rOHKL07A3BX90xw1XPc96E1knZug0XZAeXKTsJd4mPAvZQenkfvlJIZqlcaGYbmNnLMzp2czLbCZjcIDQO92euBD6JEO/kC+B+3kvkS6Hcgqce7kp6CyUNk6QGAT3NJyONaGqbCo4LlVoxp+L+7Rmt0Du99q4gaWdJm0nakAYB+CSu+W0sn0k+Kf1/MS4Mb8X9w/vimXnf4df6UvnMevB5PFnQ/Fl5Nt/luDZ4Ln4vjrNcplmNydZp2oKUcFBKvWqDGbOwfb1wd1VP4GI1LPqZDXLZtVyKhjdnTiEnsO/DkzyyOS9fM23SSf53Q80TAComteUf5N6FkrgUVzSzNbk64QrayvgacpkrN+vjb6Vz2oYGgdjYMd80s6uqtCKze/cffJ5XL0mLJhdtfjx+H49r7krDtIgmv6umuZjtBEyONvgcj1/Ksy5ewdeiuh/PAFsKtyIWxQfl7pSsGZb8w9W8r70sVmxcaKpgvZl9aGbnWIXB7zzyN909hguNc3BBMAzPutk01Xkb7nrMsuu+NF/wb318XsaewJL4nJqVcJfX56Sld8zsSdylsi/+gC+ZrsmzuDuzj/mqCIWQHtDb8Oy/Wk4knW3IKUTP4gPdvrj1f5oaVrjOyjyNxy0OVC6jqoRsnMmvjNykpBP5+nIb5GN8OYvqUVwJylatwDyo/hLuorsBnyS7T/rsDtwoaQNrWOpnHD74fwLsq2mzvGaKnBXzLS5kO5JWZWDaV1PciS/50qmWbZiVzLYCJrkPJuID8RH4Q7A1HpQ+BR/gN8Fzzt/AYyNNXhm4QgqLC80MknaXz0r+N271nYJn2KyOZ7IsgU+I2wK/jvfiEz+XTb9va2ZPmNn1ZnaDeaB8bPIXP4oL9WWz4L55yvcAM+uJC6KNzeyPlnsFc5GY2btmdoj5LOufNZJ+LX8lcbbmW34m+A/4S6kexvve7rgbN680fYSnjq9Ew+rXWTJCqWAZke2zpied3IVb87/K1ZXFGN/Fn+dfKL0JMnEprsRshis6WdxkF1zRvEfSgWpYiflt3KL+Aw1zUGpJZoXchbvyd5e/5XZiyfjzKrC0md0+TQ0thNlWwMgnyL2Ia8kbA4eaB7m/NbOfzNcn+gu+NtLCeEe/uggTtF7jQqltq+D59h3xiZh9gSvN7B4z+6+ZnYQnG3TChc3C+AM+joaU6fxM8jZpkMrHTzrjgnWagLn5PJxyL/YKCkRST0nZKsOnAlfI18rKD3LjcZdTV9yNeRIeuzpJUieY4pp9DI+tHaSUnZfcOZPkC1sejsceX8j2pc8ZJp0kAbinpN+nTefhz9B6eeGSa/OD+HOWpUFjns32Em4NDDXnE/P5M1vjCtMF+Np87cyTCe7EM9dqvlioNWR2foYnFcxHWjWEqa2YSU3xRNQTs62AwR+OOfGB7WPLZaDkzPvTzayb+Yq4hfk26zEulON13OfcFnjTfP7KJOWWojGzu/H4yKq4QHwYH5h2ky9fM+UhSG7Fien3i0jqj899GWCVp5cGBZEUgDPxwPsYXMk6ALcivysp/jQ+K7+tmX1nPhfjTDzGcUqu3Ae4FbMeDe9EmVuebfg3XLHYx6qYJ1YiAE/Brfy5aPA47E56NvKCJu27GX8P0sa5Kq9O5bNVjbO+/Ro+V+VW0jp/Scg8amZ7madQF0EmSO7GFcWVU3uade24WjNbLhWTIeliXCvvAXzU3AGy5FIagmtHn+GT0RbCXVPv4g/uy/ggvhb+MB1QbcCzCe3aILXrL2Z2eiNl2uGxlA/wSaYb4RrexWZ2THIvdMZjKsvjbpFN8Wv/H3zZmJFFnkcwY9K9vgOfdX8evmxKfu26/ErT6+F98IhkBZDihdfj7ttjgSvM7CtJK+LZf//DhcpO+ETkT/HXM9wJ/9/euYTGVUdx+KuKUYNuBItWC7rwLLRYuhAXdSNp1EIVX7iQWrAg6KJqQe2iWkFsF4Hg27rSFpFSsaGxWEUQNNqVi6JCOCnGWkSsKYKLgIkaXfzO7dwMMWlt5s719nwQZjGvf5KZ+zvvs3CBQlz4X0C5vK9QgcA4MFp8fsxsI2o43ODu75SeW4zSWYWqKX9w92IN8EUoRPczGgf1U4TwloRBdBn6XH9+OkK4GJjKt+vSwLuoNNmDAX0wKx9I+G/UNC8ECm8cQHsulrbfGSGvaWT1XotyM5+i0MhDpr6XP5C47ELFFJtQk9hd7n5vikttuB04z7WXfVZj7BzhpglUZTVjZjeaZl6NII/kEGr63RqPHUPCcxvyOu5EBssKd9/vp14BtRrtIdqLStZfd015OFo6VyFk90eRTrs4fotEtM9i51GEYXcj42dd8WbequD8xd2HqhaXeO9Gigs0X2CKUdlTFVykF6ROeaEy8eXbhUbQ3zPH/UUI7DMUqltailP30upIHkMDOx8A+l3jPObddZ5UzgXAb6YtisBJ675oBu4tfd7GUVj3NdR/dQNKkt+BwrfvoSqwx5EQHUSew3p3v8LdX/4P55tXAENIJuNM/bQaEcte2DTyfqYojXJBucYeYoZY08JRdaR9nEfTGHL3fd0+RIl580JxId8eifWqGUElyY+a2bttZyusw6IPoqgK+hAVADyPGiMnUTL1a5K6shN5yjvM7FU06v1qlJ9YBXxjZm8ja/9iVFG1HOUnvgCOeWst8XPx3EEUjhpCxsWZcFIAixxjCOB0lCb3oP6qt1Ai/z4z+9JjQni8xl/IkOshCmFCoCbMbBvKNda9mbYRNNqDqUNYrEwIyI9oEuqFZa+q5Kp3ZTRJfOF2o6m4swb4lf6Oy+J2JJ5zFIUrBk1zyrruJSbz45qaPYB6vg6gCrINaInWGBKZN5F3/SuqHhwFPnD370rico5radx6YKUvXintTrRca4eZrTWzraiR92MUsjtoZhvjbC8hj7svfreiwKQXiSi0tjoW36/BKFpJKqDRSf46YmZPoIqs5d62k6LbmPaMDCPDY43P3tuyBlWbjaIu+Ck/w4nMSXcIS38FMhguQeNRJlFl0zUozHUuEqGngZvcfWVbj0zxWksW25ALL+NZWpVWM6j0eAb1pZyP9ji9gSZs/4nK90dR1Vs/ygHudffHFvNsyemRAlMxZnY3svoeBk7Uzcuy1oKlte7+UST9b0Ff2CngkSw3bh5l8TCz7SivdivKtzyJ1iMcr+oszC+Ae5AArkYl/wNxexyFoS9HY1a2ufuJKs6czE3TczB1pG55oXaG0dSAp0wLmR5Ek4cPoRlho52wWpPuEHmNv9s8E0OhsXF0gZ9AzYCVCEyc5XD8FOcsBPCwme1DS7audPdPot9lHRKfSdRzlcJSA9KDSWYReZTNyCoElTA/46e45zz5fxL/92Wo6XALsMfdN0Xjb9cu1nMJoJm9j4ZrXpfedL1JDyaZRZSCDqOdI/s99qknzSM81OtR0+SlKFneh5L/r8TDKu8LKVPk+doE8GbgxRSX+pMeTJKcpcT0hSNoTNDvKCQ2UBdvdQEB3OJa0pfUmBSYJDmLifFFV6E+lu+7fZ4ydRfAZGFSYJIkqS11FsBkYVJgkiRJko7Q6E7+JEmSpHukwCRJkiQdIQUmSZIk6QgpMEmSJElHSIFJkiRJOkIKTJIkSdIRUmCSJEmSjpACkyRJknSEFJgkSZKkI/wD13XJ3SKVGEIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AleLz6A36Ajl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 647
        },
        "outputId": "71f228c2-0af1-4133-d5dc-d6606a0e81cb"
      },
      "source": [
        "\n",
        "model = BertForSequenceClassification.from_pretrained(pretrained_model_name_or_path='bert-base-uncased', num_labels=NUM_CLASSES, output_attentions=False, output_hidden_states=False)\n",
        "model.cuda() # Tell pytorch to run this model on the GPU.\n",
        "\n",
        "print_model_params(model)"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (8, 768)\n",
            "classifier.bias                                                 (8,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rmWzQ7HV6CjG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "85592090-ba38-439e-9222-be738b236015"
      },
      "source": [
        "bert_model = models[BERT_MODEL][0]\n",
        "lowercase = models[BERT_MODEL][1]\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained(bert_model, do_lower_case=lowercase)\n",
        "print('Text will be split into tokens using the', bert_model,'built-in tokenizer.\\n')\n",
        "\n",
        "print(\"Hello I am a MONKEY \\U0001F648\")\n",
        "print(tokenizer.tokenize(\"Hello I am a monkey \\U0001F648\"))"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Text will be split into tokens using the bert-base-uncased built-in tokenizer.\n",
            "\n",
            "Hello I am a MONKEY 🙈\n",
            "['hello', 'i', 'am', 'a', 'monkey', '[UNK]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_5Sj7T1k6EkD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "outputId": "234defd3-48e8-4a3d-9be2-7a5a487ae939"
      },
      "source": [
        "lengths = []\n",
        "for sent in X_train:\n",
        "  input_ids = tokenizer.encode(sent, add_special_tokens=True)\n",
        "  l = min(MAX_LEN, len(input_ids))\n",
        "  lengths.append(l)\n",
        "\n",
        "plot_lengths_distribution(lengths, title=\"Length distribution of tokenized train sentences\")\n",
        "print('Increase MAX_LEN if too many truncated sentences, decrease it if too much padding.')"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.1% of the training examples (18 of them) have more than 48 tokens\n",
            "Increase MAX_LEN if too many truncated sentences, decrease it if too much padding.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAFjCAYAAABYCzEiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxU5f4H8A8gi7Io0CCKCrgwKMhmpqipKMkSJriEG0iaplE/xXtT6Ea3m3U1IpfEUkxTCXcg3EWxa+ZC5QIXJb2gpqbIKILs6/n94YvJaQaH0YFh5PN+vXjVPM/3POd75rH8cs55ztERBEEAEREREdET6Go6ASIiIiJq/Vg0EhEREZFSLBqJiIiISCkWjURERESkFItGIiIiIlKKRSMRERERKcWikYjoMSEhIRg1apSm01DZqFGjEBIS0ixjZ2RkQCwWIzk5+Yltza215KHJ/RJpUjtNJ0BEit28eRPx8fH45ZdfcOfOHRgYGOCFF16Ai4sLgoKCMHjwYE2nSC1s06ZNMDMzw/jx4zWdyjPRluPIycnB0aNHERQUhG7dumk6HSKNY9FI1Ar997//RUhICNq1a4fAwED07t0blZWV+P3333Hy5EkYGxuzaGyDtmzZAhsbm1ZRbA0cOBBZWVlo1071v0ae9jieZZ9PIycnB3FxcXjppZfkisaWzoWoNeCfdqJWaM2aNaioqEBqaiocHR3l+iUSiQayIvqTrq4uDA0NW2RfpaWlMDExadF9KtOaciFqKbynkagVun79Ojp16qSwYAQAkUgk13bq1CnMnDkTL774Ivr374+xY8di27ZtCrffuXMnfH194ezsjFdeeQWbNm1CUlISxGIxMjIypHGRkZEQi8UKxxCLxYiMjJRrP3DgAKZMmQJ3d3e4urpi0qRJOHToUKPbnz9/HtOnT4ebmxsGDRqEf/zjHygrK5OLl0gk+OSTTzB69Gg4OzvD09MTb7zxBk6ePCkTd/36dbz33nsYNmwYnJ2dMWrUKHz22WcoLy9XeBxN1dRxG76zkpIS/POf/4Snpyf69++PyZMnIzMzU27cBw8eICoqCoMGDYK7uztCQ0Nx6dIluXsrxWIx/vjjD/z8888Qi8XSn1u3bsmMl5eXhzlz5sDd3R0DBgzA//3f/6n0S8bRo0cRGBiI/v37Y8SIEVi5ciVqa2vl4hTd01dfX49NmzZh7NixcHd3h4eHB3x8fPD++++jpqamScfRcG/mpUuXMGvWLAwYMACvvfZao/t8XEJCAnx8fNC/f3/4+PggISFBLqaxez//Ovbq1asRFRUFAAgNDZXm2fBnvrFcysvL8cUXX8Db2xvOzs4YOnQoFi1ahD/++KPR/SUlJeHVV1+Fs7MzvLy8sH79eoXHR6RpPNNI1Ar16NED165dQ1paGsaMGaM0fseOHfjnP/8JNzc3zJ07F+3bt8epU6fw0Ucf4caNG1i8eLE0dtOmTVi6dCkcHR2xcOFCVFRUYOPGjbC0tHzmvFesWIG1a9fi5Zdfxvz586Grq4sjR45g/vz5+PDDDzFt2jSZ+JycHMydOxfjx49HQEAAfv75Z+zevRu6urpYsmSJNO7WrVuYMmUK7t+/j3HjxsHZ2RkVFRXIzMzEqVOnMHToUABAdnY2ZsyYATMzMwQHB6Nz58747bffkJCQgPPnzyMhIQH6+voqH9fTjDtr1ixYWFggPDwcRUVF+PbbbzFnzhykp6fDxMQEAFBdXY033ngDOTk5GD9+PPr374/Lly/jjTfeQMeOHWXGi4mJwdKlS2Fubo65c+dK2y0sLKT/fvfuXYSGhsLb2xuLFi3Cb7/9hh07dqC0tBQbN25UepxHjhzBu+++CxsbG4SHh0NPTw/Jyck4fvx4k76nr7/+Gl9++SW8vLwwefJk6Onp4datWzh27Biqq6uhr6/fpOO4ffs2ZsyYAV9fX4wZM6ZJBf93330HiUSC4OBgmJiYYN++ffjkk09QXFyMd955p0n5P+6VV16BRCLBjh07MHfuXPTs2RPAo/82G1NTU4NZs2bh3Llz8PHxwRtvvIHff/8d27Ztw8mTJ5GUlARra2uZbbZv34579+5h4sSJMDMzw549exAbGwtra2uMHTtW5byJmpVARK3OuXPnBCcnJ8HBwUEYM2aMEBkZKSQmJgq5ublysXfv3hWcnZ2FhQsXyvUtWbJEcHR0FG7cuCEIgiAUFxcLrq6ugp+fn1BeXi6Nu3PnjuDm5iY4ODgIZ86ckbYvXrxYcHBwUJijg4ODsHjxYunn7OxswcHBQfjiiy/kYufNmye4u7sLJSUlMtuLxWLhwoULMrGzZ88W+vXrJ5SWlkrb3nzzTcHBwUH48ccf5cauq6uT/vvYsWMFHx8fmf0IgiCkpaUJDg4OQlJSksJjedz06dMFLy8vmTZVxm34zv75z3/KxB44cEBwcHAQtm3bJm377rvvBAcHB+Grr76SiW1o/2seXl5ewvTp0xXm7eXlJTg4OAj79++Xaf/oo48EBwcHIS8v74nHXVtbK4wYMUJ46aWXhPv370vbHz58KIwcOVLuOM+cOSPXFhgYKPj5+T1xP009jp07d8r1KdpnQ5ubm5tw584daXtVVZUwYcIEoV+/fjLtje1b0dhJSUly/008KX7Hjh2Cg4OD8Nlnn8nE/vDDD4KDg4Pw97//XW77oUOHCg8fPpS2l5eXC4MGDRJef/11hd8PkSbx8jRRK+Tu7o6kpCQEBQWhpKQEycnJ+Ne//gV/f39MmzYNN2/elMYePnwY1dXVmDhxIgoLC2V+Ro0ahfr6epw6dQoA8NNPP6GiogLTpk1D+/btpWOo46zG3r17oaOjg8DAQIV5lJWV4cKFCzLbuLm5wdXVVaZt8ODBqK2tlV7OKyoqwokTJ/Dyyy/j5Zdfltuvru6j/41dvnwZly9fRkBAAKqrq2X2P2DAAHTo0EHuUnZTPO24YWFhcscFAL///ru07YcffoCenh5CQ0NlYidNmgRTU1OVc7WysoK/v7/S/Spy8eJF3LlzB+PHj5c562dqaorJkyc3af8mJia4e/cufv31VxUzl9WpUyeVF8mMHTtW5iyegYEBwsLCUFtbi2PHjj1TPk115MgR6Orq4q233pJpHzlyJPr27Yv09HTU19fL9E2YMEFmrtu3bw83Nzdcv369JVImUgkvTxO1UmKxGMuWLQMA/PHHH/jll1+wa9cu/Prrr3j77beRlJQEAwMD5OXlAZAvUh537949AJDeN9Zwqe1xvXr1eqZ88/LyIAgC/Pz8lObRoHv37nIxnTp1AvCoWASAGzduQBAE9OvXT+n+gUf3oq1evbpJ+2+Kpx33r8dmbm4O4M/jAh7Nh5WVFYyNjWViDQwM0K1bNzx8+FClXJvyfTam4ReRZ/mzsXDhQoSHh2PatGmwsrLCSy+9hJEjR8LHxwcGBgZNGgN4dBx6enpNjm8sx969ewOAzC9ZzalhPv96a0FDLjk5OXjw4IHMrSCKHuXTqVMnpfNFpAksGom0gI2NDWxsbDBu3DhMnToV586dQ1ZWFl588UUIggAA+Oyzz2BlZaVwe0XFRFPo6OgobFe0MEIQBOjo6GD9+vWN/oXf8Jd4gycVBg3HpaqZM2cqPCMJAGZmZk815tOM29ixPe1xNUVzfJ+qcHd3x5EjR/DTTz8hIyMDGRkZ2LdvH77++mts3bpVWsAq8/hZ8JZSV1fX4vsEnjxnRK0Ni0YiLaKjowNXV1ecO3cOBQUFAAA7OzsAj85kDRky5InbN5zVuHr1Kjw9PWX6Gs6oPa7hjElRUZHMX/iKztzY2dnhxIkT6Nq16zOftXxcjx49oKOjg5ycnCfG2draAnh0uVrZ96CK5hoXePTLwOnTp1FWViZztrGmpga3bt16piJXVQ2/WFy9elWuT9GfjcYYGxvDx8cHPj4+AIDExER8/PHH2L17N9588031JKuAohxzc3MByP7S1NhZPEV/phv7pakx3bt3x4kTJ/Dw4UO5ucvLy4OJiYn0jDORNuI9jUSt0MmTJxWezausrJTeP9dQmPn5+cHAwACrV69GZWWl3DYlJSWorq4GAAwdOhRGRkZITExERUWFNCY/Px979+6V27ahIG24J7LBt99+Kxfb8FiU5cuXKzxr8zSXhoFHf8kPHz4cP/74o1wewJ9n0Pr16wcHBwds375dYQFQW1v7VJf8mmtc4NHjX+rq6rBlyxaZ9p07d6KkpEQu3tjYuNkuWzo5OcHa2hrJyckoLCyUtpeWlmL79u1NGuPx7R4fFwCKi4ulbc1xHHv37kV+fr70c3V1NTZt2gQ9PT14eXlJ2+3s7HDt2jXcvXtXJjYxMVFuzA4dOsjl/iTe3t6or69HfHy8TPvx48dx6dIljBo1SnoPLpE24plGolZo6dKlKCoqwqhRo+Dg4AAjIyNpYXf9+nUEBgZKn59obW2Njz76CB988AH8/f3x2muvwcbGBoWFhbhy5QqOHj2K/fv3o1u3bujYsSPmz5+Pzz77DJMnT0ZgYCAqKiqwfft22NnZ4dKlSzJ5BAQEYMWKFfjwww9x9epVdOrUCSdOnMCDBw/kcnZxccG7776L1atXIzAwED4+PujcuTMKCgpw8eJF/Pjjj8jOzn6q7yM6OhqXLl3C7NmzERgYCCcnJ1RVVSEzMxM2NjZ47733oKOjg5iYGMyYMQOvvfYaJkyYIPMmnSNHjmDhwoUqL7BornGBRwtetm/fjpUrV+LGjRvSR+4cOnQItra2cr84uLq6Yvfu3Vi5ciV69eoFXV1deHl5SYubZ6Gnp4eoqCgsWLAAkyZNwuuvvw49PT0kJSWhU6dOuH37ttIx/P394ebmBhcXF1hZWUEikWDnzp3Q19fHq6++2qzHYW9vj0mTJmHy5MkwNjbGvn378N///hdvv/02unTpIo2bNm0a9u/fj7CwMEyePBk1NTVITU1VeEm8f//+0NXVxdq1a1FcXIwOHTqgW7ducou3GgQFBSElJQXr16/HH3/8gRdffBE3btzA1q1b8cILL2DhwoVPfXxErQGLRqJWKDIyEunp6Th79iwOHz6MkpISmJqawsHBAbNnz5YrUCZMmAA7Ozts3LgRO3bsQElJCTp16gR7e3vMnz9f5mHgM2fORIcOHfDtt9/iiy++QJcuXTBz5kyYmpri/ffflxnXxMQE8fHxWLp0KdatW4cOHTpgzJgx+PzzzzFw4EC5vN955x04OzsjISEBW7ZsQXl5OSwtLdGnTx/84x//eOrvo3v37khKSsKaNWvw448/IjU1FWZmZnB0dERwcLA0rm/fvkhJScG6detw7NgxbN++HcbGxrCxsUFQUJDcJfmmaq5xDQwMsHnzZsTExCA9PR0HDx6Ei4sLNm3ahH/84x9yZ44jIiJQXFyMrVu34uHDhxAEAenp6WopGgHA19cXX375JdasWYPVq1fD0tISQUFBGDhwIGbOnKl0+5kzZ+L48eNISEhASUkJLC0t4erqirfeekvmQfXNcRzTp09HaWkpvvvuO9y+fRtdu3bF+++/jxkzZsjEDRgwAMuWLcPatWvx+eefw8rKClOmTIGzs7PcYrKuXbvi3//+N9avX49//etfqKmpQVBQUKNFo76+PjZs2ICvv/4aBw4cwJEjR2BqagpfX18sWLBApngl0kY6QkvcHU1ErV5ycjKioqKwZcsWDBo0SNPptGl1dXUYPHgwXFxcsGHDBk2nQ0QEgPc0EhFplKL7ULdv346HDx9K33RDRNQa8PI0EZEGffDBB6iuroa7uzsMDAxw/vx57Nu3D7a2tnj99dc1nR4RkRSLRiIiDRo2bBgSExNx+vRp6T2gkyZNwvz586XvqCYiag14TyMRERERKcV7GomIiIhIKRaNRERERKQU72lUkwcPylBfzyv9rZGlpQnu3y/VdBr0DDiH2o3zp/04h9qvYQ51dXVgbm6sfAMFWDSqSX29wKKxFePcaD/OoXbj/Gk/zqH2e9Y55OVpIiIiIlKKRSMRERERKcWikYiIiIiUYtFIREREREqxaCQiIiIipTS6erqgoABbtmxBZmYmsrOzUV5eji1btmDQoEHSmIyMDISGhjY6xoIFCzBv3jwAQHJyMqKiohTGZWVlwdDQUKYtPT0dcXFxyM3NhaWlJSZOnIi5c+eiXTsuKiciIiJ6nEaro2vXrmH9+vWwtbWFWCzG+fPn5WJ69eqFmJgYufY9e/bgp59+wtChQ+X6IiIi0KVLF5k2fX19mc/Hjx9HeHg4Bg8ejOjoaFy5cgVr1qzBgwcPEB0d/YxHRkRERPR80WjR6OTkhDNnzsDc3BxHjx5FeHi4XMwLL7yAcePGybWvWbMGdnZ2cHFxkesbMWIE+vbt+8R9x8TEoF+/ftiwYQP09PQAAMbGxoiPj0dISAjs7Oye7qCIiIiInkMavafRxMQE5ubmKm+XlZWF33//HWPHjm00prS0FPX19Qr7cnNzkZubi+DgYGnBCABTp05FfX090tLSVM6JiIiI6HmmlTfv7dmzBwAaLRqnTp2K8vJyGBoaYuTIkYiMjETXrl2l/ZcuXQIAODs7y2zXuXNnWFtbS/tJs1xc+yH/zi2lcdZduiErk3NGRETUnLSuaKyrq8PBgwfh4uICW1tbmb727dtj/PjxGDRoEIyNjZGZmYnNmzcjMzMTKSkpsLCwAABIJBIAgEgkkhtfJBKhoKCg+Q+ElMq/cwsBC79XGrdveWALZENERNS2aV3RePr0ady7dw9vvfWWXJ+fnx/8/Pykn1955RUMHDgQc+bMwebNmxEREQEAqKysBAAYGBjIjWFoaIiKigqV87K0NFF5G1Ifkcj0mfqp9eMcajfOn/bjHGq/Z51DrSsa9+7dCz09Pfj7+zcpfsSIEejZsydOnz4tLRqNjIwAANXV1XLxVVVV0n5V3L9fype5a5BEUtJon0hk+sR+av04h9qN86f9OIfar2EOdXV1nvpEl1Y93LuyshJHjhyBp6cnXnjhhSZv16VLFxQXF0s/N1yWbrhM/TiJRAIrK6tnT5aIiIjoOaJVReOxY8dQVlb2xFXTity8eVNmlXbD43iys7Nl4u7evYv8/Hylj+shIiIiamu0qmjcu3cv2rdvj1deeUVhf2FhocJtbty4gWHDhknb+vTpg549e2LHjh2oq6uTtm/btg26uroYM2aM+pMnIiIi0mIav6fxq6++AgDk5eUBAFJTU3H27FmYmZlh+vTp0riioiKcOHECY8aMgbGxscKxJk+eDCcnJ/Tr1w8mJibIysrC999/Dzs7O8yYMUMmdtGiRZg3bx5mzZoFf39/XLlyBYmJiQgODoa9vX0zHS0RERGRdtJ40bhq1SqZz0lJSQAAGxsbmaLx0KFDqKmpQUBAQKNj+fn54T//+Q9OnDiByspKWFlZYdq0aXjnnXdgaiq7YsjLywtxcXGIi4vDkiVLYGFhgXnz5uHtt99W49ERERERPR90BEHgkl814Opp9bOyMmvycxoLCh422s9Vf9qPc6jdOH/aj3Oo/drc6mkiIiIi0gwWjURERESkFItGIiIiIlKKRSMRERERKcWikYiIiIiUYtFIREREREqxaCQiIiIipVg0EhEREZFSLBqJiIiISCkWjURERESkFItGIiIiIlKKRSMRERERKdVO0wkQPStdPX1YWZkpjbPu0g1ZmZdaICMiIqLnD4tGanEurv2Qf+eW2sarr6tBwMLvlcbtWx6otn0SERG1NSwaqcXl37nFIo+IiEjL8J5GIiIiIlKKRSMRERERKcWikYiIiIiUYtFIREREREqxaCQiIiIipVg0EhEREZFSLBqJiIiISCkWjURERESklEaLxoKCAsTGxiIkJATu7u4Qi8XIyMiQixs1ahTEYrHcT2xsrFzsw4cPER0djcGDB8PNzQ2hoaHIyclRuP/09HQEBQWhf//+GDlyJOLi4lBbW6v24yQiIiLSdhp9I8y1a9ewfv162NraQiwW4/z5843GOjk5YcaMGTJtDg4OMp/r6+sxZ84cXLlyBTNnzoS5uTm2bt2KkJAQJCcno0ePHtLY48ePIzw8HIMHD0Z0dDSuXLmCNWvW4MGDB4iOjlbvgRIRERFpOY0WjU5OTjhz5gzMzc1x9OhRhIeHNxprbW2NcePGPXG8Q4cO4fz581izZg28vb0BAH5+fvDx8UFcXBxiYmKksTExMejXrx82bNgAPT09AICxsTHi4+MREhICOzu7Zz9AIiIioueERi9Pm5iYwNzcvMnx1dXVqKioaLT/8OHDsLKywujRo6VtFhYW8PPzw9GjR1FTUwMAyM3NRW5uLoKDg6UFIwBMnToV9fX1SEtLe4qjISIiInp+ac1CmJMnT8LNzQ1ubm7w9vbGjh075GJycnLg5OQEHR0dmfb+/fujrKwMN27cAABcunQJAODs7CwT17lzZ1hbW0v7iYiIiOgRjV6ebioHBwe8+OKLsLOzw4MHD7Bz5058+OGHKC4uxpw5c6RxEokEgwcPltveysoKwKOFN7169YJEIgEAiEQiuViRSISCgoJmOhIiIiIi7aQVRePatWtlPo8fPx5Tp07FV199hSlTpsDU1BQAUFlZCQMDA7ntG9oqKytl/qko1tDQ8ImXwBtjaWmi8jbU8kQiU02nQE+Jc6fdOH/aj3Oo/Z51DrWiaPwrPT09zJgxAxERETh//jyGDx8OADAyMkJ1dbVcfEObkZGRzD8VxVZVVUn7VXH/finq6wWVt6OWJZGUaDoFegoikSnnTotx/rQf51D7Ncyhrq7OU5/o0pp7Gv/K2toaAFBcXCxta+zSckNbw2XqhsvSDZepHyeRSKRxRERERPSI1haNN2/eBPBodXQDR0dHXLx4EYIge8YvKysLHTp0kD6nsW/fvgCA7Oxsmbi7d+8iPz9f2k9EREREj7T6orGoqAj19fUybVVVVdiwYQOMjY3h5uYmbff19UVBQQHS09OlbYWFhTh06BBGjx4NfX19AECfPn3Qs2dP7NixA3V1ddLYbdu2QVdXF2PGjGnmoyIiIiLSLhq/p/Grr74CAOTl5QEAUlNTcfbsWZiZmWH69Ok4duwY1q5dCx8fH9jY2KCoqAgpKSm4fv06PvroIxgbG0vH8vHxgZubGxYtWiR9I8y2bdtQX1+Pd999V2a/ixYtwrx58zBr1iz4+/vjypUrSExMRHBwMOzt7VvuCyAiIiLSAhovGletWiXzOSkpCQBgY2OD6dOnw8HBAT179kRqaioKCwthYGAAJycnREZGwsvLS2ZbPT09xMfHIyYmBgkJCaiqqkL//v3x2WefwdbWVibWy8sLcXFxiIuLw5IlS2BhYYF58+bh7bffbt4DJiIiItJCGi8aL1++/MR+Z2dnuUfuPEnHjh3x6aef4tNPP1Ua6+3tLX3dIBERERE1rtXf00hEREREmseikYiIiIiUYtFIREREREqxaCQiIiIipVg0EhEREZFSGl89TdRSdPX0YWVlpjTOuks3ZGVeaoGMiIiItAeLRmoz6utqELDwe6Vx+5YHtkA2RERE2oWXp4mIiIhIKRaNRERERKQUi0YiIiIiUopFIxEREREpxaKRiIiIiJRi0UhERERESvGRO0RPycW1H/Lv3GpSLJ/9SERE2o5FI9FTyr9zq0nPfQT47EciItJ+LBpJbVQ580ZERETahUUjqU1Tz7zxrBsREZH24UIYIiIiIlKKRSMRERERKcWikYiIiIiUYtFIREREREqxaCQiIiIipVg0EhEREZFSGn3kTkFBAbZs2YLMzExkZ2ejvLwcW7ZswaBBg6QxDx48QFJSEo4dO4arV6+itrYWvXr1QlhYGPz8/GTGS05ORlRUlMJ9ZWVlwdDQUKYtPT0dcXFxyM3NhaWlJSZOnIi5c+eiXTs+iYiIiIjocRqtjq5du4b169fD1tYWYrEY58+fl4u5cOECVq5cieHDh2PevHlo164dDh8+jAULFuDq1asIDw+X2yYiIgJdunSRadPX15f5fPz4cYSHh2Pw4MGIjo7GlStXsGbNGjx48ADR0dHqPVAiIiIiLafRotHJyQlnzpyBubk5jh49qrAA7N27Nw4fPgwbGxtp29SpUxEWFob4+HjMmjULRkZGMtuMGDECffv2feK+Y2Ji0K9fP2zYsAF6enoAAGNjY8THxyMkJAR2dnbPfoBEREREzwmN3tNoYmICc3PzJ8Z0795dpmAEAB0dHXh7e6OyshJ//PGHwu1KS0tRX1+vsC83Nxe5ubkIDg6WFozAo2K0vr4eaWlpKh4JERER0fNNa2/eu3fvHgAoLDqnTp2K8vJyGBoaYuTIkYiMjETXrl2l/ZcuXQIAODs7y2zXuXNnWFtbS/uJiIiI6BGtLBqLioqwa9cuvPTSS7CwsJC2t2/fHuPHj8egQYNgbGyMzMxMbN68GZmZmUhJSZHGSiQSAIBIJJIbWyQSoaCgQOWcLC1NnvJoqDUSiUy1Ysy2hN+fduP8aT/OofZ71jnUuqKxvr4ef//731FSUoIPPvhAps/Pz09mRfUrr7yCgQMHYs6cOdi8eTMiIiIAAJWVlQAAAwMDufENDQ1RUVGhcl7375eivl5QeTtqnSSSEq0Ys60QiUz5/Wkxzp/24xxqv4Y51NXVeeoTXVr3nMYlS5bgp59+wtKlSyEWi5XGjxgxAj179sTp06elbQ0LZ6qrq+Xiq6qq5BbWEBEREbV1WlU0xsXFYevWrXjvvfcQEBDQ5O26dOmC4uJi6eeGy9INl6kfJ5FIYGVl9ezJEhERET1HtKZoTExMxOrVqxEWFoZZs2aptO3NmzdlFsw0PI4nOztbJu7u3bvIz89X+rgeIiIiorZGK4rGAwcO4JNPPsHYsWMRGRnZaFxhYaFc2969e3Hjxg0MGzZM2tanTx/07NkTO3bsQF1dnbR927Zt0NXVxZgxY9R7AERERERaTuMLYb766isAQF5eHgAgNTUVZ8+ehZmZGaZPn46srCwsWrQInTp1gqenJ/bs2SOz/dChQ/HCCy8AACZPngwnJyf069cPJiYmyMrKwvfffw87OzvMmDFDZrtFixZh3rx5mDVrFvz9/XHlyhUkJiYiODgY9vb2LXDkRERERNpD40XjqlWrZD4nJSUBAGxsbMZv7y0AACAASURBVDB9+nTk5uaipqYGhYWFeP/99+W237Jli7Ro9PPzw3/+8x+cOHEClZWVsLKywrRp0/DOO+/A1FR2mbmXlxfi4uIQFxeHJUuWwMLCAvPmzcPbb7/dTEdKREREpL00XjRevnz5if3jx4/H+PHjmzRWRESE9LE6TeHt7Q1vb+8mxxMRERG1VRovGonaAl09fVhZmSmNs+7SDVmZfCMRERG1PiwaiVpAfV0NAhZ+rzRu3/LAFsiGiIhIdVqxepqIiIiINItFIxEREREpxaKRiIiIiJRi0UhERERESrFoJCIiIiKlWDQSERERkVIsGomIiIhIKRaNRERERKQUi0YiIiIiUkqlN8I8ePAAhYWF6NWrl7Tt5s2b2LRpE4qKihAYGIiXX35Z7UkSERERkWapVDR++umnuH79Onbv3g0AKCsrw7Rp01BQUAAAOHjwIDZv3oyBAweqP1MiIiIi0hiVLk9fuHABI0aMkH4+cOAACgoKEB8fjxMnTqBXr1745ptv1J4kEREREWmWSkXj/fv3YW1tLf184sQJODs7Y/jw4RCJRAgKCsKlS5fUniQRERERaZZKRWO7du1QVVUl/fzzzz/LXIo2NTVFUVGR+rIjIiIiolZBpaLRzs4Ohw8fhiAISE9PR3FxMTw9PaX9+fn56Nixo9qTJCIiIiLNUmkhzLRp0xAZGYmBAweisrIS3bt3lykaf/31V4jFYrUnSURERESapVLRGBgYCABIT0+HiYkJ5s6dC319fQCPHsdTUlKCKVOmqD9LIiIiItIolYpG4FHh2FA8Ps7c3BzJyclqSYqIiIiIWheV7mkcPXo00tPTG+3/4YcfMHr06GdOioiIiIhaF5WKxj/++APl5eWN9ldUVOD27dvPnBQRERERtS5qfff0vXv3YGRkpM4hiYiIiKgVUHpP4y+//IKMjAzp5yNHjuD333+XiysuLsaBAwfQt2/fJu+8oKAAW7ZsQWZmJrKzs1FeXo4tW7Zg0KBBcrHp6emIi4tDbm4uLC0tMXHiRMydOxft2skewsOHD/H555/jyJEjqKyshIuLC6KiohTm1dQxiYiIiNo6pdVRRkYG4uLiAAA6OjpIS0tDWlqawlhbW1tERUU1eefXrl3D+vXrYWtrC7FYjPPnzyuMO378OMLDwzF48GBER0fjypUrWLNmDR48eIDo6GhpXH19PebMmYMrV65g5syZMDc3x9atWxESEoLk5GT06NFD5TGJiIiIqAlF44wZMxAUFARBEODt7Y33339fbrGLjo4OOnTogE6dOqm0cycnJ5w5cwbm5uY4evQowsPDFcbFxMSgX79+2LBhA/T09AAAxsbGiI+PR0hICOzs7AAAhw4dwvnz57FmzRp4e3sDAPz8/ODj44O4uDjExMSoPCYRERERNeGeRlNTU9jY2KBbt27YsmULXn31VdjY2Mj8dO3aVeWCEQBMTExgbm7+xJjc3Fzk5uYiODhYWtwBwNSpU1FfXy9z1vPw4cOwsrKSKWotLCzg5+eHo0ePoqamRuUxiYiIiEjFhTAvvfQSLC0tmysXhS5dugQAcHZ2lmnv3LkzrK2tpf0AkJOTAycnJ+jo6MjE9u/fH2VlZbhx44bKYxIRERHRUzzc+/bt29ixYweuX7+OoqIiCIIg06+jo4PNmzerLUGJRAIAEIlEcn0ikQgFBQUysYMHD5aLs7KyAvBo4U2vXr1UGrOpLC1NVN6GWi+RyLRN7rs14/ei3Th/2o9zqP2edQ5VKhqPHz+Od955BzU1NU91D+PTqKysBAAYGBjI9RkaGqKiokImVlFcQ1vDWKqM2VT375eivl5QHkhaQSIp0ch+dfX05c6UK2LdpRuyMtvOGXGRyFRjc0LPjvOn/TiH2q9hDnV1dZ76RJdKRePy5cthbm6ONWvWoH///k+1Q1U1PPexurparq+qqkrmuZBGRkYK4xraGmJVGZOoJdXX1SBg4fdK4/Ytl3+VJxERUXNS6Z7Gq1evYsaMGS1WMAJ/XkJuuKT8OIlEIr303BCr6NJyQ1tDrCpjEhEREZGKRaOFhQX09fWbKxeFGh7KnZ2dLdN+9+5d5Ofnyzy029HRERcvXpS7zzIrKwsdOnSQPqdRlTGJiIiISMWicdy4cS3+OJo+ffqgZ8+e2LFjB+rq6qTt27Ztg66uLsaMGSNt8/X1RUFBAdLT06VthYWFOHToEEaPHi0teFUZk4iIiIhUvKcxKCgIGRkZmDdvHkJDQ9GtWzeZ5xw26Nq1a5PH/OqrrwAAeXl5AIDU1FScPXsWZmZmmD59OgBg0aJFmDdvHmbNmgV/f39cuXIFiYmJCA4Ohr29vXQsHx8fuLm5YdGiRdI3wmzbtg319fV49913Zfbb1DGJiIiISMWi0c/PDzo6OhAEAf/5z38ajcvJyWnymKtWrZL5nJSUBACwsbGRFo1eXl6Ii4tDXFwclixZAgsLC8ybNw9vv/22zLZ6enqIj49HTEwMEhISUFVVhf79++Ozzz6Dra2tTGxTxyQiIiIiFYvG8PDwJj0ORBWXL19uUpy3t7f01YBP0rFjR3z66af49NNP1TYmERERUVunUtH410u8RERERNQ2qLQQhoiIiIjaJpXONP7yyy9Nihs4cOBTJUNERERErZNKRWNISEiT7mlUZSEMEREREbV+KhWNS5culWurra3FzZs3kZycjG7duiE4OFhtyRERERFR66DycxobM2vWrCf2ExEREZH2UttCmI4dO2LSpEn45ptv1DUkEREREbUSal09bWZmhps3b6pzSCIiIiJqBdRWNFZVVWHPnj144YUX1DUkEREREbUSKt3TGBUVpbC9uLgYFy5cQGFhIRYtWqSWxKh1cHHth/w7tzSdBhEREWmYSkVjSkqKwvaOHTvC3t4eUVFRGDt2rFoSo9Yh/84tBCz8vkmx+5YHNnM2REREpCkqFY2//fZbc+VBRERERK0YXyNIREREREqpdKaxQWlpKU6dOiVdKd29e3cMGTIEJiYmak2OiIiIiFoHlYvGXbt2YdmyZSgvL4cgCAAAHR0ddOjQAZGRkZg0aZLakyQiIiIizVKpaExPT0d0dDS6d++O+fPno0+fPgCA//3vf/juu+/w4YcfwtLSEqNGjWqWZImIiIhIM1QqGr/55hv06tULO3fuhLGxsbTd09MT48ePR3BwMNavX8+ikYiIiOg5o9JCmN9++w1BQUEyBWMDExMTBAYGcoU1ERER0XNIraundXR01DkcEREREbUSKhWNYrEYKSkpKC8vl+srKytDSkoKHB0d1ZYcEREREbUOKt3T+Oabb+Kdd95BUFAQQkND0atXLwBAbm4uEhIScOPGDaxevbpZEiUiIiIizVGpaPT29kZ0dDRiY2OxZMkS6eVoQRDQvn17REdHw9vbu1kSJSIiIiLNUfk5jdOmTcPYsWNx8uRJ3Lp1C8Cjh3sPHToUpqamak+QiOTp6unDyspMaZx1l27IyrzUAhkREdHz7qneCGNmZgY/Pz9159KoyMhIpKSkNNr/448/onPnzggJCcHPP/8s1+/v748VK1bItFVXV2PVqlVITU3Fw4cP4ejoiIiICHh6eqo9fyJ1q6+rQcDC75XG7Vse2ALZEBFRW6C0aKyrq8OKFStgY2ODKVOmNBq3detW5OfnIyIiQu2rqIODg+WKOUEQ8NFHH8HGxgadO3eWtnft2hULFiyQibWxsZEbMzIyEmlpaQgNDYWtrS1SUlIwe/ZsJCQkwN3dXa35ExEREWk7pUXjnj17sGHDBuzateuJcS4uLliyZAn69OmDsWPHqi1BAHB3d5cr5H799VdUVFTI7cvMzAzjxo174nhZWVnYv38/oqKiEBYWBgAIDAxEQEAAYmNjkZiYqNb8iYiIiLSd0kfuHDx4EEOGDIGzs/MT45ydnTFs2DDs379fbck9yb59+6Cjo4OAgAC5vtraWpSVlTW67aFDh6Cvry/znmxDQ0NMnDgRZ8+eRUFBQbPkTERERKStlBaNFy9ebPJ9foMGDUJ2dvYzJ6VMTU0NDh48CHd3d3Tr1k2mLy8vD25ubvDw8MCwYcOwdu1a1NfXy8Tk5OTA3t5e7s02Li4uEAQBOTk5zX4MRERERNpE6eXp4uJiWFpaNmkwCwsLFBUVPXNSyvz0008oKiqSuzTdvXt3DBo0CGKxGKWlpdi3bx9WrFiB27dv4+OPP5bGSSQSmfsgG4hEIgB4qjONlpYmKm9DrZdI9Pw8CeB5OZbn5TjaKs6f9uMcar9nnUOlRaOxsTEePHjQpMGKiooUvpda3fbt2wd9fX25Fdz//ve/ZT4HBQVh/vz52LlzJ8LCwtCzZ08AQGVlJfT19eXGNTQ0BABUVVWpnNP9+6WorxdU3o5aJ4mkRNMpqM3zcCwikelzcRxtFedP+3EOtV/DHOrq6jz1iS6ll6d79+6NkydPNmmwkydPonfv3k+VSFOVlZUhPT0dw4YNg7m5udL4mTNnQhAEZGRkSNuMjIxQU1MjF9tQLDYUj0RERET0iNKi8ZVXXsGpU6dw9OjRJ8alp6fj1KlTGDNmjNqSU+To0aMKV003xtraGsCjy+wNRCKRwkvQEokEAGBlZaWGTImIiIieH0qLxsmTJ6NHjx5YsGABVqxYIX0LTINbt25hxYoVWLBgAezs7DB58uRmSxYA9u7diw4dOmDUqFFNir958yaAR/dbNnB0dMS1a9fkVlhnZmZK+4mIiIjoT0rvaTQyMkJ8fDzeeustrFu3DvHx8TAxMYGxsTHKyspQWloKQRBgb2+PdevWNeul3cLCQpw+fRqvvvoq2rdvL9NXWloKAwMDGBgYSNvq6uqwbt066OrqyqwA9/X1xcaNG7Fr1y7pcxqrq6uRnJwMDw8PhYtkiLQRXzdIRETq0qTXCNra2iI1NRU7d+7E4cOH8b///Q/37t2DsbExXnzxRYwZMwaTJk2CkZFRsyZ74MAB1NbWKrw0ffHiRfztb39DQEAAevTogfLychw8eBDZ2dmYPXs2unfvLo11dXWFr68vYmNjIZFI0KNHD6SkpOD27dtYunRpsx4DUUvi6waJiEhdmvzuaUNDQ4SEhCAkJKQ583mivXv3wtLSEkOGDJHr69q1Kzw8PJCWloZ79+5BV1cXffr0wbJlyxAUFCQXHxMTg5UrVyI1NRXFxcUQi8WIj4/HgAEDWuJQiIiIiLRKk4vG1mDHjh2N9nXv3h1ffvllk8cyNDTE4sWLsXjxYnWkRkRERPRcU7oQhoiIiIiIRSMRERERKcWikYiIiIiUYtFIREREREqxaCQiIiIipVg0EhEREZFSLBqJiIiISCkWjURERESkFItGIiIiIlKKRSMRERERKcWikYiIiIiUYtFIREREREqxaCQiIiIipVg0EhEREZFS7TSdAGmGi2s/5N+5pek0iIiISEuwaGyj8u/cQsDC75XG7Vse2ALZEBERUWvHy9NEREREpBSLRiIiIiJSikUjERERESnFexqJCLp6+rCyMlMaZ92lG7IyL7VARkRE1NqwaCQi1NfVcGEUERE9ES9PExEREZFSWlE0ZmRkQCwWK/zJy8uTiT137hymTJkCV1dXDB06FJ988gkqKirkxqyursbnn3+OYcOGwcXFBa+//jpOnz7dUodEREREpFW06vL0jBkz4OTkJNPWuXNn6b/n5OQgLCwMvXv3RmRkJPLz87Fx40bcunULa9euldkuMjISaWlpCA0Nha2tLVJSUjB79mwkJCTA3d29RY6HiIiISFtoVdH40ksvwdvbu9H+5cuXo1OnTkhISICxsTEAoFu3bvjggw9w+vRpeHp6AgCysrKwf/9+REVFISwsDAAQGBiIgIAAxMbGIjExsdmPhYiIiEibaMXl6ceVlpaitrZWYfupU6cQGBgoLRgBYNy4cejQoQMOHjwobTt06BD09fUxadIkaZuhoSEmTpyIs2fPoqCgoHkPgkhLNayyVvbj4tpP06kSEZGaadWZxvfeew/l5eVo164dBg0ahMWLF0MsFgMALl++jNraWjg7O8tsY2BggL59+yInJ0falpOTA3t7e5niEgBcXFwgCAJycnJgZWXV/AdEpGW4ypqIqO3SiqJRX18fPj4+GD58OMzNzXH58mVs3LgRU6dOxe7du2Fvbw+JRAIAEIlEctuLRCJcuHBB+lkikcjcC/l4HACeaSQiIiL6C60oGj08PODh4SH9PHr0aIwaNQoTJkxAXFwcvvjiC1RWVgJ4dGbxrwwNDaX9AFBZWQl9fX2FcQBQVVWlco6WliYqb0Otl0hkqukUtJ66v0POiXbj/Gk/zqH2e9Y51IqiURFHR0d4enrizJkzAAAjIyMAjx6l81dVVVXS/obYmpoahXHAn8WjKu7fL0V9vaDydtQ6SSQlmk5B66nzOxSJTDknWozzp/04h9qvYQ51dXWe+kSX1i2EeVyXLl1QXFwM4M9Lyw2XqR8nkUhk7lEUiUQKL0E3bMv7GYmIiIhkaXXRePPmTZibmwMAHBwc0K5dO2RnZ8vEVFdXIycnB3379pW2OTo64tq1aygrK5OJzczMlPYTERER0Z+0omgsLCyUa/v111+RkZGBYcOGAQBMTU3h6emJ1NRUmWIwNTUV5eXl8PX1lbb5+vqipqYGu3btkrZVV1cjOTkZHh4eChfJEBEREbVlWnFP44IFC9C+fXu4u7vD3Nwc//vf/7Bjxw6Ym5vj3XfflcZFRERg8uTJCAkJwaRJk5Cfn49vv/0Ww4cPx5AhQ6Rxrq6u8PX1RWxsLCQSCXr06IGUlBTcvn0bS5cu1cQhEhEREbVqWlE0ent7Y+/evfj2229RWloKCwsLBAQE4N1330XXrl2lcU5OTvj2228RGxuLpUuXwsTEBK+//joWLlwoN2ZMTAxWrlyJ1NRUFBcXQywWIz4+HgMGDGjJQyMiIiLSClpRNIaGhiI0NLRJsS+++CK2b9+uNM7Q0BCLFy/G4sWLnzU9IiIioueeVtzTSERERESapRVnGqnpXFz7If/OLU2nQURERM8ZFo3Pmfw7t/huYCIiIlI7Fo1EpHa6evqwsjJrUqx1l27IyrzUzBkREdGzYtFIRGpXX1fTpDPeAM96ExFpCy6EISIiIiKlWDQSERERkVIsGomIiIhIKRaNRERERKQUi0YiIiIiUopFIxEREREpxaKRiIiIiJRi0UhERERESrFoJCIiIiKlWDQSERERkVIsGomIiIhIKb57mog0SldPH1ZWZkrjrLt0Q1bmpRbIiIiIFGHRSEQaVV9Xg4CF3yuN27c8sAWyISKixvDyNBEREREpxaKRiIiIiJRi0UhERERESrFoJCIiIiKlWDQSERERkVJasXo6KysLKSkpyMjIwO3bt9GpUye4u7tjwYIFsLW1lcaFhITg559/ltve398fK1askGmrrq7GqlWrkJqaiocPH8LR0RERERHw9PRs9uMhIiIi0jZaUTR+8803OHfuHHx9fSEWiyGRSJCYmIjAwEDs3r0bvXr1ksZ27doVCxYskNnexsZGbszIyEikpaUhNDQUtra2SElJwezZs5GQkAB3d/dmPyYiIiIibaIVRWNYWBhiY2NhYGAgbfP398fYsWOxfv16LFu2TNpuZmaGcePGPXG8rKws7N+/H1FRUQgLCwMABAYGIiAgALGxsUhMTGyW4yAiIiLSVlpxT6OHh4dMwQgAdnZ26NOnD/Ly8uTia2trUVZW1uh4hw4dgr6+PiZNmiRtMzQ0xMSJE3H27FkUFBSoL3kiIiKi54BWnGlURBAE3Lt3D46OjjLteXl5cHNzQ01NDUQiEaZPn445c+ZAV/fP+jgnJwf29vYwNjaW2dbFxQWCICAnJwdWVlYtchxEpF4urv2Qf+eW0ji+lpCISDVaWzTu2bMHd+/eRUREhLSte/fuGDRoEMRiMUpLS7Fv3z6sWLECt2/fxscffyyNk0gk6Ny5s9yYIpEIAHimkUiL5d+5xdcSEhE1A60sGvPy8vDxxx9jwIABMvcv/vvf/5aJCwoKwvz587Fz506EhYWhZ8+eAIDKykro6+vLjWtoaAgAqKqqUjknS0sTlbeh1kskMtV0CqSAuueF89xy+F1rP86h9nvWOdS6olEikeCtt95Cx44dsWrVKpnLzorMnDkThw4dQkZGhrRoNDIyQk1NjVxsQ7HYUDyq4v79UtTXCypvR62TRFKi6RRIAXXPC+e5ZYhEpvyutRznUPs1zKGurs5Tn+jSqqKxpKQEs2fPRklJCbZt2ya9nPwk1tbWAIDi4mJpm0gkUngJWiKRAADvZyQiIiL6C61YPQ08Ogs4d+5cXL9+HevWrZOeNVTm5s2bAAALCwtpm6OjI65duya3wjozM1PaT0RERER/0oqisa6uDgsWLMCFCxewatUquLm5ycWUlpaiurpabrt169ZBV1dX5k0vvr6+qKmpwa5du6Rt1dXVSE5OhoeHh8JFMkT0fNHV04eVlZnSHxfXfppOlYioVdCKy9PLli3DsWPH4OXlhaKiIqSmpkr7jI2N4e3tjYsXL+Jvf/sbAgIC0KNHD5SXl+PgwYPIzs7G7Nmz0b17d+k2rq6u8PX1RWxsLCQSCXr06IGUlBTcvn0bS5cu1cQhElELq6+r4SprIiIVaEXR+NtvvwEAfvjhB/zwww8yfTY2NvD29kbXrl3h4eGBtLQ03Lt3D7q6uujTpw+WLVuGoKAguTFjYmKwcuVKpKamori4GGKxGPHx8RgwYECLHBMRERGRNtGKojEhIUFpTPfu3fHll182eUxDQ0MsXrwYixcvfpbUiIiIiNoErSgaiYg0peHeR2X4hhkiet6xaCQiegLe+0hE9AiLRiIiNeAZSSJ63rFoJCJSA56RJKLnnVY8p5GIiIiINItFIxEREREpxaKRiIiIiJRi0UhERERESrFoJCIiIiKlWDQSERERkVIsGomIiIhIKRaNRERERKQUH+5NRNSCmvrmGIBvjyGi1oVFIxFRC2rqm2MA4MCqSXw1IRG1GiwaiYhaKb6akIhaExaNRERarqmXvHlGkoieBYtGIiItxzOSRNQSuHqaiIiIiJTimUYiojaCl7GJ6FmwaCQiaiM0dRnbplsP3P7jptI4FqtErRuLRiIikqHuM5K3/7jJey6JngMsGomISIamzkjywedErRuLRiIiahVUefA5z0oStbw2XTRWV1dj1apVSE1NxcOHD+Ho6IiIiAh4enpqOjUiInoCLuohanltumiMjIxEWloaQkNDYWtri5SUFMyePRsJCQlwd3fXdHpERNSIpp6V1NSrGF1c+yH/zq0W3y9Rc2qzRWNWVhb279+PqKgohIWFAQACAwMREBCA2NhYJCYmajZBIiJ6ZpoqLvPv3OLiH3rutNmi8dChQ9DX18ekSZOkbYaGhpg4cSJWrFiBgoICWFlZaTBDIiJqKeouLoke97yceW6zRWNOTg7s7e1hbGws0+7i4gJBEJCTk6NS0airq6PuFJ+Kra0trMzbt3icJvfdHHFNmc+2+t1o8s+NOueltcdpQ46ampPmGLOpcd27dcXoN+OVxqV/M6dJ49n37I2BA/srjWunb4jamiqlcQAgsuqCQwePKI3z9XsFkoI7att3a48Dmv7dqJuhgR5mLjmsNC79mznNWk/o6uo80/g6giAIasxHawQEBKBz587YsGGDTHtubi5effVVfPLJJzJnIYmIiIjasjb77unKykro6+vLtRsaGgIAqqqa9lsLERERUVvQZotGIyMj1NTUyLU3FIsNxSMRERERteGiUSQSoaCgQK5dIpEAABfBEBERET2mzRaNjo6OuHbtGsrKymTaMzMzpf1ERERE9EibLRp9fX1RU1ODXbt2Sduqq6uRnJwMDw8PdO7cWYPZEREREbUubfaRO66urvD19UVsbCwkEgl69OiBlJQU3L59G0uXLtV0ekREREStSpt95A7waNHLypUrsXfvXhQXF0MsFmPhwoUYMmSIplMjIiIialXadNFIRERERE3TZu9pJCIiIqKmY9FIREREREqxaCStVlBQgNjYWISEhMDd3R1isRgZGRkKY9PT0xEUFIT+/ftj5MiRiIuLQ21tbQtnTI/LysrCv/71L/j7+8PNzQ0jR45EREQEfv/9d7nYc+fOYcqUKXB1dcXQoUPxySefoKKiQgNZ0+P++9//Ijw8HF5eXnBxccHQoUMxa9YsnDt3Ti6Wc6gd1q9fD7FYjHHjxsn1cQ5bp4yMDIjFYoU/eXl5MrHPModtdvU0PR+uXbuG9evXw9bWFmKxGOfPn1cYd/z4cYSHh2Pw4MGIjo7GlStXsGbNGjx48ADR0dEtnDU1+Oabb3Du3Dn4+vpCLBZDIpEgMTERgYGB2L17N3r16gUAyMnJQVhYGHr37o3IyEjk5+dj48aNuHXrFtauXavho2jbbt68ibq6OkyaNAkikQglJSXYu3cvpk+fjvXr12Po0KEAOIfaQiKR4Ouvv0aHDh3k+jiHrd+MGTPg5OQk0/b4IwSfeQ4FIi1WUlIiFBYWCoIgCEeOHBEcHByEM2fOyMX5+/sLQUFBQm1trbRt+fLlgqOjo3Dt2rWWSpf+4uzZs0JVVZVM27Vr1wRnZ2dh8eLF0rY333xTePnll4XS0lJp286dOwUHBwfh1KlTLZYvNU15ebkwZMgQYc6cOdI2zqF2WLx4sRASEiJMnz5deO2112T6OIet15kzZwQHBwfhyJEjT4x71jnk5WnSaiYmJjA3N39iTG5uLnJzcxEcHAw9PT1p+9SpU1FfX4+0tLTmTpMa4eHhAQMDA5k2Ozs79OnTR3pJpbS0FKdOnUJgYCCMjY2lcePGjUOHDh1w8ODBFs2ZlGvfvj0sLCzw8OFDAJxDbZGVlYU9e/YgKipKro9zqD1KS0sV3nqljjlk0UjPvUuXLgEAnJ2dZdo7d+4Ma2traT+1DoIg4N69e9JfBi5fvoza2lq5P+3JVwAAEM1JREFU+TMwMEDfvn2Rk5OjiTTpL0pLS1FYWIirV69i+fLluHLlCjw9PQFwDrWBIAhYsmQJAgMD0bdvX7l+zqF2eO+99zBgwAC4urpi5syZuHz5srRPHXPIexrpuSeRSAAAIpFIrk8kEqGgoKClU6In2LNnD+7evYuIiAgAyufvwoULLZofKfb+++/j8OHDAAB9fX1MnjwZc+fOBcA51Abff/89cnNzsWbNGoX9nMPWTV9fHz4+Phg+fDjMzc1x+fJlbNy4EVOnTsXu3bthb2+vljlk0UjPvcrKSgCQuwwKAIaGhlz514rk5eXh448/xoABA6QrN5XNX0M/aVZ4eDiCg4ORn5+P1NRUVFdXo6amBgYGBpzDVq60tBRffPEF5syZAysrK4UxnMPWzcPDAx4eHtLPo0ePxqhRozBhwgTExcXhiy++UMsc8vI0PfeMjIwAANXV1XJ9VVVV0n7SLIlEgrfeegsdO3bEqlWroKv76H9PnD/tIBaLMXToUEyYMAEbNmzAxYsXpffGcQ5bt6+//hr6+vp44403Go3hHGofR0dHeHp64syZMwDUM4csGum513AqvuHU/OMk/9/evQdFVf4PHH+DiAqCeCEvoKLJLiHKMppoYpOgIHjjoiNe0GJiBh0dCnPELMOxIsrLGOKIFBqmgglEpBNeJ03GW4oFkiaKgiOCmIjchfP9wx/7c9s1SkWwPq+ZnfE8z+ec8znnkZ0P53l2KS195G/W4tmpqKggJCSEiooKvvjiC53pExm/50/79u3x8PBg37591NTUyBi2YSUlJXz11VfMmjWLW7duUVRURFFREbW1tdTX11NUVER5ebmM4XOqd+/elJeXA0/nvVSKRvGv17SoOycnR6f95s2bFBcXG1z0LZ6d2tpaQkNDKSgoIC4ujoEDB+r0q1QqTExM9Mavrq6OvLw8Gb82qqamBkVRqKyslDFsw8rKyqivr2f16tV4eHhoX+fOnSM/Px8PDw/i4+NlDJ9ThYWF2g8VPo0xlKJR/OvZ29szcOBAkpOTaWho0Lbv3LkTY2NjPD09WzG7/7aGhgbeeustsrOzWb9+PRqNRi/GwsKCUaNGkZ6eTmVlpbY9PT2dqqoqJkyY8CxTFn9y+/ZtvbZ79+6RmZlJ79696d69u4xhG2Zra0tsbKzey97eHhsbG2JjY/H19ZUxbOMM/RyePn2aEydO4ObmBjyd91IjRVGUp5e2EM/exo0bgQcfovj+++8JCAjA1tYWS0tL5syZA8Dhw4eZP38+I0eOxMfHh4sXL7J9+3ZmzJhBZGRkK2b/3/bRRx+RmJjI2LFj8fb21ukzNzdn3LhxAOTm5hIYGIi9vT3Tp0+nuLiYLVu24OrqSnx8fGukLv7P3Llz6dChAy4uLlhbW3Pjxg1SU1MpLi5m7dq1+Pj4ADKGz5ugoCDu3r1Lenq6tk3GsO2aO3cunTp1wsXFha5du/L777+TnJyMhYUFu3fvpk+fPsCTj6EUjeK5p1arDbbb2Nhw6NAh7faBAwfYsGED+fn5dOvWjYCAABYsWICJiXyJQGsJCgri5MmTBvv+PH6nT59m9erVnD9/ns6dO+Pj40N4eLjBP3cmnp3du3eTnp7OpUuXuHv3LhYWFmg0GoKDgxkxYoROrIzh88NQ0Qgyhm1VYmIiGRkZXLt2jXv37tGtWzfc3NxYtGiRtmBs8iRjKEWjEEIIIYRolqxpFEIIIYQQzZKiUQghhBBCNEuKRiGEEEII0SwpGoUQQgghRLOkaBRCCCGEEM2SolEIIYQQQjRLikYhhBBCCNEsKRqFEEK0eampqajVak6cONHaqQjxnyV/CkMI8UwUFhayefNmTp06xY0bNzA1NaVHjx4MHToUPz8/Ro4c2dop/isFBQWRk5PD2bNnWzuVZuXl5XHgwAH8/PywtbVt7XSEEH8iRaMQosX9+uuvBAUFYWJigq+vL4MGDaKmpoarV69y7NgxzM3NpWgU5OXlsWHDBkaMGCFFoxBtkBSNQogWFxsbS3V1Nenp6Tg4OOj1l5aWtkJWQggh/glZ0yiEaHEFBQVYWVkZLBgBrK2t9dqysrIIDg5m+PDhDBkyhMmTJ7Nz506D++/atYsJEybg5OTE+PHj2bp1KykpKXpr4CIiIlCr1QaPoVariYiI0Gvfu3cvM2fOxMXFBWdnZ6ZPn84PP/zwyP3Pnj3LnDlz0Gg0uLq6snz5ciorK/XiS0tL+fDDD/Hw8MDJyYlRo0bxxhtvcOzYMZ24goIClixZgpubG05OTri7uxMdHU1VVZXB63hciqKwY8cO/P39cXZ2xsXFhaCgII4fP64TV1RUhFqtJiYmhsOHDxMQEMCQIUNwc3MjOjqa+/fv6x07MzOTKVOmMGTIEF577TU2bNhAVlYWarWa1NRUAGJiYli2bBkAc+fORa1WGxyTxsZGvvzyS8aNG4eTkxNeXl6kpaU91XshhDBMnjQKIVpcv379uHLlCvv27cPT07PZ+OTkZD744AM0Gg2hoaF06tSJrKwsIiMjuXbtGkuXLtXGbt26laioKBwcHAgPD6e6upqEhAS6d+/+xHmvW7eOTZs2MWbMGMLCwjA2Nmb//v2EhYWxYsUKZs+erROfl5dHaGgo/v7+TJo0iZMnT7J7926MjY1ZtWqVNq6oqIiZM2dSVlbG1KlTcXJyorq6mnPnzpGVlcXo0aMByMnJYd68eVhaWjJjxgx69uzJb7/9xrZt2zh79izbtm2jffv2T3ydAEuWLGHPnj14eXnh7+9PXV0dGRkZBAcHExMTg4eHh078jz/+yI4dOwgMDCQgIICDBw+SkJBAly5dCA0N1cbt3buX8PBw+vXrx8KFC2nXrh3ffvsthw4d0jne+PHjKS0tJTk5mdDQUAYOHAg8+L/zsHXr1lFTU8OMGTMwNTVl586dRERE0K9fP4YNG/ZU7oUQ4hEUIYRoYWfOnFEGDx6sqFQqxdPTU4mIiFC2b9+uXLp0SS/25s2bipOTkxIeHq7Xt2rVKsXBwUG5du2aoiiKUl5erjg7Oyve3t5KVVWVNu7GjRuKRqNRVCqVcvz4cW370qVLFZVKZTBHlUqlLF26VLudk5OjqFQqZc2aNXqx8+fPV1xcXJSKigqd/dVqtZKdna0TGxISojg6Oir37t3Ttr355puKSqVSjhw5onfshoYG7b8nT56seHl56ZxHURRl3759ikqlUlJSUgxey8PmzJmjaDSav4xpOl5SUpJOe319veLn56eMHTtWaWxsVBRFUQoLCxWVSqU4OzsrhYWF2tjGxkZl4sSJyujRo3X2d3NzU0aNGqXcuXNH237v3j3F3d1d7xpSUlL0xuzPfVOnTlVqa2u17cXFxcrgwYOVt99+u9l7IYR4MjI9LYRocS4uLqSkpODn50dFRQWpqamsXLkSHx8fZs+eTWFhoTY2MzOTuro6pk2bxu3bt3Ve7u7uNDY2kpWVBcBPP/1EdXU1s2fPplOnTtpj9OrVi8mTJz9RzhkZGRgZGeHr62swj8rKSrKzs3X20Wg0ODs767SNHDmS+/fvc/36dQDu3LnD0aNHGTNmDGPGjNE7r7Hxg7flCxcucOHCBSZNmkRdXZ3O+YcNG4aZmZneVPbj+u677zA3N2fcuHE657l79y7u7u5cv36dgoICnX08PDx0PqxiZGSEq6srpaWl2un43NxcSkpK8PPzo0uXLtpYc3NzAgMDHyvXWbNmYWpqqt3u2bMnAwYM0MtPCPH0yfS0EOKZUKvVfPLJJwBcv36dU6dO8c0333D69GkWLFhASkoKpqam5OfnA/D6668/8li3bt0CHkzzAtqpzIe9+OKLT5Rvfn4+iqLg7e3dbB5N+vbtqxdjZWUFPCgWAa5du4aiKDg6OjZ7fniw1i8mJuZvnf9x5efnU1lZySuvvPLImLKyMgYMGKDdbu5azc3NtePz8H5NDLX9HY86b1NRLoRoOVI0CiGeORsbG2xsbJg6dSqzZs3izJkz/PLLLwwfPhxFUQCIjo7mhRdeMLi/ocLh7zAyMjLYbujDG4qiYGRkRHx8PO3atTO436BBg3S2HxXXdLzHERwcbPCJJIClpeVjHfPPFEWhW7durFmz5pEx9vb2Otstca1/R9OTWCHEsydFoxCi1RgZGeHs7MyZM2coKSkBwM7ODoCuXbv+5ZMvQDs9evnyZUaNGqXT1/Sk7mFNU6R37tzRPhUDdKbHm9jZ2XH06FH69OnzxE8tH9avXz+MjIzIy8v7y7j+/fsDD4qk5u7Dk+rfvz8FBQU4Oztjbm7+1I5rY2MDwJUrV/T6DLU9qqgXQrQN8iubEKLFHTt2zODTvJqaGu26vKbCzNvbG1NTU2JiYqipqdHbp6Kigrq6OgBGjx5Nx44d2b59O9XV1dqY4uJiMjIy9PZtKkib1kQ22bJli17slClTAFi7di0NDQ16/Y87NWxlZcWrr77KkSNH9PKA/39K5+joiEqlIikpyWBRe//+fe2U95Py9fWlsbGRtWvXGux/3Gt1cnLC2tqatLQ0ysvLte2VlZUkJSXpxZuZmQHoxAoh2g550iiEaHFRUVHcuXMHd3d3VCoVHTt21BZ2BQUF+Pr6ar8/sVevXkRGRvLee+/h4+PDlClTsLGx4fbt21y8eJEDBw6wZ88ebG1t6dKlC2FhYURHRxMYGIivry/V1dUkJSVhZ2fH+fPndfKYNGkS69atY8WKFVy+fBkrKyuOHj3KH3/8oZfz0KFDWbRoETExMfj6+uLl5UXPnj0pKSkhNzeXI0eOkJOT81j34/333+f8+fOEhITg6+vL4MGDqa2t5dy5c9jY2LBkyRKMjIz49NNPmTdvHlOmTCEgIEDnL+ns37+f8PBw/P39mz1ffX09GzduNNjn6enJhAkT8Pf35+uvvyY3N5exY8fStWtXiouLyc7O5urVqxw8ePAfX6eJiQlLly7lnXfeYfr06UybNo127dqRlpaGlZUVRUVFOk8XhwwZgrGxMZs2baK8vBwzMzNsbW31PlwkhGgdUjQKIVpcREQEBw8e5OeffyYzM5OKigosLCxQqVSEhIToFT4BAQHY2dmRkJBAcnIyFRUVWFlZMWDAAMLCwnS+DDw4OBgzMzO2bNnCmjVr6N27N8HBwVhYWPDuu+/qHLdz585s3ryZqKgo4uLiMDMzw9PTk88++4yXX35ZL++FCxfi5OTEtm3bSExMpKqqiu7du2Nvb8/y5csf+3707duXlJQUYmNjOXLkCOnp6VhaWuLg4MCMGTO0cS+99BJpaWnExcVx6NAhkpKSMDc3x8bGBj8/P70p+Uepr69n/fr1Bvv69+/PoEGDiIqKwtXVlV27dhEXF0d9fT3W1tY4OjqyePHix77WyZMnY2JiwsaNG/n888/p0aMH06ZNQ61Ws3DhQjp06KCN7dOnDx9//DHx8fGsXLmS+vp6/Pz8pGgUoo0wUlpyxbIQQrSS1NRUli1bRmJiIq6urq2djviThIQEoqOjSU5ORqPRtHY6Qoi/QdY0CiGEaDF1dXV6a0IrKyvZvn07VlZWzX71kBCi7ZDpaSGEEC2msLCQkJAQJk6ciK2tLaWlpaSlpVFUVERkZKTOF3ULIdo2KRqFEEK0mG7duqHRaMjIyKCsrAwTExNUKhWLFy/Gx8entdMTQvwDsqZRCCGEEEI0S9Y0CiGEEEKIZknRKIQQQgghmiVFoxBCCCGEaJYUjUIIIYQQollSNAohhBBCiGZJ0SiEEEIIIZr1P9kiRymdj+1+AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 720x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9os6CdRA6Gjs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dataloader      = prepare_data(X_train, y_train, True)\n",
        "dev_dataloader        = prepare_data(X_dev, y_dev, False)\n",
        "prediction_dataloader = prepare_data(X_test, y_test, False)"
      ],
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKCnbnxu6JjU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 719
        },
        "outputId": "3093bae1-7c2d-4bec-9a33-f90e78b83275"
      },
      "source": [
        "\n",
        "adam         = AdamW(model.parameters(), lr=LEARN_RATE, eps=EPSILON)\n",
        "total_steps  = len(train_dataloader) * EPOCHS # Total number of training steps is nb_batches times nb_epochs.\n",
        "linear_sch   = get_linear_schedule_with_warmup(adam, num_warmup_steps=nb_warmup_steps, num_training_steps=total_steps)\n",
        "\n",
        "training_stats = train(model=model, optimizer=adam, train_dataloader=train_dataloader, dev_dataloader=dev_dataloader, \n",
        "                       epochs=EPOCHS, verbose=True, scheduler=linear_sch)"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "  Batch    40  of    189.    Elapsed: 0:00:31.\n",
            "  Batch    80  of    189.    Elapsed: 0:01:01.\n",
            "  Batch   120  of    189.    Elapsed: 0:01:31.\n",
            "  Batch   160  of    189.    Elapsed: 0:02:01.\n",
            "\n",
            "  Average training loss: 1.76\n",
            "  Training epcoh took: 0:02:23\n",
            "  Accuracy: 0.46\n",
            "  Validation Loss: 1.50\n",
            "  Validation took: 0:00:10\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "  Batch    40  of    189.    Elapsed: 0:00:30.\n",
            "  Batch    80  of    189.    Elapsed: 0:01:00.\n",
            "  Batch   120  of    189.    Elapsed: 0:01:30.\n",
            "  Batch   160  of    189.    Elapsed: 0:02:00.\n",
            "\n",
            "  Average training loss: 1.43\n",
            "  Training epcoh took: 0:02:21\n",
            "  Accuracy: 0.47\n",
            "  Validation Loss: 1.45\n",
            "  Validation took: 0:00:10\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "  Batch    40  of    189.    Elapsed: 0:00:30.\n",
            "  Batch    80  of    189.    Elapsed: 0:01:00.\n",
            "  Batch   120  of    189.    Elapsed: 0:01:30.\n",
            "  Batch   160  of    189.    Elapsed: 0:02:00.\n",
            "\n",
            "  Average training loss: 1.32\n",
            "  Training epcoh took: 0:02:21\n",
            "  Accuracy: 0.48\n",
            "  Validation Loss: 1.45\n",
            "  Validation took: 0:00:10\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:07:36 (h:mm:ss)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "irGvNReb6LhO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 583
        },
        "outputId": "bb14bb31-d083-4c4a-da14-33dd89fe122e"
      },
      "source": [
        "plot_loss(training_stats)\n",
        "training_stats"
      ],
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuoAAAGaCAYAAABZrWb6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3gU1f4/8PfMbnonnYQaTAjpVCmK9NBb6IICCihFUUS8lmv56k8polSv4BURSAhJqIberihFIIQuEmogjZCekGR35/dHYGVNgJTdzG54v57nPrpnZ858snCu7z05c0aQJEkCEREREREZFVHuAoiIiIiIqDwGdSIiIiIiI8SgTkRERERkhBjUiYiIiIiMEIM6EREREZERYlAnIiIiIjJCDOpEVGclJyfDz88PixcvrnYfc+bMgZ+fnx6rqrse9Xn7+flhzpw5lepj8eLF8PPzQ3Jyst7ri4uLg5+fH44ePar3vomIDEEpdwFE9PSoSuDdu3cvvL29DViN6SksLMR3332H+Ph4pKeno169emjVqhVef/11+Pj4VKqPGTNmYOfOndi0aRP8/f0rPEaSJHTr1g25ubk4dOgQLC0t9fljGNTRo0dx7NgxvPTSS7C3t5e7nHKSk5PRrVs3jBkzBh999JHc5RCRkWNQJ6JaM3fuXJ3XJ06cwPr16zFixAi0atVK57169erV+HpeXl44ffo0FApFtfv47LPP8Mknn9S4Fn344IMP8Msvv6Bfv35o27YtMjIysG/fPiQmJlY6qEdERGDnzp2IjY3FBx98UOExR44cwa1btzBixAi9hPTTp09DFGvnF7jHjh3DkiVLMHjw4HJBfeDAgejbty/MzMxqpRYioppiUCeiWjNw4ECd12q1GuvXr0doaGi59/4pPz8ftra2VbqeIAiwsLCocp0PM5ZQV1RUhB07dqBTp05YsGCBtn3atGkoKSmpdD+dOnWCp6cntm7ditmzZ8Pc3LzcMXFxcQDKQr0+1PTPQF8UCkWNvrQREdU2rlEnIqPTtWtXjB07FufPn8fEiRPRqlUrDBgwAEBZYF+4cCGGDRuGdu3aITAwED169MD8+fNRVFSk009Fa6Yfbtu/fz+GDh2KoKAgdOrUCV999RVUKpVOHxWtUX/QlpeXh3//+99o3749goKCMHLkSCQmJpb7ebKysvDee++hXbt2CAsLw7hx43D+/HmMHTsWXbt2rdRnIggCBEGo8ItDRWH7UURRxODBg5GdnY19+/aVez8/Px+7du2Cr68vgoODq/R5P0pFa9Q1Gg3+85//oGvXrggKCkK/fv2wZcuWCs9PSkrCxx9/jL59+yIsLAwhISEYMmQINmzYoHPcnDlzsGTJEgBAt27d4Ofnp/Pn/6g16nfv3sUnn3yCzp07IzAwEJ07d8Ynn3yCrKwsneMenH/48GH88MMP6N69OwIDA9GrVy9s3LixUp9FVVy8eBFTp05Fu3btEBQUhD59+mDFihVQq9U6x6WkpOC9995Dly5dEBgYiPbt22PkyJE6NWk0GqxatQr9+/dHWFgYWrZsiV69euFf//oXSktL9V47EekHZ9SJyCjdvn0bL730EsLDw9GzZ08UFhYCANLS0hATE4OePXuiX79+UCqVOHbsGFauXIkLFy7ghx9+qFT/Bw8exLp16zBy5EgMHToUe/fuxX//+184ODhgypQplepj4sSJqFevHqZOnYrs7Gz8+OOPmDRpEvbu3aud/S8pKcH48eNx4cIFDBkyBEFBQfjzzz8xfvx4ODg4VPrzsLS0xKBBgxAbG4tt27ahX79+lT73n4YMGYLly5cjLi4O4eHhOu/98ssvuHfvHoYOHQpAf5/3P/2///f/sHr1arRp0wYvv/wyMjMz8emnn6JBgwbljj127BiOHz+OF154Ad7e3trfLnzwwQe4e/cuJk+eDAAYMWIE8vPzsXv3brz33ntwcnIC8Ph7I/Ly8jBq1Chcv34dQ4cORYsWLXDhwgVERkbiyJEj2LBhQ7nf5CxcuBD37t3DiBEjYG5ujsjISMyZMwcNGzYst4Srus6cOYOxY8dCqVRizJgxcHFxwf79+zF//nxcvHhR+1sVlUqF8ePHIy0tDaNHj0bjxo2Rn5+PP//8E8ePH8fgwYMBAMuXL8eiRYvQpUsXjBw5EgqFAsnJydi3bx9KSkqM5jdHRPQPEhGRTGJjYyVfX18pNjZWp71Lly6Sr6+vFB0dXe6c4uJiqaSkpFz7woULJV9fXykxMVHbdvPmTcnX11datGhRubaQkBDp5s2b2naNRiP17dtX6tixo06/7777ruTr61th27///W+d9vj4eMnX11eKjIzUtq1Zs0by9fWVli1bpnPsg/YuXbqU+1kqkpeXJ7366qtSYGCg1KJFC+mXX36p1HmPMm7cOMnf319KS0vTaR8+fLgUEBAgZWZmSpJU889bkiTJ19dXevfdd7Wvk5KSJD8/P2ncuHGSSqXStp89e1by8/OTfH19df5sCgoKyl1frVZLL774otSyZUud+hYtWlTu/Ace/H07cuSItu3rr7+WfH19pTVr1ugc++DPZ+HCheXOHzhwoFRcXKxtT01NlQICAqSZM2eWu+Y/PfiMPvnkk8ceN2LECMnf31+6cOGCtk2j0UgzZsyQfH19pd9//12SJEm6cOGC5OvrK33//feP7W/QoEFS7969n1gfERkXLn0hIqPk6OiIIUOGlGs3NzfXzv6pVCrk5OTg7t276NChAwBUuPSkIt26ddPZVUYQBLRr1w4ZGRkoKCioVB8vv/yyzutnn30WAHD9+nVt2/79+6FQKDBu3DidY4cNGwY7O7tKXUej0eCNN97AxYsXsX37djz//POYNWsWtm7dqnPchx9+iICAgEqtWY+IiIBarcamTZu0bUlJSTh16hS6du2qvZlXX5/3w/bu3QtJkjB+/HidNeMBAQHo2LFjueOtra21/15cXIysrCxkZ2ejY8eOyM/Px5UrV6pcwwO7d+9GvXr1MGLECJ32ESNGoF69etizZ0+5c0aPHq2z3Mjd3R1NmjTBtWvXql3HwzIzM5GQkICuXbuiefPm2nZBEPDaa69p6wag/Tt09OhRZGZmPrJPW1tbpKWl4fjx43qpkYhqB5e+EJFRatCgwSNv/Fu7di2ioqJw+fJlaDQanfdycnIq3f8/OTo6AgCys7NhY2NT5T4eLLXIzs7WtiUnJ8PNza1cf+bm5vD29kZubu4Tr7N3714cOnQI8+bNg7e3N7799ltMmzYNs2fPhkql0i5v+PPPPxEUFFSpNes9e/aEvb094uLiMGnSJABAbGwsAGiXvTygj8/7YTdv3gQANG3atNx7Pj4+OHTokE5bQUEBlixZgu3btyMlJaXcOZX5DB8lOTkZgYGBUCp1/3OoVCrRuHFjnD9/vtw5j/q7c+vWrWrX8c+aAKBZs2bl3mvatClEUdR+hl5eXpgyZQq+//57dOrUCf7+/nj22WcRHh6O4OBg7XlvvfUWpk6dijFjxsDNzQ1t27bFCy+8gF69elXpHgciql0M6kRklKysrCps//HHH/Hll1+iU6dOGDduHNzc3GBmZoa0tDTMmTMHkiRVqv/H7f5R0z4qe35lPbj5sU2bNgDKQv6SJUvw2muv4b333oNKpULz5s2RmJiIzz//vFJ9WlhYoF+/fli3bh1OnjyJkJAQbNmyBR4eHnjuuee0x+nr866Jt99+GwcOHMDw4cPRpk0bODo6QqFQ4ODBg1i1alW5Lw+GVltbTVbWzJkzERERgQMHDuD48eOIiYnBDz/8gFdeeQXvvPMOACAsLAy7d+/GoUOHcPToURw9ehTbtm3D8uXLsW7dOu2XVCIyLgzqRGRSNm/eDC8vL6xYsUInMP3vf/+TsapH8/LywuHDh1FQUKAzq15aWork5ORKPZTnwc9569YteHp6AigL68uWLcOUKVPw4YcfwsvLC76+vhg0aFCla4uIiMC6desQFxeHnJwcZGRkYMqUKTqfqyE+7wcz0leuXEHDhg113ktKStJ5nZubiwMHDmDgwIH49NNPdd77/fffy/UtCEKVa7l69SpUKpXOrLpKpcK1a9cqnD03tAdLsi5fvlzuvStXrkCj0ZSrq0GDBhg7dizGjh2L4uJiTJw4EStXrsSECRPg7OwMALCxsUGvXr3Qq1cvAGW/Kfn0008RExODV155xcA/FRFVh3FNCxARPYEoihAEQWcmV6VSYcWKFTJW9Whdu3aFWq3G6tWrddqjo6ORl5dXqT46d+4MoGy3kYfXn1tYWODrr7+Gvb09kpOT0atXr3JLOB4nICAA/v7+iI+Px9q1ayEIQrm90w3xeXft2hWCIODHH3/U2Wrw3Llz5cL3gy8H/5y5T09PL7c9I/D3evbKLsnp3r077t69W66v6Oho3L17F927d69UP/rk7OyMsLAw7N+/H5cuXdK2S5KE77//HgDQo0cPAGW71vxze0ULCwvtsqIHn8Pdu3fLXScgIEDnGCIyPpxRJyKTEh4ejgULFuDVV19Fjx49kJ+fj23btlUpoNamYcOGISoqCt988w1u3Lih3Z5xx44daNSoUbl92yvSsWNHREREICYmBn379sXAgQPh4eGBmzdvYvPmzQDKQtfSpUvh4+OD3r17V7q+iIgIfPbZZ/j111/Rtm3bcjO1hvi8fXx8MGbMGKxZswYvvfQSevbsiczMTKxduxbNmzfXWRdua2uLjh07YsuWLbC0tERQUBBu3bqF9evXw9vbW+d+AAAICQkBAMyfPx/9+/eHhYUFnnnmGfj6+lZYyyuvvIIdO3bg008/xfnz5+Hv748LFy4gJiYGTZo0MdhM89mzZ7Fs2bJy7UqlEpMmTcL777+PsWPHYsyYMRg9ejRcXV2xf/9+HDp0CP369UP79u0BlC2L+vDDD9GzZ080adIENjY2OHv2LGJiYhASEqIN7H369EFoaCiCg4Ph5uaGjIwMREdHw8zMDH379jXIz0hENWec/2UjInqEiRMnQpIkxMTE4PPPP4erqyt69+6NoUOHok+fPnKXV465uTl++uknzJ07F3v37sX27dsRHByMVatW4f3338e9e/cq1c/nn3+Otm3bIioqCj/88ANKS0vh5eWF8PBwTJgwAebm5hgxYgTeeecd2NnZoVOnTpXqt3///pg7dy6Ki4vL3UQKGO7zfv/99+Hi4oLo6GjMnTsXjRs3xkcffYTr16+Xu4Fz3rx5WLBgAfbt24eNGzeicePGmDlzJpRKJd577z2dY1u1aoVZs2YhKioKH374IVQqFaZNm/bIoG5nZ4fIyEgsWrQI+/btQ1xcHJydnTFy5EhMnz69yk/DrazExMQKd8wxNzfHpEmTEBQUhKioKCxatAiRkZEoLCxEgwYNMGvWLEyYMEF7vJ+fH3r06IFjx45h69at0Gg08PT0xOTJk3WOmzBhAg4ePIiff/4ZeXl5cHZ2RkhICCZPnqyzswwRGRdBqo07gYiISIdarcazzz6L4ODgaj80iIiI6jauUSciMrCKZs2joqKQm5tb4b7hREREAJe+EBEZ3AcffICSkhKEhYXB3NwcCQkJ2LZtGxo1aoThw4fLXR4RERkpLn0hIjKwTZs2Ye3atbh27RoKCwvh7OyMzp0744033oCLi4vc5RERkZFiUCciIiIiMkJco05EREREZIQY1ImIiIiIjBBvJr0vK6sAGo3+VgE5O9siMzNfb/0R0d84vogMh+OLyDBEUYCTk02VzmFQv0+jkfQa1B/0SUSGwfFFZDgcX0TGgUtfiIiIiIiMEIM6EREREZERYlAnIiIiIjJCDOpEREREREaIQZ2IiIiIyAhx1xciIiKixygqKkB+fg7U6lK5SyEjpVCYwdbWAVZWVdt+8UkY1ImIiIgeobS0BHl5WXB0dIGZmQUEQZC7JDIykiShtLQY2dl3oFSawczMXG99c+kLERER0SPk5WXD1tYB5uaWDOlUIUEQYG5uCRsbB+TnZ+u1bwZ1IiIiokdQqUpgYWEldxlkAiwtrVBaWqLXPrn0Rc8On0tF3MEk3M0tRj17Cwzp7IP2AR5yl0VERETVoNGoIYoKucsgEyCKCmg0ar32yaCuR4fPpeKn7RdRotIAADJzi/HT9osAwLBORERkorjkhSrDEH9PuPRFj+IOJmlD+gMlKg3iDibJVBERERERmSoGdT3KzC2uUjsRERFRXTRt2iRMmzap1s+ta7j0RY+c7S0qDOWOthYyVENERESkq1On1pU6bsOGLfD0rG/gauhJGNT1aEhnH5016g+oNRpk5RXDyY6BnYiIiOTz4Yef6ryOjo5EWloKpk9/S6fd0dGpRtdZuHCpLOfWNQzqevTghtGHd33pGOSJnX/cxNzIBMwZHQYHzq4TERGRTHr16qPz+sCBvcjJyS7X/k/37t2DpaVlpa9jZmZWrfpqem5dw6CuZ+0DPNA+wAOurnbIyMgDALRoXA8LoxMxNzIB745uCXsb/T2xioiIiEifpk2bhPz8fMye/S8sXrwQf/55EWPGjMPEiZPx668HsGXLRly69Cdyc3Pg6uqGPn36Y+zY8VAoFDp9AMCSJd8DAE6ePI4ZM6bg88/n4urVK9i0KRa5uTkICgrBO+/8C97eDfRyLgDExkYjKmotMjPvwMfHB9OmzcSKFct1+jQVDOq1wLeBI94cFoyF0YmYH5WAd0aFwc6aYZ2IiOhp9OCZK5m5xXA20meuZGdnYfbsmejZMxzh4X3h7l5WX3z8NlhZWWPEiDGwtrbCiRPHsXLldygoKMDUqW88sd+ffvoBoqjA6NHjkJeXi8jIn/HJJx9gxYqf9HLuxo0xWLhwLkJDW2LEiFFISUnBe+/Ngp2dHVxd3ar/gciEQb2W+DV0woyIYHwbcxoLok5h1qgw2FrxVztERERPE1N55sqdOxmYM+dD9Os3UKf944//DxYWfy+BGTQoAvPmfYGNGzfg1Vdfg7n54yciVSoV/vvfn6BUlkVQe3sHfPvtfFy5chlNmzar0bmlpaVYuXI5AgKC8M03y7THNWv2DD7//GMGdXq8Fo3rYfqQICyKPY0F60/hnZGhsLZkWCciIjIlv51JwaHTKdU6N+l2DlRqSaetRKXBj/EX8L9Tt6vUV6dgT3QM8qxWHU9iaWmJ8PC+5dofDumFhQUoKSlFSEgYNm+Ow/Xr1/DMM76P7bdv3wHaAA0AISGhAIDbt289Mag/6dyLF88jJycHr78+WOe4Hj3CsWjR14/t21gxqNeywKbOmDo4CEvizuDr6ES8PSIUVhb8YyAiInoa/DOkP6ldLq6ubjph94ErV5KwYsVynDz5BwoKCnTeKyjIf2K/D5bQPGBnZw8AyMvLq/G5qallX57+uWZdqVTC09MwX2gMjQlRBiHNXPD6oEAs23QWC6MTMXN4CMM6ERGRiegYVP2Z7HeW/VbhM1ec7S3w7piWNS1Nbx6eOX8gLy8P06dPgrW1LSZOnAIvL2+Ym5vj0qWLWL58MTQaTQU96RJFRYXtkvTkLyo1OddU8cmkMgnzdcXkAQG4cjsX38acRnGJWu6SiIiIyMCGdPaBuVI3fpkrRQzp7CNTRZWXkHACOTk5eP/9f2P48FHo2PE5tGnTTjuzLTcPj7IvT8nJN3XaVSoVUlKqt1RJbgzqMmrd3A2TBrTAX8nZWBR7GsWlDOtERER1WfsAD7zUuzmc7cueq+Jsb4GXejc3qhtJH0UUy2LjwzPYpaWl2Lhxg1wl6WjevAUcHBywZctGqFQqbfvu3TuQl5crY2XVx/UWMmvr7w61WsLKbeexJPY0ZkQEw0xZ8a92iIiIyPQ9eOaKqQkKCoadnT0+//xjRESMgCAI2LkzHsay8sTMzAwTJkzCwoXz8Oabr6NLl25ISUnB9u1b4eXlDUEQ5C6xyjijbgTaB3pgfB9/nLuWhSVxZ1GqevIaLyIiIqLa5ODgiLlzF8LZ2QUrVixHZOQatG7dDq+/PkPu0rSGDh2BN9+chdTUFCxd+i0SExPw5Zdfw9bWDubmpvd0eEGqyyvwqyAzMx8ajf4+ioefTFpZB0/dwk87/kRoMxe8PjgQSgW/RxFVpDrji4gqh+NLV2rqdXh4NJK7DKoBjUaDfv16oHPnLnj33Q8Meq3H/X0RRQHOzrZV6o9J0Ih0DvXCiz19ceryHXy3+RxUas6sExEREVVWcXH5HXV27PgFubk5CAtrJUNFNcM16kama0tvqDUSIvf8hRVbz2PSgBZQiPw+RURERPQkp0+fwvLli/HCC11hb++AS5cu4pdftqBpUx906dJd7vKqjEHdCPVo3QBqtYTo/ZehEAW80q8FRNH0boAgIiIiqk3163vBxcUVMTHrkZubA3t7B4SH98WUKdNgZmZ6T4NnUDdS4e0aQq3RIPbgFShEAeP7+kM0wbuViYiIiGqLl5c35s5dKHcZesOgbsT6tm8MtVrCpkNXIYoCXurdnGGdiIiI6CnBoG7kBnRqApVGwrbfr0GhEDG2p69J7gNKRERERFXDoG4CBj/XBGqNBtuP3IBCFDC6+zMM60RERER1HIO6CRAEARGdfaBWS9j1x00oRAEjujZjWCciIiKqwxjUTYQglIVzteZ+WFeUhXeGdSIiIqK6iUHdhAhC2bIXtUa6vwxGxJDnm8pdFhEREREZAIO6iREEAS/29IVGo8G2369BqRAwoGMTucsiIiIiIj3jIy9NkCgIGBfeHB0DPbDp16v45fA1uUsiIiKip1B8/FZ06tQaKSm3tW0REf3x+ecfV+vcmjp58jg6dWqNkyeP661POTGomyhREDC+jz+eDXBH7MEr2HH0htwlERERkZGbPXsmunfvhKKiokce89Zb09CrV2cUFxfXYmVVs2fPTkRHr5O7DIPj0hcTJooCJvb1h1otIXr/ZShEAT3aNJC7LCIiIjJSPXr0wu+//4pDhw6iR4/wcu9nZd3FiRN/oGfP3rCwsKjWNdati4UoGnYueO/eXfjrr0sYPny0TntoaEvs3fsbzMzMDHr92sIZdROnEEW82r8FWvm6InLvX9h3MlnukoiIiMhIPffcC7CyssaePTsrfH/fvj1Qq9Xo2bN8iK8sc3NzKJXyzAWLoggLCwuDf1GoLZxRrwOUChGTBwZg2cazWLPrEkRRwAuhXnKXRUREREbG0tISzz3XGfv370Fubi7s7e113t+zZyecnZ3RoEEjzJ//JU6cOIa0tDRYWlqiZcvWmDr1DXh61n/sNSIi+iMsrBXef/9jbduVK0n45pt5OHv2DBwcHDBw4BC4uLiWO/fXXw9gy5aNuHTpT+Tm5sDV1Q19+vTH2LHjoVAoAADTpk3CqVMnAQCdOrUGAHh4eCImZitOnjyOGTOmYNGi79CyZWttv3v37sKaNatw/fo1WFvboGPH5/DaazPg6OioPWbatEnIz8/HRx99iq+/nosLF87Bzs4ew4aNxJgxL1Xtg9YTBvU6QqkQ8dqgQCzdeAard/wJhSjgueDHDyQiIiKqfcdST2JL0g5kFWfDycIRA3zC0dajZa1dv0ePcOzatR0HDuzFgAGDte2pqSk4e/Y0IiJG4sKFczh79jS6d+8FV1c3pKTcxqZNsZg+fTLWrNkAS0vLSl8vM/MOZsyYAo1GgxdffAmWllbYsmVjhUtr4uO3wcrKGiNGjIG1tRVOnDiOlSu/Q0FBAaZOfQMA8NJLE1BUVIS0tBRMn/4WAMDKyvqR14+P34ovvvgEAQFBeO21GUhPT0Ns7HpcuHAOK1as1qkjNzcHb789A126dEO3bj2xf/8eLF++GE2bNkP79h0r/TPrC4N6HWKmFDF1cCAWxZ7BqviLUIoi2gd6yF0WERER3Xcs9STWXYxFqaYUAJBVnI11F2MBoNbCeps27eDo6IQ9e3bqBPU9e3ZCkiT06NELPj7N0KVLd53zOnZ8HlOmjMeBA3sRHt630tdbu/Yn5ORkY+XKn+Hn1xwA0Lt3P4waNbjcsR9//H+wsPj7S8CgQRGYN+8LbNy4Aa+++hrMzc3Rps2ziIvbgJycbPTq1eex11apVFi+fDGaNfPF4sX/gbm5OQDAz685Pv74fWzduhERESO1x6enp+Hf//4/7fr9fv0GIiKiH375ZTODOtWcmVKB6UOC8G3Maaz85TxEUUC7Fu5yl0VERFRnHE05gcMpf1Tr3Ks5N6CSVDptpZpSrL0Qg99vH6tSX+0926CdZ6sq16BUKtG1a3ds2hSLO3fuwMXFBQCwZ88ueHs3QIsWgTrHq1QqFBTkw9u7AWxt7XDp0sUqBfXDh39DUFCINqQDgJOTE3r06I2NGzfoHPtwSC8sLEBJSSlCQsKweXMcrl+/hmee8a3Sz3rx4nlkZd3VhvwHunbtgaVLv8Xvv/+mE9RtbW3RvXsv7WszMzP4+wfg9u1bVbquvjCo10HmZgrMGBqMhRsSsWLreShEAa2bu8ldFhER0VPvnyH9Se2G0qNHOOLiNmDfvl0YPnw0rl27isuXL2H8+FcBAMXF9/Dzz6sQH78VGRnpkCRJe25+fn6VrpWWloqgoJBy7Q0bNirXduVKElasWI6TJ/9AQUGBznsFBVW7LlC2nKeia4miCG/vBkhLS9Fpd3NzhyAIOm12dvZISrpc5Wvrg6xBPT09HatXr0ZiYiLOnj2LwsJCrF69Gu3atXviuX5+fo98r0OHDvjxxx/1WarJsTBX4I2IYCyMTsR/tpyDQhQQ5lv+pg0iIiKqmnaerao1kw0AH/z2BbKKs8u1O1k44s2WU2paWqUFBYXA09MLu3fvwPDho7F79w4A0C75WLhwHuLjt2LYsFEIDAyCra0tAAEff/wvndCuT3l5eZg+fRKsrW0xceIUeHl5w9zcHJcuXcTy5Yuh0WgMct2HiaKiwnZD/cxPImtQv3r1KlasWIFGjRrBz88PCQkJlT537ty55drOnj2L1atXo2PH2l9DZIysLJSYOTwEC9afwrJNZzFtSBBCmrnIXRYREdFTa4BPuM4adQAwE80wwKf62yFWV/fuPfHzzz8iOfkm9u7dBT8/f+3M84N16NOnz9QeX1xcXOXZdABwd/dAcvLNcu03blzXeZ2QcAI5OTn4/PN5CA39e71+xU8uFSpoK8Nd6KUAACAASURBVM/Dw1N7rYf7lCQJyck30aSJT6X6kYusm0wGBATgyJEj2LVrF1555ZUqnTtw4MBy/yssLIQgCOjXr5+BKjY9VhZKvDU8BA3cbLF04xmcvZIpd0lERERPrbYeLTG6+VA4WZRtC+hk4YjRzYfW6q4vD/Ts2RsAsGTJQiQn39TZO72imeXY2PVQq9VVvk779h1x5kwi/vzzorYtKysLu3dv1znuwd7nD89el5aWllvHDgBWVlaV+tLQvHkLODnVw6ZNMSgt/fvL0f79e5GRkY4OHYx7clfWGfWyX6PoR0lJCXbt2oU2bdrAw4M7nTzM2tIMb40IxfzIBCyOO4M3IoLRonE9ucsiIiJ6KrX1aClLMP+nJk2aolkzXxw69D+Ioohu3f6+ibJDh07YuTMeNja2aNy4Cc6dO4Pjx4/BwcGhytcZPfol7NwZj7femoqIiJGwsLDEli0b4e7uifz8v7THBQUFw87OHp9//jEiIkZAEATs3BmPilad+Pk1x65d27F48ddo3rwFrKys0anT8+WOUyqVeO216fjii08wffpkdO/eE+npaYiJWY+mTX3Qv3/5nWeMSd14bBOAgwcPIjc3FwMGDJC7FKNka2WGt0eGwt3JCotiTuPi9Sy5SyIiIiKZPZhFDwtrpd39BQDeeGMWevXqg927t2PJkm9w584dfPPN0sfuV/4oLi4uWLToP2jSxAc//7wKGzZEIjy8D4YNG6lznIODI+bOXQhnZxesWLEckZFr0Lp1O7z++oxyfQ4cOBS9evVGfPw2fPLJB/jmm3mPvH6fPv3x8cefo7j4HpYu/Rbx8VvRo0c4vv32uwr3cjcmgiTX6vh/2LNnD6ZOnVrpm0n/acaMGdi/fz9+++23ck/ZqozMzHxoNPr7KFxd7ZCRkae3/vQlt6AEcyMTkJlzDzOHh8C3geOTTyIyMsY6vojqAo4vXamp1+HhUX53EqKKPO7viygKcHau2mqSOrE9Y35+Pg4cOIDOnTtXK6QDqPIHVxmurnZ677OmXF2BL6d2wnvLfsO3MYn4dFIHNOcyGDJBxji+iOoKjq+/paeLUCrrzAIEMjBRFPU6fupEUN+5cyeKi4vRv3//avfxtMyoP/DW8BB8te4kPvr+d8waGYYmntX7gkMkB2MfX0SmjONLl0ajgUpl+G0BqW7QaDSPHD/VmVGvE18Rt27dCjs7O3Tp0kXuUkyGk50FZo8Kg42lGRZEncL1VP6fMhEREZExMfmgnp6ejqNHj6Jnz546j4alJ6tnb4nZo8NgZaHA/KgE3EhjWCciIiIyFiYR1G/cuIEbN25U+F58fDw0Gk2Nlr08zVwcrPDO6JYwN1NgftQpJGdU/UEGRERERKR/sq9RX7ZsGQAgKSkJALB582acOHEC9vb2ePHFFwEAL7/8MgBg37595c7fsmUL3NzcqrVTDJVxc7TC7FFh+HLdScyPTMC7Y1rC09lG7rKIiIiInmqyB/Vvv/1W53VsbCwAwMvLSxvUH+XKlSs4d+4cxo8fr32aFVWPez1rzB4Vhq/WJWBuZALmjG4J93pV3yuViIiIiPTDaPZRl9vTtuvLo9y6U4C5605CqRDx7ugwuDkxrJPxMdXxRWQKOL50paZeh7t7QwiCIHcpZOQkSUJa2g297qPOaWjS4eVig3dGhqFUpcG8yATcyS6SuyQiIiLZKBRKlJaWyF0GmYDS0hIoFPpdrMKgTuV4u9ni7RGhKCpWa59iSkRE9DSytXVEdnYGSkqKwUUIVBFJklBSUozs7AzY2ur3ie9c+nIfl76UdzUlF/OjTsHOygzvjmkJJzsLuUsiAlA3xheRseL4Kq+oqAD5+dlQq1Vyl0JGSqFQwtbWEVZWj96MozpLXxjU72NQr1jS7RwsiDoFB1sLvDs6DI62DOskv7oyvoiMEccXkWFwjTrpnU99B8wcHoLsvGLMi0xATgHX6RERERHVBgZ1eqJnvB3x5rBgZObcw/yoBOQVMqwTERERGRqDOlWKX0MnvBERjPSsIsyPOoX8olK5SyIiIiKq0xjUqdL8G9fD9KFBSMksxIKoUyi8x7BOREREZCgM6lQlgU2cMW1IIG7dyceC9YkovMc74ImIiIgMgUGdqizYxwWvDwrCjbQ8LNxwCkXFDOtERERE+sagTtUS+owLpgwMwNXbefh2QyKKS9Ryl0RERERUpzCoU7W18nPDpAEt8NetHHwbk4jiUoZ1IiIiIn1hUKcaaevvjlf6tcCfN7KxOPY0SlUM60RERET6wKBONdY+wAMT+vrjwrUsLI47g1KVRu6SiIiIiEwegzrpRccgT7zUuznOXrmLZRvPQKVmWCciIiKqCQZ10pvnQ+pjbC8/JCZl4rvN5xjWiYiIiGqAQZ30qkuYF0Z3fwYnL2Xg+63nodYwrBMRERFVh1LuAqju6d66AdQaCev3XYZCFPBqvxYQRUHusoiIiIhMCoM6GUSvtg2h1kiIOZAEhShgQh9/hnUiIiKiKmBQJ4Pp82wjqNUabPz1KkRRwMu9m0MUGNaJiIiIKoNBnQyqf8cmUGskbPntGpSigLG9/CAwrBMRERE9EYM6GdzATmVh/ZfD1yGKAsb08GVYJyIiInoCBnUyOEEQMOT5plCpNdh57CYUooiR3ZoxrBMRERE9BoM61QpBEDC8SzOoNRJ2H78JpUJAxAs+DOtEREREj8CgTrVGEASM6vYM1BoJ24/egEIhYPBzTRnWiYiIiCrAoE61ShDK1qir1RK2/X4dClHEwE5N5C6LiIiIyOgwqFOtEwUB48L9oNZosPnQVShEAf06NJa7LCIiIiKjwqBOshAFAeN7+0OjkRD3vytQKAT0btdI7rKIiIiIjAaDOslGFAVM6OsPtUbChv1JUIgierZpIHdZREREREaBQZ1kpRBFvNq/BdQaCVF7/4JCFNCtlbfcZRERERHJTpS7ACKFKGLygACEPeOCtbsv4cCpW3KXRERERCQ7BnUyCkqFiCkDAxHs44zVO/7Er4m35S6JiIiISFYM6mQ0zJQipg4ORECTeli1/SJ+O5Mid0lEREREsmFQJ6NiplRg+pAgNG/khP/GX8CR86lyl0REREQkCwZ1MjrmZgrMiAiGr7cjVm69gD8upstdEhEREVGtY1Ano2RhpsAbw4Lh42WP77ecw8lLGXKXRERERFSrGNTJaFmaK/HmsBA09rDD8k1nceryHblLIiIiIqo1DOpk1KwslJg5PBQN3GyxbOMZnLmSKXdJRERERLWCQZ2MnrWlEm+PDEV9Fxssjj2Dc9fuyl0SERERkcExqJNJsLE0w6yRYfCoZ43FMadx8XqW3CURERERGRSDOpkMWyszzBoVCldHK3wTk4hLN7PlLomIiIjIYBjUyaTYW5tj1qgwONtbYuGGRFxOzpG7JCIiIiKDYFAnk+NgY45ZI8PgaGOOr6NP4crtXLlLIiIiItI7BnUySU52FnhnVBjsrM2wYP0pXE/Nk7skIiIiIr1iUCeTVc/eEu+MCoO1hRLzoxJwI41hnYiIiOoOBnUyaS4OVpg9OgwW5grMjzqF5Ix8uUsiIiIi0gsGdTJ5ro5WeGdUGJQKAfMjE3D7ToHcJRERERHVGIM61QnuTtZ4Z1QYBEHAvMgEpGQyrBMREZFpY1CnOsPT2QazRoVBI0mYF5mAtKxCuUsiIiIiqjYGdapTvFxs8M6oMKjUZWE9I7tI7pKIiIiIqoVBneocb1dbzBoZiuISNeZFJiAz557cJRERERFVGYM61UkN3e3w9shQFNxTYW7kSdzNZVgnIiIi08KgTnVWYw97vD0iFHmFpZgXmYCsvGK5SyIiIiKqNAZ1qtOa1rfHW8NDkV1QgvlRCcgpKJG7JCIiIqJKkTWop6enY/78+Rg7dizCwsLg5+eHo0ePVvp8jUaDNWvWoH///ggODsazzz6LiRMn4saNGwasmkxNM28HzBwWgszce5gfmYDcQoZ1IiIiMn6yBvWrV69ixYoVSEtLg5+fX5XPnz17NubPn4927drhww8/xOTJk2Fvb4/s7GwDVEumzLeBI96ICEFGdhHmR55CflGp3CURERERPZZSzosHBATgyJEjcHJywp49ezB16tRKn7tt2zbs2LEDa9euRUhIiAGrpLrCv5ETpkcE49sNpzE/KgHvjAqDjaWZ3GURERERVUjWGXVbW1s4OTlV69yffvoJ3bt3R0hICFQqFYqKuF82PVlA43qYPjQIt+8U4Ov1p1B4TyV3SUREREQVMsmbSfPz83HmzBn4+fnho48+QlhYGEJDQ9GvXz8cOnRI7vLIyAU1dcbrg4JwIy0fC6NPoaiYYZ2IiIiMj0kG9Rs3bkCSJKxatQpHjhzBxx9/jK+++goAMHnyZJw+fVrmCsnYhT7jgikDA3E1JQ/fbEjEvRKGdSIiIjIusq5Rr67CwkIAQEFBATZt2gRPT08AwHPPPYfu3bvjP//5D5YuXVqlPp2dbfVep6urnd77JP0Jd7WDrZ0F5q05geWbz+OjV9rB0twkh8RTieOLyHA4voiMg0mmEgsLCwBAy5YttSEdAJydndGhQwecPHmyyn1mZuZDo5H0VqOrqx0yMvL01h8Zhl99e7zSzx8rtp7HR9/9jjcigmFuppC7LHoCji8iw+H4IjIMURSqPDFskktf3NzcAAAuLi7l3nN2dkZubm5tl0Qm7NkWHpjQxx8Xr2dhSdwZlKrUcpdEREREZJpB3d3dHS4uLkhLSyv3XlpaWrV3kqGnV8cgT7zcuznOXr2LpRvPQqXWyF0SERERPeVMIqjfuHGj3NNGw8PDkZCQgKSkJG1bcnIyfvvtN3To0KG2S6Q64LmQ+hjXyw+nkzKxfBPDOhEREclLkCRJfwuzq2HZsmUAgKSkJGzbtg1Dhw6Ft7c37O3t8eKLLwIAunbtCgDYt2+f9rz09HQMHjwYgiBg7NixUCgUWLNmDfLy8hAXF4dGjRpVqQ6uUacH9p5Ixtrdl9DazxWTBwZAIZrE99mnCscXkeFwfBEZRnXWqMse1P38/Cps9/Ly0gbzioI6AFy7dg1ffvkljh07BkmS0LJlS8yePfuRfT4Ogzo9bNcfNxG19y+09XfDq/1bMKwbGY4vIsPh+CIyDJMM6saCQZ3+afvR69iwPwntA9wxsW8LiKIgd0l0H8cXkeFwfBEZRnWCukluz0hUG3q3awSVWsLG/12BQhTxcp/mEAWGdSIiIqodDOpEj9G/Q2Oo1Rps+e0aFAoBY3v5MawTERFRrWBQJ3qCgZ2aQK2R8Mvh6xBFAS/28IXAsE5EREQGxqBO9ASCIGDI802h1kjYcfQGFKKAUd2eYVgnIiIig2JQJ6oEQRAw7AUfqNUSdh+/CaUoYlgXH4Z1IiIiMhgGdaJKEgQBI7s1g1qjwY5jN6BQlM20M6wTERGRITCoE1WBIAgY3cNXu2ZdIQoY9FxTucsiIiKiOohBnaiKRKFs9xe1Rrq/G4yI/h0ay10WERER1TEM6kTVIAoCXg5vDvX9fdaVooDezzaSuywiIiKqQxjUiapJFAVM7OsPjSRhw4EkKEQBPds2lLssIiIiqiMY1IlqQBQFvNLPH2q1BlH7LkOhENGtlbfcZREREVEdIMpdAJGpU4giJg0IQNgzLli7+xIOJNySuyQiIiKqAxjUifRAqRDx2qBAhPg4Y/XOP/G/xNtyl0REREQmjkGdSE+UChGvDw5CYNN6+Gn7Rfx2JkXukoiIiMiEMagT6ZGZUsS0wUHwb+yE//5yAUfOpcpdEhEREZkovQR1lUqFnTt3Ijo6GhkZGfrokshkmZspMH1oMPwaOmLFtvP442K63CURERGRCaryri9z587F0aNHERsbCwCQJAnjx4/H8ePHIUkSHB0dER0djYYNuU0dPb0szBSYERGMhdGJ+M/mcxAFAa38XOUui4iIiExIlWfUf/31V7Ru3Vr7et++ffjjjz8wceJELFiwAADw/fff669CIhNlaa7Em8NC0KS+Hb7bfBan/rojd0lERERkQqoc1FNTU9Go0d9PYNy/fz+8vb0xa9Ys9O3bFyNHjsThw4f1WiSRqbKyUGLmsFA0dLfDsk1ncDopU+6SiIiIyERUOaiXlpZCqfx7xczRo0fRoUMH7esGDRpwnTrRQ6wtlXh7RAi8XGyxJO4Mzl5lWCciIqInq3JQ9/DwQEJCAgDgr7/+ws2bN9GmTRvt+5mZmbC2ttZfhUR1gLWlGd4eGQpPZ2ssjj2DC9fuyl0SERERGbkqB/W+ffti06ZNmDx5MiZPngxbW1t07txZ+/6FCxd4IylRBWytysK6m6MVvo09jT9vZMldEhERERmxKgf1yZMnY/DgwTh16hQEQcBXX30Fe3t7AEBeXh727duH9u3b671QorrA3tocs0aFwdneEt9sOI3LyTlyl0RERERGSpAkSdJXZxqNBgUFBbC0tISZmZm+uq0VmZn50Gj09lHA1dUOGRl5euuP6pbs/GJ8tS4BOfnFeHtkKHzqO8hdkknh+CIyHI4vIsMQRQHOzrZVO0efBahUKtjZ2ZlcSCeqbY62Fpg9Kgz21ub4en0irqbkyl0SERERGZkqB/WDBw9i8eLFOm1r165Fy5YtERoairfffhulpaV6K5CornKys8A7o8JgY6nE1+tP4XoqZ7CIiIjob1UO6j/88AOuXLmifZ2UlIQvvvgCbm5u6NChA+Lj47F27Vq9FklUVzk7WGL2qDBYmCuwYP0pJKfny10SERERGYkqB/UrV64gMDBQ+zo+Ph4WFhaIiYnBypUr0adPH2zatEmvRRLVZS6OVpg9KgxmShHzohJw606B3CURERGREahyUM/JyYGTk5P29e+//45nn30WtrZli+Pbtm2L5ORk/VVI9BRwc7LGO6PCIIoC5kUmICWTYZ2IiOhpV+Wg7uTkhNu3bwMA8vPzcebMGbRu3Vr7vkqlglqt1l+FRE8Jj3rWmD0qDJAkzItMQFpWodwlERERkYyqHNRDQ0MRFRWFHTt24IsvvoBarcbzzz+vff/69etwc3PTa5FETwtPZxvMGhUGlVrC3HUJSM8ukrskIiIikkmVg/qMGTOg0Wjw5ptvIi4uDoMGDUKzZs0AAJIkYc+ePWjZsqXeCyV6Wni72mLWyFCUlKoxb10C7uQwrBMRET2NqvXAo+zsbJw8eRJ2dnZo06aNtj0nJwebNm1Cu3bt0Lx5c70Wamh84BEZm+upeZgXmQAbKyXeHd0S9ewt5S7JaHB8ERkOxxeRYVTngUd6fTKpKWNQJ2N0NSUX86MSYGdtjndHt4STnYXcJRkFji8iw+H4IjKMWg3qN27cwN69e3Hz5k0AQIMGDdCtWzc0bNiwOt3JjkGdjNXlWzlYsP4UnGwt8O7oMDjYMqxzfBEZDscXkWHUWlD/5ptvsGLFinK7u4iiiMmTJ+ONN96oapeyY1AnY3bpZja+jj4FF4eyPdftbczlLklWHF9EhsPxRWQY1QnqVb6ZNCYmBt999x2Cg4OxdOlS7Nq1C7t27cLSpUsRGhqK7777DnFxcVXtlogew7eBI96MCMGd7CLMj0pAflGp3CURERGRgVV5Rn3IkCEwMzPD2rVroVQqdd5TqVQYM2YMSktLTS6sc0adTMG5a3exKOY0PJ3LHpBkY2kmd0my4PgiMhyOLyLDqJUZ9aSkJPTp06dcSAcApVKJPn36ICkpqardElElBDSuh+lDgnD7TgEWRJ1C4T3OrBMREdVVVQ7qZmZmKCx89BMTCwoKYGb2dM7yEdWGwKbOmDo4CDfT8/F1dCKKilVyl0REREQGUOWgHhQUhPXr1+POnTvl3svMzER0dDRCQkL0UhwRVSykmQteHxSI66l5WLghEfdKGNaJiIjqmiqvUf/jjz/w8ssvw8bGBkOHDtU+lfTy5cuIi4tDQUEBVq1ahdatWxukYEPhGnUyRccvpuO7zefQzNsBM4eFwMJcIXdJtYLji8hwOL6IDKPWtmfct28fPvvsM6SkpOi0169fHx999BFeeOGFqnYpOwZ1MlVHz6fh+63n0LyhE96ICIa5Wd0P6xxfRIbD8UVkGLX6wCONRoOzZ88iOTkZQNkDjwICAhAdHY3Vq1cjPj6+Ot3KhkGdTNnhs6lYue08WjSphxlDg2CmrNthneOLyHA4vogMozpBvfzWLZW+mIjg4GAEBwfrtGdlZeHq1avV7ZaIqqF9oAdUGg1+jL+IpRvPYurgIJgpq3wLChERERkR/pecqI54Lrg+Xgr3w+mkTCzfdBYqtUbukoiIiKgGGNSJ6pDOoV54sacvTl2+g/9sPsewTkREZMIY1InqmK4tvTGq2zM4cSkDK7edh1rDsE5ERGSKqr1GnYiMV482DaDWSIjefxmiKOCVvi0gioLcZREREVEVVCqo//jjj5Xu8OTJk9Uuhoj0J7xdQ6g1GsQevAKFKGB8H3+IAsM6ERGRqahUUP/qq6+q1KnAMEBkFPq2bwy1WsKmQ1ehEAWMC2/OsE5ERGQiKhXUV69ebeg6iMhABnRqApVGwrbfr0Ehinixpy+/TBMREZmASgX1tm3bGroOIjKgwc81gVqtwfajN6AQBYzq/gzDOhERkZGT9WbS9PR0rF69GomJiTh79iwKCwuxevVqtGvX7onnzpkzBxs3bizXHhISgujoaEOUS2SyBEFAxAs+UGsk7PrjJhQKAcO7NGNYJyIiMmKyBvWrV69ixYoVaNSoEfz8/JCQkFCl862srPDJJ5/otNWrV0+fJRLVGYIgYETXZlBrJOw8dhMKUcTQzk0Z1omIiIyUrEE9ICAAR44cgZOTE/bs2YOpU6dW6XylUomBAwcaqLrqOZZ6EluSdiC7OBuOFo4Y4BOOth4t5S6LCEBZWB/d/RmoNRLij1yHQhQw+PmmcpdFREREFZA1qNva2ta4D7VajaKiIr30VVPHUk9i3cVYlGpKAQBZxdlYdzEWABjWyWgIgoAXe/pCrdZg6+/XoFAIGNCxidxlERER0T+Y9AOPCgoK0KpVKxQVFcHR0RGDBg3CW2+9BQsLC1nq2ZK0QxvSHyjVlCLm0ha4W7vCw8YdFgpzWWojepgoCHipd3OoNRI2/XoVSoWIPs82krssIiIieojJBnVXV1e88sor8Pf3h0ajwf79+7Fq1SokJSVh5cqVstSUVZxdYXuBqhBzjy8GADhb1kN9W3d42njA08Yd9W084G7tCjOFWW2WSgRREDChjz80GgkxB5KgEAX0attQ7rKIiIjoPpMN6m+//bbO6379+sHd3R0//PADfvvtN3Ts2LFK/Tk713zpjIt1PdwpvFuu3cnSARNbjcSNnNtIzrmNmzm3cT7zT6glDYCypQietm5o4FAfDRw8y/5pXx8edm5Qiooa10X0OHNebov5a09g/b7LsLe3xIDnfOQuqVJcXe3kLoGozuL4IjIOJhvUKzJhwgT88MMPOHz4cJWDemZmPjQaqUbX79u4p84adQAwE80woGlvNLHwQRM3H8CtrF2lUSG98A5SClKRUpCG2wVpuJp5E8eST0FCWR0KQQF3a9eymXdbD+0svItVPYiCWKNaiR42rqcviopKsWLTWdwrLEGXlt5yl/RYrq52yMjIk7sMojqJ44vIMERRqPLEcJ0K6i4uLjAzM0NOTo4s139ww2hldn1RikrUt/VAfVsPnfYSdSnSCtNxO78swKcUpOJa7g2cSE/UHmMmmsHDxg317wf3B0HeycKRW+1RtSgVIiYPDMCyjWfx865LUChEPB9SX+6yiIiInmp1KqinpqaitLRU1r3U23q0RFuPltWekTBXmKGBnRca2HnptN9T3UNKQbo2vN/OT8XFu3/haOoJ7TGWCov7wd3j/gx82b/bm9sywNMTKRUiXhsUiCVxZ/DT9otQiAI6BnnKXRYREdFTyySC+o0bNwAADRuW3ehWXFyM0tLSclsyLlu2DADQqVOn2i2wFlgqLdHEoSGaOOje7FdQWvhQeC/7Z+Kds/g95Zj2GBulNTxt3R+agfeAp607bM1savvHICNnphQxbUggFsWcxn9/uQBRFNA+wOPJJxIREZHeyR7UH4TrpKQkAMDmzZtx4sQJ2Nvb48UXXwQAvPzyywCAffv2AQAyMjIwePBg9OvXD02bNtXu+nL48GH06dMHbdq0qf0fRCY2ZtZo5tgEzRz/3gdbkiTkleYjJT8NtwtStevgj6Um4J76nvY4e3O7svBu667dgcbDxh1WSks5fhQyEmZKBaYNDca3GxKxctt5KEQBbf3d5S6LiIjoqSNIklSzOyhryM/Pr8J2Ly8vbTDv2rUrgL+Dem5uLj777DMkJiYiPT0dGo0GjRs3xuDBgzFu3DgoFFXfKUUfN5M+zBhvxpEkCdnFObh9fwb+7yCfpnMDrJOFo3bpzIMg72HtDnNuIflUKS5RY+GGRFxOzsGUgQFo3dxN7pK0jHF8EdUVHF9EhlGdm0llD+rG4mkI6o+ikTS4ey8Lt/NT/w7xBWlIK0iHSlIDAAQIcLGq9/fyGVsP1LfxgJu1C5Si7L+YIQMpKlZhYXQirqbk4vXBgQh7xlXukgCY1vgiMjUcX0SGwaBeA09zUH8UtUaNjKLMsln3/L+3kcwougPN/T3gRUGE24MtJG3+XgfvYuUMBfeArxOKilWYH3UKN9LyMH1oEIJ9XOQuqU6MLyJjxfFFZBgM6jXAoF55pRoV0gsztFtIPgjymfeytHvAK0UlPKzddJbPeNp4oJ6lI/eAN0GF90oxL+oUbmUUYEZEEAKbOMtaT10eX0Ry4/giMgwG9RpgUK+5YnUJUgvSHgrvZf+eVZytPcZcYQ5Pa3edXWjq23rAwdyeW0gaufyiUsyPTEDK3UK8ERGMFo3l2wb1aRxfRLWF44vIMBjUa4BB3XCKVEVlW0jev3n1wTr4vJJ87TFWSivt8pmyfeDL/mlnXrW/0GRYeYUlmBeZgPSsIswc/vlG1QAAIABJREFUHgK/hk6y1MHxRWQ4HF9EhsGgXgMM6rUvv6SgbP93nRn4VBSqirTH2JrZ3F8689AuNDbusDazkrHyp1tuQQnmRiYgM+ceZg4PgW8Dx1qvgeOLyHA4vogMg0G9BhjUjYMkScgpyb0/A//3DawpBakoVpdoj3O0cNAJ7vVty/aAt1CYy1j90yMnvxhfrktATn4x3h4RCh8vh1q9PscXkeFwfBEZBoN6DTCoGzeNpEHWvZz7M/Cpfwf5wnSoNCrtcc6W9bTLZh4EeXdrV5hxD3i9y8orxldrTyKvqASzRoahiad9rV2b44vIcDi+iAyDQb0GGNRNk0bS4E5RZtms+/2lM7cLUpFWmKHdQlKAADdrl7/D+/1lNG5WLtxCsobu5t7Dl2tPovCeCu+MCkMjD7tauS7HF5HhcHwRGQaDeg0wqNctKo0K6YV3tA9vun1/Bj6jKFO7haRCUMD9wR7wth7aIO9iVY9bSFbBnewifLXuJO6VqDF7dEs0cDP8DcAcX0SGw/FFZBgM6jXAoP50KFGXIq0wvSy856dqg3zmvSztMWaiGTxs3P5+Cuv9IO9k4cgtJB8hPbsIX609iVKVBu+ODoOXq2HDOscXkeFwfBEZBoN6DTCoP93uqe7h/7d359FR1ff/x18zk33fJ2ELECBhAhI2E+S4gr+f37rAV21pWaxLaa3Yc7Bf+6va057Tfu2pX4v+qlSshZ5T+Z5a/f7QFqUVlWJrKybIrhnCEtYYMtn3fWZ+fySMxARZJpeZO3k+/Ce5uffOZxLfzOt+5nPfc6atL8D7ZuFbq9TU3ezbJ8oW2R/cM33LZ7JiM5UQEUeAl+Sqb9dTr+yR1yv9cOlMZaXGGvZY1BdgHOoLMAZB3Q8EdQylvafd13Wm8pw18G097b59YsNjBnSgyer/JNa4cOOCarA6U9em/3plrywW6bGls2RPiTHkcagvwDjUF2AMgrofCOq4WF6vVy09rb4PcDpzTpDvdHf69kuIiO/vAW/3BfnMWLuiw6ICOHrjfVbTqqf/uFdhNqt+uGyWMpKGv+c99QUYh/oCjEFQ9wNBHf7yer1q7GryzcCfDfJVbS51e3p8+yVHJvmWzpwN8pkxdkWEUAvJ09WtevqVPYqKsOmHS2cpbZjDOvUFGIf6AoxBUPcDQR1G8Xg9qu9s6L959fNZeFdbtXq9bkl9LSTTolM+Xz4Tl6lRsZnKiElTmDUswM/g8pysatEv/7hXMVFhemzZLKUkDN87CdQXYBzqCzAGQd0PBHVcaW6PWzUddQM+wKmyzaWajlpfD3irxaqMsy0kz1kHnxadaooe8MfPNGvNq3sVHx2hHy6bpeT4yGE5L/UFGIf6AoxBUPcDQR3BosfTq+r2mkEz8HUd9b4e8GHWMGXGZAxYPpMVm6mUqKSg6wFf/lmTnnltnxLjIvXY0plKjPM/rFNfgHGoL8AYBHU/ENQR7Lrc3apqc30e3lv7vm7oavTtE2GLUFaMXVlxn8++j4rLVGJEQkBbSB6paNSzr+1XSkKkfrh0lhJiI/w6H/UFGIf6AoxBUPcDQR1m1dHb0dcDvrXKN/te2Vallu5W3z7RYdG+5TN969/7ZuDjI4z/FNGzDp1q0P/9n/3KSI7WD74xU/Exlx/WqS/AONQXYAyCuh8I6gg1rd1t/X3fz52Br1J7b4dvn7jw2P6lM5kDesHHhA9/S0VJOniiXr/adEBZKTF69BszFRd9eZ1uqC/AONQXYAyCuh8I6hgJvF6vmrtb+oP72dn3vgDf5e727ZcUmTgguI+K6+sBH2nzb8mKJH16vE7Pbzqg0elx+sHXCxQTdelhnfoCjEN9AcYgqPuBoI6RzOP1qKGzqa//u28GvkpV7dXq8fT69kuNStGo/htXzwZ5e0y6wi+xB/yB8lqtff0TZWfG6z+WFCg68tJaUFJfgHGoL8AYBHU/ENSBwTxej2o76vpm3fuXzlS2VcnVXuNrIWmRRRkxaZ+H9/5lNBnRaV/aQnLvkRqt+9OnmpCVoEe+NuOSwjr1BRiH+gKMQVD3A0EduHi9nl5Vt9f29X/vXwd/pq1KNe11vhaSNotN9rM94OMyfUE+LTrF10JyV1m1frO5VJNGJ+iRrxUoMuLiesNTX4BxqC/AGAR1PxDUAf91u3vkaq/uWz7Tvwb+TFuV6jobfPuEW8OVFZvhC+4t9VH6y9/rNMWeqdVfLVBk+IXDOvUFGIf6AoxBUPcDQR0wTmdvp6raq1XZv3zmbJBv6m727eN12xTlSdLMsRM1Jj5LWf0tJBMi4nw94HdW7dGb5VvV2NWopMgk3ZFzi67OnBWopwWEJF6/AGMQ1P1AUAeuvPaedt+ymd0nj6us+pTC49rksXb59okNj1FWrF022XS06bjcXrfvZ+HWcN09+XbNsRdIsshisajvP0kWi6x9X32+PYAf+gQEOy6EAWMR1P1AUAcC74P9lfr922WaNjlOt1yfJFdHtW/5zLGmk8P2OOeGdqsskqUv3Fv6v7ae/c7Sv68s0he+/nwf36XB0BcKA44b4uKhf9+hz90/pkFjGXyewcdZv3BuDX3cEM9z0ONf9O/E+iXPTec9z9m/xaX8Tj5/fKvO99ws/V8PdRF37s8H//0GPreBf4ehtg09pi/7f2Pocw/8f9Mi64X/NkM8t8u5QN1ZtUevlL2uHk+Pb1u4NVxL8+4irAPD5HKC+qX1RAMAA103Y5Tcbo/++93DirBE68FF1yjM1hf+Vm3/P+c97t8n3aqzcw5eeeX1nr2l9ezXfd/37fP512f3HXhc//fnPc7zhXPrvOf58jGde9zAMQ1+fI/vPB6vR17v0M/lwmO6uN+JznmeF/6dXOi5yXc8Auvc0P75BWpfwO8+J6Cf1ePp0ZvlWwnqQAAR1AEElRtnjZHb49Ur247ot2859Z07HLJZrUqOTFJDV+Og/ZMjk7Rw3PUBGCkuxZCB/5yA7/nCBcNQFw+eL1wcSIMvQnz7DNp2/guWgece+oLlfOc59yLKv4uxwRc+F7yI8uNi7IvH/e30B0P+3YaqOQBXDkEdQNBZOGes3B6vXtt+VGFWi751m0N35Nwy5Fvzd+TcEsCR4mKduwxEF16JgStsT/WB814IAwgcgjqAoPS/rx6nXrdHr//jmGxWi+67daYkcbMbYAAuhIHgRFAHELRunTdebo9Xf/7ncVmtFn3z32bq6sxZ3KwNDLOzF7xcCAPBhaAOIKjdMX+C3G6v3tpxQrVNHapu6FB9c5dSEiJ15/U5mpefGeghAiHh6sxZXAgDQYagDiDoLb52gk65WrS/vM63ra65Sy+/XSZJhHUAQEiyBnoAAHAhFotFFTWtg7Z393r0xj/KAzAiAACMR1AHYAp1zV3n3e72eK7waAAAMB5BHYAppCZEnvdn//HCDr3y3mGVVzb5+kMDAGB2rFEHYAp3Xp+jl98uU3fv57PnEWFWXV8wSg0tXfr7vkpt212h9KQoFToyVeSwa1RabABHDACAfwjqAEzh7A2jb/yjfMiuL+2dvdpzuEYlzir95aMT2rLjhMZlxKkw367CqXalJEQFcPQAAFw6i5f3iSVJdXWt8niG71dBeyvAOBeqr6bWLu0sq1aJ06Vjlc2SpCljk1SUb9ec3AzFRYdfqaECpsPrF2AMq9Wi1NS4SzqGoN6PoA6Yx6XUV3VDu0qcLhU7XTpT1y6b1aLpE1NV6LCrYFKaIiNsBo8WMBdevwBjENT9QFAHzONy6svr9eqUq1UlTpdKDrrU0NKlyHCbZk5JU5HDLsf4FIXZuL8e4PULMMblBHXWqAMYESwWi7Iz45WdGa+7b8zRkdONKna6tKusWsWlLsVFh2tuXoaK8u3KGZ0oq8US6CEDAEY4ZtT7MaMOmMdw1lev26NPj9Wr2FmlfUdq1d3rUWpClAoddhU57BqTcWmzH4DZ8foFGIMZdQC4RGE2qwomp6lgcpo6unq170itip0ubS05pb8Wn9To9FgVOfo6x6QlRQd6uACAEYQZ9X7MqAPmcSXqq7m9u29ZjNOloxVNkqRJoxNV6LBr7tQMJcREGPr4QKDw+gUYg5tJ/UBQB8zjStdXbWOHSg72dY75rKZNVotF+RNSVOSwq2BymqIjeXMSoYPXL8AYBHU/ENQB8whkfVVUt6rY6VKJs0p1zV2KCOtbOlPosGv6xFQ6x8D0eP0CjEFQ9wNBHTCPYKgvj9er8s+aVOx06eOD1Wrt6FFsVJhm52aoyGHXlHFJdI6BKQVDfQGhiKDuB4I6YB7BVl+9bo+cJxpU4qzSnsO16upxKzk+UoVT7Sp02DXOHicLoR0mEWz1BYQKur4AQACE2ay6KidVV+WkqqvbrX1Ha1XidOm9Xae1decpZaXGqNDRF9rtyTGBHi4AwCSYUe/HjDpgHmapr9aOHu06VK2SUpcOnW6UJE3ISlCRw66rp2YoMS4ywCMEBjNLfQFmw9IXPxDUAfMwY33VN3dq58FqFTurdMrVKotFcmQnq9CRqVlT0hUTxRucCA5mrC/ADEwX1Kurq7Vx40bt379fn376qdrb27Vx40YVFhZe0nncbrcWL16sw4cP6/HHH9e99957yWMhqAPmYfb6qqxt83WOqWnsVJjNqhmTUlXksOuqnFSFh9kCPUSMYGavLyBYmW6N+vHjx7V+/XplZ2crNzdXe/fuvazzvPrqq6qoqBjm0QGAMUalxerO6ybq36+doGNnmlVS6tLOgy7tPlSj6EibZk/JUGG+XVPHJctq5SZUABipAhrU8/PzVVxcrOTkZG3btk2rVq265HM0Njbq+eef1wMPPKC1a9caMEoAMIbFYlHOqETljErUkgWTVHayUcXOKu06VK1/fXJGibERmjs1Q/PyMzU+M57OMQAwwgQ0qMfFXdr0/1Cee+45jRkzRosWLSKoAzAtm9Wq/Akpyp+QohX/y60D5XUqdrr0972faduuCmUkR6uov3NMVmpsoIcLALgCTH330qFDh/Taa69p48aNzDQBCBkR4TbNycvQnLwMtXf2aPehGhU7XXrrwxN688MTyrbH+9o9JsfTOQYAQpWpg/qTTz6phQsXas6cOaxRBxCSYqLCde2MUbp2xig1tHTp47JqlTir9D/vH9X/e/+ocsclqdBh15y8DMVGhQd6uACAYWTaoL5161bt3btXb7/99rCc71Lvwr0Y6enxw35OAH1GYn2lp8drysQ0LfuKQ5/VtOqDPRX6x94Kvbz1kP7w3mHNzrPr+lljNNdhV1SEaf95RxAYifUFBCNT/kve1dWlp59+Wvfcc4/Gjh07LOekPSNgHtSXFCFp4azRWjBzlE66WlTc3zmmpLRKkRE2zZqcrqJ8uxzjk2WzWgM9XJgI9QUYw3TtGS/XK6+8ooaGBt1xxx2+JS9VVVWSpKamJlVUVMhutys8nLeBAYQ2i8Wi8ZkJGp+ZoK/dOEmHTjeqxFmlXWU1+qi0SvEx4Zqbl6EiR6ZyRidwPw8AmIgpg3plZaXa29u1aNGiQT9bt26d1q1bp7/+9a/KyckJwOgAIDCsVoumZidranaylt2cq0+O9XWO+eeBM9q+5zOlJUap0GFXkcOu0enDv9wPADC8TBHUT506JUkaN26cJOnuu+8e9OmldXV1+slPfqK77rpLN910kzIzM6/4OAEgWISHWTVrSrpmTUlXR1ev9hyuUYnTpb8Wn9RfPjqpMelxKsq36+qpGUpLjA70cAEAQwh4UF+3bp0kqby8XJK0efNm7d69WwkJCVq+fLkk6d5775Ukbd++XZKUm5ur3NzcAec5uwRmypQpWrhw4ZUYOgCYQnRkmOZPz9L86VlqauvWrrJqFTurtOnv5dr093JNHpOoov7OMfExEYEeLgCgX8CD+nPPPTfg+9dff12SNHr0aF9QBwAMj8TYCC2YPUYLZo9RdWOHdjpdKna69N/vHtYr244of0KKihx2FUxOo3MMAASYxev1Dl+rExOj6wtgHtTX8PJ6vTpd3aoSp0slB12qb+5SRLhVMyenq9Bh17QJKQqz0TlmpKC+AGOMmK4vAIDhY7FYNM4er3H2eN11Q46OVjSp2OnSxwddKnG6FBsVprl5GSp02DV5bJKsdI4BgCuCGfV+zKgD5kF9XRm9bo9Kj9erxOnSniM16u7xKDk+0tc5ZmxGHO0eQxD1BRiDGXUAwLAJs1k1Y1KaZkxKU2d3r/YdqVWx06X3Pj6trSWnNCotVoUOuwoddmUk0TkGAIYbM+r9mFEHzIP6CqyW9m7tOlSjktIqHa5okiTljEpQocOuuVPtSoylc4yZUV+AMS5nRp2g3o+gDpgH9RU86po6VXLQpeJSlypqWmW1WOQYn6xCh12zpqQrOpI3bs2G+gKMQVD3A0EdMA/qKzhV1PR3jnG6VNvUqfCwvqUzRQ67pk9MVXgYnWPMgPoCjMEadQBAwIxJj9OY6+N053UTVV7ZrJJSl3aWubSrrFrRkWGak5uuIoddueOSZbVyEyoAXAgz6v2YUQfMg/oyD7fHo4MnGvRRaV/nmK5utxLjIlQ41a6ifLuy7fF0jgky1BdgDGbUAQBBxWa1atrEVE2bmKquHrf2H61VidOlv+2u0Lsfn5Y9JUZF/Z1jMlNiAj1cAAgqzKj3Y0YdMA/qy/zaOnu0+1CNikurdOhUo7ySxmfGq6i/c0xyfGSghzhiUV+AMbiZ1A8EdcA8qK/Q0tDS5bsJ9aSrRRZJednJKnLYNTs3XTFR4YEe4ohCfQHGIKj7gaAOmAf1FbrO1LWpxOlSsdOl6oYOhdksuiqnr3PMVTmpigi3BXqIIY/6AoxBUPcDQR0wD+or9Hm9Xp2oalFxqUs7D7rU1NatqAibZk9JV2G+XVOzk2Wz0u7RCNQXYAxuJgUAhASLxaIJWQmakJWgJTdN0sFTDSopdWn34Wp9+GmVEmLCNXeqXUUOuyaOSqBzDICQxIx6P2bUAfOgvkaunl63DpTXqdjp0v6jdep1e5SeFKVCR6aKHHaNSosN9BBNj/oCjMHSFz8Q1AHzoL4gSe2dvdpzuEYlzio5TzbI65XGZcSpMN+uwql2pSREBXqIpkR9AcYgqPuBoA6YB/WFL2pq7dLOsmoVl7p0/EyzJGnK2CQVOeyak5ehuGg6x1ws6gswBkHdDwR1wDyoL3wZV0N7X+eYUpeq6ttls1o0fWKqCh12FUxKU2QEnWO+DPUFGIOg7geCOmAe1Bcuhtfr1SlXa1+P9oMuNbR0KTLcpplT+to9OsanKMxG55gvor4AY9D1BQCAfhaLRdmZ8crOjNfdN+boyOlGFTtd2tW/RCYuOlxz8zJU6LBr0phEWekcAyDIMKPejxl1wDyoL/ijp9ejT4/XqcTp0r4jteru9Sg1IUqFjr52j2MyLm3GK9RQX4AxWPriB4I6YB7UF4ZLR1ev9h2pVbHTpdLj9fJ4vRqdHqsiR1/nmLSk6EAP8YqjvgBjENT9QFAHzIP6ghGa27v7lsU4XTpa0SRJmjQ6UYUOu+bmZSghNiLAI7wyqC/AGAR1PxDUAfOgvmC0msYO7TzY1znms9o2WS0WOSYka54jUwWT0xQdGbq3eFFfgDEI6n4gqAPmQX3hSqqoblWx06USZ5XqmrsUEWZVweQ0FTrsmj4xNeQ6x1BfgDEI6n4gqAPmQX0hEDxer8o/a1JxqUsfl1WrtaNHsVFhmp2boSKHXVPGJYVE5xjqCzAGQd0PBHXAPKgvBFqv2yPniXoVO13ae7hWXT1uJcdH6uqpGSpyZGqcPU4Wk4Z26gswBn3UAQC4AsJsVl2Vk6arctLU1e3WvqO1KnG6tG1Xhd7ZeVpZqTEqdNhV6LDLnhwT6OECMClm1Psxow6YB/WFYNXa0aNdh/o+UOnw6UZJ0oSsBBU57Lp6aoYS4yIDPMILo74AY7D0xQ8EdcA8qC+YQX1zp0oOulRS6tKp6lZZLNLU7GQVOuyaPSVDMVHB+aY29QUYg6DuB4I6YB7UF8zms9o2lfR3jqlp7FSYzaoZk1JV5LDrqpxUhYfZAj1EH+oLMAZB3Q8EdcA8qC+Yldfr1bEzzSopdWnnQZea23sUHWnT7CkZKsy3a+q4ZFmtgb0JlfoCjEFQ9wNBHTAP6guhwO3x6ODJBpWUurT7cI06u91KjI3Q3P7OMROy4gPSOYb6AoxBUPcDQR0wD+oLoaa7x60D5XUqdrp0oLxWvW6vMpKjVdTfOSYrNfaKjYX6AoxBUPcDQR0wD+oLoay9s0e7D9Wo2OlS2ckGeSVl2+N97R6T443tHEN9AcYgqPuBoA6YB/WFkaKhpUsfH3Sp2OnSiaoWWSTljkvq6xyTm6G46PBhf0zqCzAGQd0PBHXAPKgvjERV9e0qcbpUXFolV0OHbFaLpk9MVVG+XTMmpSkyfHg6x1BfgDEI6n4gqAPmQX1hJPN6vTrpalFxqUslB11qau1WZIRNsyanqyjfLsf4ZNms1ss+P/UFGONygnpwftoCAAAYksVi0fjMBI3PTNDXbpykQ6caVOx0adehGn1UWqX4mHDNzevrHJMzOiEgnWMADA9m1Psxow6YB/UFDNbT69Enx/o6x+w/WqueXo/SEqN8N6GOSb+4mTzqCzAGS1/8QFAHzIP6Ar5cR1ev9hyuUYnTpdIT9fJ6pTHpcSrKt+vqqRlKS4w+77HUF2AMgrofCOqAeVBfwMVrauvWxwddKnG6VF7ZLEmaPCZRRQ675uRlKD4mQpL0UWmV3vhHueqbu5SSEKk7r8/RvPzMQA4dCCkEdT8Q1AHzoL6Ay1Pd2OHrHHOmrl02q0X5E1KUmhCpDz+pUnevx7dvRJhV3/y3PMI6MEwI6n4gqAPmQX0B/vF6vTpd3aoSZ1/nmPrmriH3S02I1C8fmn+FRweEpssJ6pffvwkAAJiSxWLROHu8vnrjJD393WvOu1/deQI8gCuDoA4AwAhmtViUmhA55M/Otx3AlUFQBwBghLvz+hxFhA2MBBFhVt15fU6ARgRA4gOPAAAY8c7eMErXFyC4ENQBAIDm5WdqXn4mN2sDQYSlLwAAAEAQIqgDAAAAQYigDgAAAAQhgjoAAAAQhAjqAAAAQBAiqAMAAABBiKAOAAAABCGCOgAAABCECOoAAABAEOKTSftZrRZTnBNAH+oLMA71BQy/y6kri9fr9RowFgAAAAB+YOkLAAAAEIQI6gAAAEAQIqgDAAAAQYigDgAAAAQhgjoAAAAQhAjqAAAAQBAiqAMAAABBiKAOAAAABCGCOgAAABCECOoAAABAEAoL9ABCSXV1tTZu3Kj9+/fr008/VXt7uzZu3KjCwsJADw0wtQMHDuhPf/qTSkpKVFlZqaSkJM2cOVOrV69WdnZ2oIcHmNonn3yi3/zmN3I6naqrq1N8fLzy8vK0atUqzZo1K9DDA0LO+vXrtWbNGuXl5Wnz5s1fui9BfRgdP35c69evV3Z2tnJzc7V3795ADwkICRs2bNCePXt0yy23KDc3VzU1NfrDH/6gxYsXa9OmTcrJyQn0EAHTOn36tNxut7761a8qPT1dLS0teuutt7R8+XKtX79e8+fPD/QQgZBRU1OjF198UTExMRe1v8Xr9XoNHtOI0draqp6eHiUnJ2vbtm1atWoVM+rAMNizZ4+mTZumiIgI37YTJ07o9ttv16233qqnnnoqgKMDQk9HR4cWLlyoadOm6aWXXgr0cICQ8dhjj6myslJer1fNzc0XnFFnjfowiouLU3JycqCHAYScWbNmDQjpkjR+/HhNnjxZ5eXlARoVELqio6OVkpKi5ubmQA8FCBkHDhzQm2++qccff/yijyGoAzAlr9er2tpaLo6BYdLa2qr6+nodO3ZMzz77rA4fPqx58+YFelhASPB6vfrP//xPLV68WFOnTr3o41ijDsCU3nzzTblcLj3yyCOBHgoQEp544gm98847kqTw8HB9/etf14MPPhjgUQGh4c9//rOOHj2qF1544ZKOI6gDMJ3y8nL97Gc/0+zZs7Vo0aJADwcICatWrdKSJUtUVVWlzZs3q7u7Wz09PYOWnQG4NK2trXrmmWf07W9/WxkZGZd0LEtfAJhKTU2NvvOd7ygxMVHPPfecrFb+GQOGQ25urubPn6+77rpLv/vd71RaWnpJa2kBDO3FF19UeHi47rvvvks+llc4AKbR0tKilStXqqWlRRs2bFB6enqghwSEpPDwcC1YsEDvvvuuOjs7Az0cwLSqq6v18ssva+nSpaqtrVVFRYUqKirU1dWlnp4eVVRUqKmp6bzHs/QFgCl0dXXpwQcf1IkTJ/T73/9eEydODPSQgJDW2dkpr9ertrY2RUVFBXo4gCnV1dWpp6dHa9as0Zo1awb9fMGCBVq5cqUeffTRIY8nqAMIem63W6tXr9a+ffu0bt06FRQUBHpIQMior69XSkrKgG2tra165513lJWVpdTU1ACNDDC/MWPGDHkD6a9+9Su1t7friSee0Pjx4897PEF9mK1bt06SfL2dN2/erN27dyshIUHLly8P5NAA03rqqae0fft23XjjjWpsbBzwARGxsbFauHBhAEcHmNvq1asVGRmpmTNnKj09XWfOnNEbb7yhqqoqPfvss4EeHmBq8fHxQ75Gvfzyy7LZbBd8/eKTSYdZbm7ukNtHjx6t7du3X+HRAKFhxYoV2rlz55A/o7YA/2zatEmbN2/W0aNH1dzcrPj4eBUUFOj+++/X1VdfHejhASFpxYoVF/XJpAR1AAAAIAjR9QUAAAAIQgR1AAAAIAgR1AEAAIAgRFAHAAAAghBBHQAAAAhCBHUAAAAgCBHUAQAAgCBEUAcABMyKFSt00003BXoYABCUwgI9AADA8CopKdE999xz3p/bbDY5nc4rOCIAwOUgqANAiLrtttt03XXXDdputfJmKgCYAUEdAEKUw+HQokWLAj0MAMBlYloFAEaoiooK5ebmau3atdqyZYtuv/12TZ9kNJGpAAAET0lEQVQ+XTfccIPWrl2r3t7eQceUlZVp1apVKiws1PTp0/WVr3xF69evl9vtHrRvTU2NnnzySS1YsEDTpk3TvHnzdN999+nDDz8ctK/L5dL3v/99zZ07VzNmzNADDzyg48ePG/K8AcAsmFEHgBDV0dGh+vr6QdsjIiIUFxfn+3779u06ffq0li1bprS0NG3fvl2//vWvVVlZqV/84he+/T755BOtWLFCYWFhvn3ff/99rVmzRmVlZXrmmWd8+1ZUVOgb3/iG6urqtGjRIk2bNk0dHR3av3+/duzYofnz5/v2bW9v1/LlyzVjxgw98sgjqqio0MaNG/XQQw9py5YtstlsBv2GACC4EdQBIEStXbtWa9euHbT9hhtu0EsvveT7vqysTJs2bVJ+fr4kafny5Xr44Yf1xhtvaMmSJSooKJAk/fznP1d3d7deffVV5eXl+fZdvXq1tmzZorvvvlvz5s2TJP30pz9VdXW1NmzYoGuvvXbA43s8ngHfNzQ06IEHHtDKlSt921JSUvTLX/5SO3bsGHQ8AIwUBHUACFFLlizRLbfcMmh7SkrKgO+vueYaX0iXJIvFom9961vatm2b3nvvPRUUFKiurk579+7VzTff7AvpZ/f97ne/q61bt+q9997TvHnz1NjYqH/+85+69tprhwzZX7yZ1Wq1DupSU1RUJEk6efIkQR3AiEVQB4AQlZ2drWuuueaC++Xk5AzaNmnSJEnS6dOnJfUtZTl3+7kmTpwoq9Xq2/fUqVPyer1yOBwXNc6MjAxFRkYO2JaUlCRJamxsvKhzAEAo4mZSAEBAfdkadK/XewVHAgDBhaAOACNceXn5oG1Hjx6VJI0dO1aSNGbMmAHbz3Xs2DF5PB7fvuPGjZPFYtHBgweNGjIAjAgEdQAY4Xbs2KHS0lLf916vVxs2bJAkLVy4UJKUmpqqmTNn6v3339fhw4cH7Pvb3/5WknTzzTdL6lu2ct111+mDDz7Qjh07Bj0es+QAcHFYow4AIcrpdGrz5s1D/uxsAJekvLw8ffOb39SyZcuUnp6uv/3tb9qxY4cWLVqkmTNn+vb70Y9+pBUrVmjZsmVaunSp0tPT9f777+tf//qXbrvtNl/HF0n68Y9/LKfTqZUrV2rx4sXKz89XV1eX9u/fr9GjR+sHP/iBcU8cAEIEQR0AQtSWLVu0ZcuWIX/27rvv+taG33TTTZowYYJeeuklHT9+XKmpqXrooYf00EMPDThm+vTpevXVV/X888/rj3/8o9rb2zV27Fg9+uijuv/++wfsO3bsWL3++ut64YUX9MEHH2jz5s1KSEhQXl6elixZYswTBoAQY/HyHiQAjEgVFRVasGCBHn74YX3ve98L9HAAAF/AGnUAAAAgCBHUAQAAgCBEUAcAAACCEGvUAQAAgCDEjDoAAAAQhAjqAAAAQBAiqAMAAABBiKAOAAAABCGCOgAAABCECOoAAABAEPr/ipClttaL0KwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Train Loss</th>\n",
              "      <th>Valid Loss</th>\n",
              "      <th>Valid Acc</th>\n",
              "      <th>Train Time</th>\n",
              "      <th>Valid Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.7614</td>\n",
              "      <td>1.5022</td>\n",
              "      <td>0.4551</td>\n",
              "      <td>0:02:23</td>\n",
              "      <td>0:00:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.4347</td>\n",
              "      <td>1.4477</td>\n",
              "      <td>0.4691</td>\n",
              "      <td>0:02:21</td>\n",
              "      <td>0:00:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.3220</td>\n",
              "      <td>1.4460</td>\n",
              "      <td>0.4788</td>\n",
              "      <td>0:02:21</td>\n",
              "      <td>0:00:10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Train Loss  Valid Loss  Valid Acc Train Time Valid Time\n",
              "epoch                                                         \n",
              "1          1.7614      1.5022     0.4551    0:02:23    0:00:10\n",
              "2          1.4347      1.4477     0.4691    0:02:21    0:00:10\n",
              "3          1.3220      1.4460     0.4788    0:02:21    0:00:10"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lvw80I6P6O95",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 82,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CsxT5WHt6RIR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "dc874df1-0e28-4968-b229-69f4a054a508"
      },
      "source": [
        "# if 5 folds:  20% for testing (1 fold), 70.00% for training and 10.00% for validation\n",
        "# if 10 folds: 10% for testing (1 fold), 78.75% for training and 11.25% for validation\n",
        "\n",
        "if CROSS_VALIDATION:\n",
        "  kf = KFold(n_splits=NUM_FOLDS, random_state=SEED, shuffle=True)\n",
        "\n",
        "  mf1s = []\n",
        "  accs = []\n",
        "  fold_num = 1\n",
        "  for train_index, test_index in kf.split(df_data):\n",
        "      print(\"##### Fold number:\", fold_num, \"#####\")\n",
        "      fold_num += 1\n",
        "      train_df = df_data.iloc[train_index]\n",
        "      test_df  = df_data.iloc[test_index]\n",
        "\n",
        "      train_df, dev_df = train_test_split(train_df, test_size=0.125) # change percentage (hyperparams?)\n",
        "\n",
        "      X_train, y_train = train_df.sentence.values, train_df.label.values\n",
        "      X_dev, y_dev = dev_df.sentence.values, dev_df.label.values\n",
        "      X_test, y_test = test_df.sentence.values, test_df.label.values\n",
        "      \n",
        "      train_dataloader      = prepare_data(X_train, y_train, 1)\n",
        "      dev_dataloader        = prepare_data(X_dev, y_dev, 0)\n",
        "      prediction_dataloader = prepare_data(X_test, y_test, 0)\n",
        "\n",
        "      training_stats = train(model, optimizer=adam, train_dataloader=train_dataloader, dev_dataloader=dev_dataloader, epochs=EPOCHS, verbose=False, scheduler=linear_sch)\n",
        "      predictions, true_labels = predict(model, prediction_dataloader)\n",
        "      mf1, acc = evaluate(predictions, true_labels, verbose=False)\n",
        "      mf1s.append(mf1)\n",
        "      accs.append(acc)\n",
        "  \n",
        "  print(\"#####################################################################\")\n",
        "  print(\"PARAMS: epochs:\",EPOCHS,\", lr_rate:\",LEARN_RATE,\"epsilon:\",EPSILON,\"...\")\n",
        "  print(\"#####################################################################\")\n",
        "  print(\"F1(CV):\",mf1s)\n",
        "  print(f\"Mean-folds-F1: {sum(mf1s)/len(mf1s)}\")\n",
        "  print(\"Acc(CV):\",accs)\n",
        "  print(f\"Mean-folds-Acc: {sum(accs)/len(accs)}\")"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "##### Fold number: 1 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:57 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 2 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:56 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 3 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:56 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 4 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:55 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "##### Fold number: 5 #####\n",
            "Epoch 1 of 3 ...\n",
            "Epoch 2 of 3 ...\n",
            "Epoch 3 of 3 ...\n",
            "\n",
            "Training complete!\n",
            "Total training took 0:06:55 (h:mm:ss)\n",
            "Predicting labels for test sentences...\n",
            "COMPLETED.\n",
            "\n",
            "#####################################################################\n",
            "PARAMS: epochs: 3 , lr_rate: 2e-05 epsilon: 1e-08 ...\n",
            "#####################################################################\n",
            "F1(CV): [0.5476654014065779, 0.5383039432871108, 0.5317713649340343, 0.5359667374362341, 0.5277145657370729]\n",
            "Mean-folds-F1: 0.536284402560206\n",
            "Acc(CV): [0.5545210014483758, 0.5443823711980137, 0.5410717980550382, 0.5452100144837575, 0.535182119205298]\n",
            "Mean-folds-Acc: 0.5440734608780966\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9RsHZhTt6Un-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}