Hierarchical Attention Network Text Classification.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Hierarchical Attention Network Text Classification.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"Zq1kk6B9dlIv","colab_type":"text"},"cell_type":"markdown","source":["## Hierarchical Attention Document Classifiation"]},{"metadata":{"id":"XEyahhfGdLgj","colab_type":"code","outputId":"2d90d5cf-1918-4111-f99c-eda1518f1f94","executionInfo":{"status":"ok","timestamp":1546292297638,"user_tz":-330,"elapsed":86011,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":1044}},"cell_type":"code","source":["!pip install torch\n","!pip install torchtext\n","!pip install -U spacy\n","!python -m spacy download en\n"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Collecting torch\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/60/66415660aa46b23b5e1b72bc762e816736ce8d7260213e22365af51e8f9c/torch-1.0.0-cp36-cp36m-manylinux1_x86_64.whl (591.8MB)\n","\u001b[K    100% |████████████████████████████████| 591.8MB 26kB/s \n","tcmalloc: large alloc 1073750016 bytes == 0x60e7a000 @  0x7f265a2e92a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n","\u001b[?25hInstalling collected packages: torch\n","Successfully installed torch-1.0.0\n","Collecting torchtext\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c6/bc/b28b9efb4653c03e597ed207264eea45862b5260f48e9f010b5068d64db1/torchtext-0.3.1-py3-none-any.whl (62kB)\n","\u001b[K    100% |████████████████████████████████| 71kB 4.7MB/s \n","\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from torchtext) (4.28.1)\n","Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from torchtext) (1.0.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchtext) (1.14.6)\n","Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from torchtext) (2.18.4)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2018.11.29)\n","Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.6)\n","Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.22)\n","Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n","Installing collected packages: torchtext\n","Successfully installed torchtext-0.3.1\n","Requirement already up-to-date: spacy in /usr/local/lib/python3.6/dist-packages (2.0.18)\n","Requirement already satisfied, skipping upgrade: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.2)\n","Requirement already satisfied, skipping upgrade: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.0.1)\n","Requirement already satisfied, skipping upgrade: plac<1.0.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.9.6)\n","Collecting numpy>=1.15.0 (from spacy)\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ff/7f/9d804d2348471c67a7d8b5f84f9bc59fd1cefa148986f2b74552f8573555/numpy-1.15.4-cp36-cp36m-manylinux1_x86_64.whl (13.9MB)\n","\u001b[K    100% |████████████████████████████████| 13.9MB 3.1MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: ujson>=1.35 in /usr/local/lib/python3.6/dist-packages (from spacy) (1.35)\n","Requirement already satisfied, skipping upgrade: preshed<2.1.0,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.0.1)\n","Requirement already satisfied, skipping upgrade: regex==2018.01.10 in /usr/local/lib/python3.6/dist-packages (from spacy) (2018.1.10)\n","Requirement already satisfied, skipping upgrade: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy) (2.18.4)\n","Requirement already satisfied, skipping upgrade: dill<0.3,>=0.2 in /usr/local/lib/python3.6/dist-packages (from spacy) (0.2.8.2)\n","Requirement already satisfied, skipping upgrade: thinc<6.13.0,>=6.12.1 in /usr/local/lib/python3.6/dist-packages (from spacy) (6.12.1)\n","Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2018.11.29)\n","Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n","Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.6)\n","Requirement already satisfied, skipping upgrade: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.22)\n","Requirement already satisfied, skipping upgrade: six<2.0.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.11.0)\n","Requirement already satisfied, skipping upgrade: wrapt<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (1.10.11)\n","Requirement already satisfied, skipping upgrade: tqdm<5.0.0,>=4.10.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (4.28.1)\n","Requirement already satisfied, skipping upgrade: msgpack-numpy<0.4.4 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.4.3.2)\n","Requirement already satisfied, skipping upgrade: msgpack<0.6.0,>=0.5.6 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.5.6)\n","Requirement already satisfied, skipping upgrade: cytoolz<0.10,>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from thinc<6.13.0,>=6.12.1->spacy) (0.9.0.1)\n","Requirement already satisfied, skipping upgrade: toolz>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cytoolz<0.10,>=0.9.0->thinc<6.13.0,>=6.12.1->spacy) (0.9.0)\n","\u001b[31mfeaturetools 0.4.1 has requirement pandas>=0.23.0, but you'll have pandas 0.22.0 which is incompatible.\u001b[0m\n","\u001b[31mcufflinks 0.14.6 has requirement plotly>=3.0.0, but you'll have plotly 1.12.12 which is incompatible.\u001b[0m\n","Installing collected packages: numpy\n","  Found existing installation: numpy 1.14.6\n","    Uninstalling numpy-1.14.6:\n","      Successfully uninstalled numpy-1.14.6\n","Successfully installed numpy-1.15.4\n","Requirement already satisfied: en_core_web_sm==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.0.0/en_core_web_sm-2.0.0.tar.gz#egg=en_core_web_sm==2.0.0 in /usr/local/lib/python3.6/dist-packages (2.0.0)\n","\n","\u001b[93m    Linking successful\u001b[0m\n","    /usr/local/lib/python3.6/dist-packages/en_core_web_sm -->\n","    /usr/local/lib/python3.6/dist-packages/spacy/data/en\n","\n","    You can now load the model via spacy.load('en')\n","\n"],"name":"stdout"}]},{"metadata":{"id":"Ynb3s5rceDFB","colab_type":"code","colab":{}},"cell_type":"code","source":["from torchtext import data\n","from torchtext import datasets\n","import random\n","import re\n","import numpy as np\n","from tqdm import tqdm\n","import seaborn as sns\n","import matplotlib.pyplot as plt\n","%matplotlib inline\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"rMYKIH7SeIbY","colab_type":"code","colab":{}},"cell_type":"code","source":["SEED = 10\n","torch.manual_seed(SEED)\n","torch.cuda.manual_seed(SEED)\n","torch.backends.cudnn.deterministic = True #for reproducible results\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"7vURasyfeUba","colab_type":"code","colab":{}},"cell_type":"code","source":["# Initializing Hyperparameters\n","#MAX_DOCUMENT_LEN = 900"],"execution_count":0,"outputs":[]},{"metadata":{"id":"0iNFLQcOeL8L","colab_type":"code","colab":{}},"cell_type":"code","source":["def clean_text(text):\n","    text = text.lower()\n","    text = re.compile('<.*?>').sub(\" \", text)\n","    text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n","    text = re.sub(r\"i'm\", \"i am\", text)\n","    text = re.sub(r\"he's\", \"he is\", text)\n","    text = re.sub(r\"she's\", \"she is\", text)\n","    text = re.sub(r\"that's\", \"that is\", text)\n","    text = re.sub(r\"what's\", \"what is\", text)\n","    text = re.sub(r\"where's\", \"where is\", text)\n","    text = re.sub(r\"\\'ll\", \" will\", text)\n","    text = re.sub(r\"\\'ve\", \" have\", text)\n","    text = re.sub(r\"\\'re\", \" are\", text)\n","    text = re.sub(r\"\\'d\", \" would\", text)\n","    text = re.sub(r\"won't\", \"will not\", text)\n","    text = re.sub(r\"can't\", \"cannot\", text)\n","    #text = re.sub(r\"[-()\\\"#/@;:<>{}+=~|.?,]\", \"\", text)\n","    text = text.split()\n","    return text"],"execution_count":0,"outputs":[]},{"metadata":{"id":"jjQdnVx4eQPA","colab_type":"code","colab":{}},"cell_type":"code","source":["TEXT = data.Field(tokenize=clean_text,lower=True)    \n","LABEL = data.LabelField(dtype = torch.float)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"1vk4sr1ofWs1","colab_type":"code","colab":{}},"cell_type":"code","source":["from torchtext import datasets\n","train_data, test_data = datasets.IMDB.splits(text_field=TEXT,label_field=LABEL)\n","train_data, valid_data = train_data.split(split_ratio= 0.8, random_state=random.seed(SEED))"],"execution_count":0,"outputs":[]},{"metadata":{"id":"-ZpouPRSfdOo","colab_type":"code","outputId":"22d2828c-0df0-4bf3-e966-e054c27becd1","executionInfo":{"status":"ok","timestamp":1546293414966,"user_tz":-330,"elapsed":19374,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":89}},"cell_type":"code","source":["print('Number of training examples: {}'.format(len(train_data)))\n","print('Number of validation examples: {}'.format(len(valid_data)))\n","print(vars(train_data[0]))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Number of training examples: 20000\n","Number of validation examples: 5000\n","{'text': ['wow', 'this', 'movie', 'bored', 'the', 'pants', 'off', 'me', 'when', 'i', 'saw', 'it', 'bland', 'pointless', 'and', 'unmoving', 'apparently', 'ash', 'and', 'co', 'can', 'travel', 'through', 'time', 'with', 'the', 'help', 'of', 'the', 'spirit', 'of', 'the', 'forest', \"'princess\", \"mononoke'\", 'much', 'there', 'they', 'meet', 'a', 'dorky', 'kid', 'named', 'sam', 'and', 'the', 'plot', 'begins', 'so', 'tom', 'ash', 'and', 'huck', 'sam', 'get', 'high', 'with', 'nature', 'become', 'hippies', 'and', 'try', 'to', 'free', 'celebi', 'the', 'spirit', 'from', 'some', 'weirdo', 'hunter', 'guy', 'i', \"don't\", 'even', 'know', 'what', 'else', 'went', 'on', 'it', 'all', 'went', 'by', 'in', 'a', 'blur', \"ash's\", 'friends', 'were', 'hardly', 'in', 'it', 'and', 'all', 'the', 'fight', 'scenes', 'were', 'boring', 'after', 'saving', 'the', 'day', 'ash', 'and', 'his', 'infamous', 'friends', 'must', 'return', 'to', 'their', 'time', 'while', 'watching', 'sam', 'float', 'away', 'with', 'celebi', 'that', 'scene', 'was', 'just', 'creepy', 'oo', 'then', 'after', 'returning', 'to', 'their', 'time', 'ash', 'learns', 'that', 'his', 'new', 'friend', 'is', 'actually', 'his', \"rival's\", 'grandpa', 'and', 'i', 'think', 'that', 'is', 'it', 'pretty', 'retarded', \"isn't\", 'it', 'if', 'you', 'love', 'your', 'children', 'you', 'will', 'not', 'expose', 'them', 'to', 'this', '1', 'out', 'of', '10'], 'label': 'neg'}\n"],"name":"stdout"}]},{"metadata":{"id":"hS8OVy6z02Qw","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"VQdILW0OyofJ","colab_type":"text"},"cell_type":"markdown","source":["**Review length distribution plots  of Training, Validation and Training splits**"]},{"metadata":{"id":"x1un3jSAveml","colab_type":"code","outputId":"e12a14ef-0b4d-4c87-80da-15292ebb04e2","executionInfo":{"status":"ok","timestamp":1546293415437,"user_tz":-330,"elapsed":19230,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":379}},"cell_type":"code","source":["train_data_dist = [len(train_data[i].text) for i in range(len(train_data))]\n","valid_data_dist = [len(valid_data[i].text) for i in range(len(valid_data))]\n","test_data_dist = [len(test_data[i].text) for i in range(len(test_data))]\n","\n","sns.set(style='whitegrid')\n","f, axes = plt.subplots(1, 3)\n","\n","sns.distplot(a = train_data_dist,ax=axes[0], axlabel='Train')\n","sns.distplot(a = valid_data_dist,ax=axes[1], axlabel='Validation')\n","sns.distplot(a = test_data_dist,ax=axes[2], axlabel='Test')"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.axes._subplots.AxesSubplot at 0x7f089a1acd30>"]},"metadata":{"tags":[]},"execution_count":32},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAe0AAAFZCAYAAAC173eYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xt8lOWd+P3PPaecJuQAmRDl4IgH\naJS2RFptLKiF0rJd+9vdVtgW+vT52T106RkepVl/Qp8FVKQ8LWhbdVlf21ppXMt2e7Ci3aaVJRFq\npKhRUBBiwiGZYUJgcpzD/fwxuSeTZCYzSeZ03/N9v159lcnkTq7Bi/t7f6/re12XoqqqihBCCCGy\nninTDRBCCCFEYiRoCyGEEDohQVsIIYTQCQnaQgghhE5I0BZCCCF0QoK2EEIIoROWRL5p27ZtHD16\nFEVRqKurY+HCheH3Ghsb2blzJ2azmSVLlrBu3bqY1/h8PjZu3EhraytFRUXs2rWLkpKS1HwyIYQQ\nwmDiBu3Dhw/T2tpKfX09J0+epK6ujvr6+vD7W7ZsYc+ePVRWVrJmzRpWrFiBx+OJes0zzzxDWVkZ\n3/3ud6mvr+eVV17hYx/7WMzf3dzcnJxPKQyjpqYm5b9D+p2IlI4+B9LvxEix+l3coN3U1MSyZcsA\nmDdvHt3d3Xi9Xux2O21tbZSUlFBVVQXA0qVLaWpqwuPxRL2moaGBr33tawCsWrVqSg2HUCdP1z+o\nZJO2T+73pku8z6fn/36TlaufOZ2Mer8bTT5L/J8ZS9w5bbfbTVlZWfh1eXk5LpcLAJfLRXl5+Zj3\nYl1z5swZXnrpJdauXcs3v/lNLl68OKkPJIQQQuSihOa0I01m11PtGlVVcTqdfOUrX+EHP/gBjz32\nGPfee++418Z70tXzkJK0XQghxETEDdoOhwO32x1+3dnZSUVFRdT3Ojo6cDgcWK3WqNfMmDGDxYsX\nA3Drrbeye/fuuA006nCRtH1yv1cIIXJZ3OHx2tpa9u/fD0BLSwsOhwO73Q7ArFmz8Hq9tLe34/f7\naWhooLa2NuY1S5Ys4cCBA+GvO53OVH0uIYQQwnDiZtqLFi2iurqa1atXoygKmzZtYt++fRQXF7N8\n+XI2b97M+vXrAVi5ciVOpxOn0znmGoC1a9dy77338uyzz1JYWMhDDz2U2k8nhBBCGEhCc9obNmwY\n8Xr+/PnhPy9evHjEErBY1wAUFBSwa9euibZRCCGEEMiOaEIIIYRuSNAWQgghdEKCthBCCKETErSF\nEEIInZCgLYQQQuiEBG0hhNCxHz/3Jr9tOp3pZog0mfA2ptnu+abT4T9/4parMtUMkeO0fih9UKSS\nzx/kP/77HQA+KX0tJ0imLYQQOnWhuy/TTRBpJkFbCCF0yn1RgnaukaAthBA65ZKgnXMkaAshhE65\nuiRo5xoJ2kIIoVOSaeceCdpCCKFTrq5eAMwmJcMtEekiQVsIIXRKy7RtVrmV5wr5Ly2EEDp1uWcQ\nAH9AzXBLRLpI0BZCCJ0KqqFgHQgEM9wSkS6G2xEN4FLPIHk2c6abIaZg27ZtHD16FEVRqKurY+HC\nheH3Ghsb2blzJ2azmSVLlrBu3bqY1/h8PjZu3EhraytFRUXs2rWLkpKSTH0sIZIqGAwF7aAa+rNJ\n5rYNz3CZtqqq/OS3b/Hj37yZ6aaISTp8+DCtra3U19ezdetWtm7dOuL9LVu2sHv3bvbu3cvBgwc5\nceJEzGueeeYZysrKePbZZ1m5ciWvvPJKJj6SECkRjBgVDwQl284Fhsu0fUPDRIN+6cB61dTUxLJl\nywCYN28e3d3deL1eANra2igpKaGqqgqApUuX0tTUhMfjiXpNQ0MDX/va1wBYtWpVBj6NEKkTjIja\nPn8Qq0VGGI3OeEFbgrXuud1uqqurw6/Ly8txuVwAuFwuysvLR7zX1tZGV1dX1GvOnDnDSy+9xMMP\nP8yMGTPYtGkTpaWlcdvQ3Nw8pe9pfS/0kNFsuxD35+hJIn8vIn20OW2AQFCK0XKB8YK2T4K20ajq\nxG9G2jWqquJ0OvnKV77CD37wAx577DHuvffeuNfX1NSM+35zc/O43+MaPD30c65KuM3ZLt5nNqJs\nf0iJzLT9UoyWEww3p+3zBzLdBDFFDocDt9sdft3Z2UlFRUXU9zo6OnA4HDGvmTFjBosXLwbg1ltv\n5cSJE2n6FEKkXmTQDsiyr5xguKAtc9n6V1tby/79+wFoaWnB4XBgt9sBmDVrFl6vl/b2dvx+Pw0N\nDdTW1sa8ZsmSJRw4cCD8dafTmZkPJUQKRI5CSaadG4w3PC5BW/cWLVpEdXU1q1evRlEUNm3axL59\n++js7KSmpobNmzezfv16AFauXInT6cTpdI65BmDt2rXce++9PPvssxQWFvLQQw9l8qMJkTSqqo6o\nHpegnRsMGLRleNwINmzYMOL1/Pnzw/OLixcvpr6+Pu41AAUFBezatSs1jRQig0bXncnweG4w3vC4\nFKIJIXJAcFTUlkw7NxguaMvwuBAiFwRHraqQJV+5wYBBW4bHhRDGp0qmnZMMF7SlelwIkQtGZ9oS\ntHOD4YK2DI8LIXLB2DltGR7PBYYL2oO+4eHx0Z1aCCGMYvQcthzPmRsMF7QjM2059UYIYVSjd/eV\nTDs3GDpoSycWQhjV2OpxSVJygQGD9vDwuAwXCSGMSua0c5PhdkQblExbCDFJ27Zt4+jRoyiKQl1d\n3Yj3Ghsb2blzJ2azmSVLlrBu3bqo1yxcuJCNGzfS0tISPgb27rvv5rbbbktqW8cEbSnCzQmGC9oy\npy2EmIzDhw/T2tpKfX09J0+epK6ujnvuuSf8/pYtW9izZw+VlZWsWbOGFStW4PF4xlyjbbH7rW99\ni9tvvz1l7dWGxy1mBX9AlftdjjBg0A5E/Fk6sRAiMU1NTSxbtgyAefPm0d3dTW9vLwBtbW2UlJRQ\nVVUFwNKlS2lqasLj8Yy5xuv1pqW9WtC2Wkz4AwEZWcwRhpvTjqzNkG39hBCJcrvdlJWVhV+Xl5fT\n3d0NgMvlory8fMR7Lpcr6jUulwuAp556ii984Qt885vfxOPxJL292vC41WIGpIYnVxgu05bzZYUQ\nyaCOXlM1gWs+/elPU1payoIFC3j88cd55JFHuP/+++Ner51kl8j7nd2+oV8aGl08/V4bzc0XJ9zm\nTIn3WfUknZ/FcEE7MrmWo+qEEIlyOBy43e7w687OznAh2ej3Ojo6cDgcWK3WMddUVFTgdDrDX7vj\njjvYvHlzQm2oqamJ+V5zc/OI90+fuwS/6aCoMJ9LvT3MnHkFNTXXJfR7Mm30Z9GzVHyW8R4CDDc8\njmTaQohJqK2tZf/+/QC0tLTgcDgoKCgAYNasWXi9Xtrb2/H7/TQ0NFBbWxv1Grvdzle/+lXa2toA\nOHToENdee23S23vgz2eA4V0gZXg8NySUaUdb0qDJumUQkmkLISZh0aJFVFdXs3r1ahRFYdOmTTz3\n3HN4PB6WL1/O5s2bWb9+PQArV67E6XTidDrHXAPw+c9/nm984xsUFBRQWFjIAw88kPwGDyUoZpMC\ngF9qeHJC3KAdbRmEtqQBsmsZxOg5KL8sgRBCTMCGDRtGvO7p6QkPfS5evHjEvS/WNQA333wzP//5\nz1PTyCFajDabQgOmkmnnhrhBO9oyCK/Xi91uz7plEKMfNKUTCyGMSktSzOahTFtGFnNC3KDtdrup\nrq4Ov9aWNNjt9qjLINra2ujq6op6DYSWQTz55JNMnz6d//N//s+I66OZSDXl6E577PjbBC63xfuI\nGaPn6kk9t10II9AGFi3mUKYtNTy5YcLV4+leBjGRaspBXwDqz4RfO53zqLmhasLtTQc9V09mqu3y\noCDEsLGZtgTtXBC3ejzaMoiKioqo72nLIGJdc8stt7BgwQIgtAzi7bffTtoHAdlAXwiRO9Qxc9py\nv8sFcYN2rCUNkH3LIEYfVSdPniITXj3eyX+/8p70P5FSkXuPgxTe5oq4w+PRlkHs27eP4uLirFsG\nMaYQTTqxyIBHnz1Kp6eXfJuFT916daabIwxqONMOBW3JtHNDQnPao5c0zJ8/P/znbFoGIcPjIhvM\nLC+k09PLn992EQgEMZuNt4eRyLzhOW0pRMslhrqbjA7asuRLZEKezRz+s7fPl8GWCCOT6vHcZKig\nrT15KqHRIsm0RUZEHgk76JMbqUgNlZE7osnweG4wVNAOhrf1G6qmlDltkQEjgnbE+e5CJFN4TluW\nfOUUYwXtoT5rkR2CRAb5R2TaErRFagRH7T0ekL3Hc4KxgvaowgyZ0xaZEJldS9AWqTI8HahgMim4\nL/ZluEUiHYwVtIMjnzx9ErRFBowcHpc+KFJDGx5XFAWToowpxBXGZKigPXpbPynMEJngk+FxkQZa\n0DYpoURl9OZSwpgMFbS1OR2LSZZAiMzxjRgelz4oUiPyHAiTSTLtXGGooB0cnWlLJxYZIJm2SAft\nfmdSFEyK3O9yhaGC9ugN9CXTFpkQGbR9suRLpMiIOW0ZHs8Zhgra4UI0mdMWGSSFaCIdIjeTkuHx\n3DHh87SzWVDmtA1j27ZtHD16FEVRqKurY+HCheH3Ghsb2blzJ2azmSVLlrBu3bqY12zcuJGWlhZK\nS0sBuPvuu7nttttS1u5AUB0xTCnD4yJVtF4WzrQlaOcEYwVtqR43hMOHD9Pa2kp9fT0nT56krq5u\nxKE0W7ZsYc+ePVRWVrJmzRpWrFiBx+OJec23vvUtbr/99rS0XRsOVwjdVKUQTaTKiExblnzlDEMH\nbTlfVp+amppYtmwZAPPmzaO7uxuv1wtAW1sbJSUlVFVVAbB06VKamprweDwxr0knbTc0m9XMgC8g\nmbZImeElX4os+cohhprTVodidLgQTeYTdcntdlNWVhZ+XV5ejsvlAsDlclFeXj7mvfGueeqpp/jC\nF77AN7/5TTweT0rbrs1hayd9yZy2SJWgZNo5yZCZtkX24jUUdRIZhHbNpz/9aUpLS1mwYAGPP/44\njzzyCPfff3/c65ubmyf1PV1e/1ADQhl2+5lzNDf3T6Dl2S2RvxeRHmOrx0P9XtGOORSGZKygHa4e\nl0I0PXM4HLjd7vDrzs5OKioq8Hg8Y97r6OjA4XBgtVqjXuN0OsNfu+OOO9i8eXNCbaipqRn3/ebm\n5qjf0955GX55nuKiAi71eiktm05NzQcT+p3ZLtZnNrJsfkgZXT0OoUOSrBYJ2kZmqOHxMafeSCGa\nLtXW1rJ//34AWlpacDgc2O12AGbNmoXX66W9vR2/309DQwO1tbUxr/nqV79KW1sbAIcOHeLaa69N\nadt94Tlt04jXQiRbONNGCQdtOSTJ+AyZaZtMCgqSaevVokWLqK6uZvXq1SiKwqZNm9i3bx+dnZ3U\n1NSwefNm1q9fD8DKlStxOp04nc4x1wB8/vOf5xvf+AYFBQUUFhbywAMPpLTtWpDOs4bmtAekEE2k\nyOjqcQC/TAkanrGC9qjhooBUj+vWhg0bRryeP39+eKhy8eLFI5aAxboG4Oabb+bnP/95ahoZxeig\nLZm2SJXgqOpxkEw7FxhqeHz0cJFfhsdFmmnrtG1DQVuWfIlUiT6nLUHb6AwVtLXh8XCmLR1YpJm2\nxMtiMWFSJGiL1BldPQ5IopIDjBW0w0+eoUPhpQOLdNOGw80mBbPZJDuiiZSJNqctiYrxGStoR2Ta\nZpMiQ0Ui7cJB22zCYjYxKKd8iRSJnmnLPc/ojBW0IzNtGR4XGeAbGg4PZdqKDI+LlFHD52kPz2nL\nhlLGZ6igrW1jahoaLpLlDyLdfEMPihazgsVkkm1MRcpoPUtRFMxDw+OyWsH4DBW0A8PjRZJpi4wY\nntM2YTYr4cxbiKSLUj0uG0oZn6GCdnhzFQVMJqmkFOmnBW2TScFiNjEghWgiRYLR5rRlbwrDM1TQ\njjxYwqSYJNMWaadl1hazEi6GlNOXRCpI9XhuMlTQHs60Fcwm2dJPpJ82p60Nj4NU9IrUGFk9Hvqz\njC4an7GC9uij6oLqpI51FGKyRqzTNsmhISJ1IqvHtb4mD4jGZ7CgHWUDfXnyFGkUOadtlrWzIoWi\nZdpSiGZ8xgrawWjVlHLDFOkTiDhpTuuDkmmLVAgnKUQmKdLXjM5QQVsdtbkKSCcW6aU9JJqGNlcB\nCdoiNaLtiCYnGxqfoYL2iExbhsdFBmgPiaFiSJlnFKkT/ZQvud8ZnbGCdpRMW548RTppc4rmiDlt\nybT1Y9u2baxatYrVq1fz2muvjXivsbGRz3zmM6xatYpHH300oWsOHDjA9ddfn5K2apm2aeiAJJDp\nwFxgyXQDkkmLzwpEFAHJk6dIH21zC9OIoC27ounB4cOHaW1tpb6+npMnT1JXV8c999wTfn/Lli3s\n2bOHyspK1qxZw4oVK/B4PGOuqa+vB2BgYIDHH3+cioqKlLRXZThJCfc1CdqGZ/xMWzqxSCMt0zYp\nijw46kxTUxPLli0DYN68eXR3d9Pb2wtAW1sbJSUlVFVVYTKZWLp0KU1NTVGv8Xq9APzoRz/ic5/7\nHDabLSXtjdi1WbYxzSGGCtojTr2RakqRAVr1uGICU3idtmTaeuB2uykrKwu/Li8vp7u7GwCXy0V5\nefmI91wuV9RrXC4Xp06d4tixY3zyk59MWXujL3GV+53RGWx4PNqctjx5ivTRbppmRarH9W4yGzNp\n1zzwwAPcd999E76+ubk54ff7evsAaGtvw3PZH/pz2xmamy9P+PdmQrzPqifp/CwJBe1t27Zx9OhR\nFEWhrq6OhQsXht9rbGxk586dmM1mlixZwrp16+Jec+DAAb70pS9x/PjxpH6Y6NXjcsMU6RMeHpdC\nNN1xOBy43e7w687OTkpLS6O+19HRgcPhwGq1jrnGZrPx7rvvsmHDhvDX1qxZw1NPPRW3DTU1NTHf\na25uHvF+3u/+Gy55mTtnLnkXeuD4SRwzZ1JTsyDxD50hoz+LnqXis4z3EBB3eDyyOGPr1q1s3bp1\nxPtbtmxh9+7d7N27l4MHD3LixIlxr0llcUa0U29kjkekkz8QxKSMLA6SB0d9qK2tZf/+/QC0tLTg\ncDgoKCgAYNasWXi9Xtrb2/H7/TQ0NFBbWxv1miuvvJLf/e53PPPMMzzzzDM4HI6EAvZEDVePRxTe\nygOi4cXNtGMVWtjt9hHFGUC4OMPj8cS8RivOePjhh5P+YUbsECTVlCIDgkEVszn0LCw7ounLokWL\nqK6uZvXq1SiKwqZNm3juuefweDwsX76czZs3s379egBWrlyJ0+nE6XSOuSZdRp5qKNOBuSJu0Ha7\n3VRXV4dfa4UWdrs9anFGW1sbXV1dUa9xuVwcO3aMr3/966kJ2pFz2rJuUWSAPxgMZz1a8JagrR/a\nkLamp6cnPPS5ePHi8HKu8a4Z7fe//33yGhhBVVUUZAfIXDPhQrR0F2dMpDDj3LmLAHR0nufSpVBh\nxrHjbxO43Dah35kuei7E0HPbUykQGM60ZXhcpFJQDQVskCVfuSRu0I5WnKHNR6ejOGMihRl/PvMG\nvOWlamYVWHqg/SxO5zxqbqiK9zHTTs+FGJlqux4eFPyBIJahqnEpRBOppKoqQzFbCm9zSNxCtGiF\nFna7Hci+4ozIwozwofAyxyPSKBBQw3uOS9AWqaQyNtOWoG18cTPtaMUZ+/bto7i4OOuKM4IRWwTJ\nnLbIhEAwMtOWA0NE6kRm2mYZHs8ZCc1pjy60mD9/fvjP2VScoRWimeTUG5Eh/oBKns0MIJuriJRS\n1eFh8fD9Tg5IMjxDbWMqe4+LTAtEVI/Lki+RSsEombb0NeMzVtAORqzT1gozZE5bpJE/oGKR6nGR\nBmpE9bgE7dxhzKAduW5ROrFIo0AgGB4WN5tknbZIHVVVMWnV4xK0c4ahgnbUo+pkjkekUSCoYjGN\n3hFNTvkSyRcMDmfaWqIiSYrxGSpoR85pmxUpRBPpNyLTNksfFKkTqh5Xwq/NJkUy7Rxg0KCNFKKJ\ntAsGVYIqUdZpS6Ytki+oquH9KGAoaAekrxmdsYJ2tDltyXJEmmhTMTKnLdIhshANQnvdS18zPoMG\n7chTb6QT69G2bdtYtWoVq1ev5rXXXhvxXmNjI5/5zGdYtWoVjz76aELXHDhwgOuvvz6lbdYeEEdX\nj8uNVKRCqBBNhsdzzYQPDMlmw9uYSqatZ5HnsZ88eZK6uroRG/hs2bKFPXv2UFlZyZo1a1ixYgUe\njyfmNak8wz2Sdixi5DptkyJLvkRqRK7TBgnaucJYmXaU87RlTlt/Yp3hDow4w91kMoXPcB/vGu0M\nd5vNltJ2a31Ny7QBLBaz3EhFSkTuiAahaRnpa8ZnrKAd5TxtyXL0x+12U1ZWFn6tnccORD3D3eVy\nxbzm1KlTHDt2jE9+8pMpb7fW17RMG8AqN1KRIsHg6Exb5rRzgaGGxyOrx8Mb6MuOaLqX7jPcIbFj\nQEd/z8We0BnuFy920fpez9BXg3h7enVxrGgijPI5jGBMpm1S8AeCY5aCCWMxVtAOjl3yJU+e+hPr\nDHePx5OWM9xh/HPcIfqZ4ufcPfBf56l0zGDunFDWX/D225hNim7PTo+k5zPgJyubH1KCUdZpQ2jE\nx2oxZ6pZIsUMNTw+vCOaIvs+65ieznCPFB4ej5jTtlpkyFIkX2SCotH6nfQ3YzN8pi1BW39ineHe\n2dlJTU1NVp3hHkmbirFEzGlbzCZ6Ar60t0UYmzYVaDKNzbQlaBubsYJ25Dam0oF1LdoZ7tpQZTad\n4R4pdqYtu1SJ5IqWacuUYG4w1PB45JIvs5zyJdIsEKV6vLfPx8Cg9EGRXFrQHr3kCyRoG52xgnaU\nbUx9Mjwu0iS8uYp55I00qKrhvilEMgQi7nWa4W1zZWTHyAwXtLUkR1EULGY5qk6kT2DUNqYQcSOV\nh0eRROE57SjV45JpG5uhgvboDfQVReFCd38GWyRyyfCcdpQhS59kPyJ5tAfE0duYgjwgGp2hgnZg\n1F68JpMim6uItBmuHo+SaUv2I5IosuhWI0u+coOhgna0Q+HllC+RLtEybcvQnwflRiqSSMu0R5+n\nDeDzSV8zMkMF7WBQJXLzPrNJkQIgkTbDp3wN/7MaXoYjw+MieaJm2tLXcoLxgvaoakoZHhfpMnzK\n18jNVUCGLEVyjbvkS+a0Dc1YQVsdu9mABG2RLtrZ7eYR1eNDw+NSiCaSSJv2G33KF8gDotEZK2iP\nybSV8NyPEKkWLdOW4iCRClEzbVnylROMFbTV0efLypy2SB9/lDntcKYtN1KRRIGoB4ZI0M4Fhgra\nqqqOePI0mWQ3KpE+QVmnLdIkeqYtozq5wFBBOxhUIcqTpyz7EukQLdO2yI5oIgWibWMqKxVyg+GC\ntjx5ikyJPqetFaJJHxTJM7zka/hrckhSbjBU0A6MCdoyxyPSJ3r1uBziIJJveHMVWfKVawwXtJWI\nT6R1aL90YpEG42Xa8uAokina5ioWGVnMCYYL2pJpi0wZt3pchsdFEg0Xog1/TR4Qc4OxgnYgOKZ6\nHKQTi/TQMm2rJaIQTVunHZDhcZE845+nLfc7IzNW0B69ucrQDVOGx0U6aHOJZlOU0R7JtEUSjbeN\n6aDUTxia4YJ2RA0QZkUybZE+WtWuJSLT1h4cZXMVkUzBKJuraKM6smWusRkmaKtDm6goUZ48JdMW\n6aANWVqj7D0u1eMimaINj2sFkAOD0teMzJLpBiRLeLjIJHPaIjO0fjZiyZcUB+nKtm3bOHr0KIqi\nUFdXN+K9xsZGdu7cidlsZsmSJaxbty7qNQsXLuTIkSNs374di8WCzWbj4Ycfpry8PGntHL7fDX9t\nONOWvmZkhsm0/eNsoC+ZtkgHf7QlXyYZstSLw4cP09raSn19PVu3bmXr1q0j3t+yZQu7d+9m7969\nHDx4kBMnTsS85sknn2T79u385Cc/4YMf/CDPPPNMUtsajJJpK4qCxWxiwOdP6u8S2cUwmbZWuTti\nswGpphRpdKbTC8BLR85QkBf6p2WRTFs3mpqaWLZsGQDz5s2ju7ub3t5eANra2igpKaGqqgqApUuX\n0tTUhMfjGXON1+tl165dQGjarqOjg5qamqS2NaCOTVIA8mxmybQNLqGgHW34R5MtQ0bRCjMk0xbp\nFIgyRSMPjvrhdruprq4Ovy4vL6e7uxsAl8s14l5VXl5OW1sbXV1dY65xuVzY7XZeeukltm7dytVX\nX82dd96ZUBuam5sTev/kuz0AeLou0PqeN/y+QoBLl3vj/pxsoIc2JiqdnyVu0I4c/jl58iR1dXXU\n19eH39+yZQt79uyhsrKSNWvWsGLFCjweT9RrtCGj2bNn88gjj/DMM8/wj//4j0n5IIEow+Mypy3S\nKRjeXEV2RDMCVZ346YCR1yxZsoSPfvSj7Nixg8cffzyhe914GXlzc3P4/a5AK7zcRcX0GcydM/ww\nUXzyBAO+QNIz+2SL/Cx6l4rPMt5DQNw57WhDRl5v6MkucsjIZDKFh4xiXbNr1y5mz54dHjKaOXNm\nMj4fECPL0arH5YYp0iB6pi1rZ/XC4XDgdrvDrzs7OyktLY36XkdHBw6HI+o1FRUVvPjii0BonnnF\nihVJz8SiVY8D2KwmqZ8wuLhB2+12U1ZWFn6tDf9A9CEjl8s17jUvvfQSn/jEJ3C73QkPGSVCGwKP\nuo2pDI+LNAhGmWdUFAWTSZHNVXSgtraW/fv3A9DS0oLD4aCgoACAWbNm4fV6aW9vx+/309DQQG1t\nbdRr7HY7u3fv5q233gLg6NGjOJ3OpLY1WvU4gM1qlqBtcBMuREv3kFGiczweb6hisre3h9b3WgG4\n4BkE4NSpVprzPBNud6rpeU5Hz21PlWBQHTE0rrGYFBke14FFixZRXV3N6tWrURSFTZs28dxzz+Hx\neFi+fDmbN29m/fr1AKxcuRKn04nT6RxzDcDWrVv5zne+g9lsJj8/n+3btye1rbEy7TybmUF/MHRM\ncZS+KPQvbtCONfwT7T1tyMhqtcYcMlq+fHl4yGj37t1xG5joHM8Zlxd+eZ7iYjtz58wGQLVeghOn\nmFl1JTU118b9Xemk5zmdTLU92x8UAjFulGazSYbHdWLDhg0jXvf09IT7+uLFi0fU88S6BuDGG2/k\nZz/7WWoaSfTCWwhl2hCajskxTg9xAAAgAElEQVS3GWZxkIgQd3g81vAPZNeQUWCc4XGpHhfpECu7\nMZsU2cZUJFW0wluAvKGgLbuiGVfcR7FoQ0b79u2juLg4q4aMxisCkqFJkQ4xh8fNJnwyzyiSKNrm\nKjActGWttnElNH4yevhn/vz54T9ny5BRIDC2E5sk0xZpFGt43GJR6O2XXapE8gwXPY78emdXaDMY\nmY4xLsNsYxoIasPjw1+TTFukU1CNVYhmkuFKkVSxCtG0fe+lvxmXgYJ2tHXaciyiSJ9gUB0zxwih\nozoDQVVGfETSRDtPG8Cqnakt0zGGZbigPfKoOu2pU4YmReoFgsHow+OS/YgkC8SoHg9n2hK0Dcsw\nawK06vHI4UlL+KlTMhy90cN+96ONV4gGoeynqMCakt8tcku0o4gh4gFRgrZhGTDTHv6adGB90tMR\niZFiLfmyWEJfk34okiXWnHbkA6IwJuNk2lHmeKQD61Oi+91D5o9I1ASDKkGVqJm2VYbHRZLF2lzF\nInPahmecTDswdrjIZFIwKYpkODqjl/3uI4VXL8TYEQ0k0xbJM7xaJsbwuDwgGpZhMu1Y1ZQWsyId\nWOcycURiIlumRn7PwFDdxEB/f3jve02Ptw+A1954k8uu/ITan62yfSvZXBE709YeEKWOx6gME7S1\n5TTK6MIMixxVpzex9rv3eDxp2e8ext/zHsbuv365dxD+4yz2oiLmzpk74nsv9HbyztlzXOW8hpoF\nlQn9/myk5/3yJytbH1KGYnbUJAVgwCcrZozKOMPjweg7BFnMErT1Ri/73UfSzmyXQjSRDtpqmdiF\naJJpG5VhMu3gOHM8crPUl1j73Xd2dlJTU5M1+91H0s5sl3XaIh3C25iOSruk+Nb4DBO0o+2IBqHh\nIm+fPHXqTbT97rWhymzZ7z6SVgg53jpteXgUySLbmOYuwwyP+6McGAJDZxn7ApMqZhIiUX7JtEUa\nxTrlyypTMYZnmKA9vORm5Netsv+4SINEgrYMWYpkiVXDY5a+ZniGCdqxlnyZZbMBkQZa0DZHPTBE\nsh+RXDEzbZmKMTzDBO1o52mDDE2K9PD7o9dUgOyIJpIvdpIimbbRGSdoB6MXAsnQpEiHhOa0pQ+K\nJIl5ypdJQVHkAdHIjBO0Y65blKFJkXq+KKfMaSwWOSJWJNfwkq/oD4myTtu4jBO0x9lcBSRoi9QK\nJFSIJjdSkRzBQPRMG2RvCqMzXtCW4XGRAf7xMm0Z7RFJFs60oxU+muWQJCMzXtCWQjSRAeMVokkf\nFMkWa3MVkK2bjc5AQTvWgSHaki8ZmhSp4w/GzrQVRcFmMckhDiJpgkE16tA4SNA2OuME7YDMaYvM\nGe/AEIA8m5l+ybRFkgz6A5hH7yQ1RBsel10gjck4QTvOukUJ2iKVwku+YqQ/BXkW+gYk0xbJ0T8Q\nwGqJFbRNqOpwnxTGYpygHaN61yqFaCINxlvyBUNBu1+CtkiOAV8gPIo42nCiIkHbiIwTtGOeejNU\nuStDkyKFxitEg+FMW4YsRTIMDPpjZtpW2RfA0IwTtAPRb5qyF69Ih0F/qH/Fyn4K8iwEgio+ObhG\nJEH/4HiZthTfGplxgrZ2yteoRMdqNQPQ2+9Ld5NEDtFGcmIG7fzQ0fUyry2mSnv4G29OGyRRMSoD\nBe3ohWh51tBH7JX5RJFCWs2EtpHKaAV5ErRFcmjD3vGCttTxGJNhgnYwxo5oNotk2iL1tKzGEuNG\nKkFbJEt/nFEdybSNzTBBO1Yh2vDwuNwsReqEg3aMG2lhvhWQfiimrj9upi3Ft0ZmmKA9fDTiyK+b\nTQp5NjM9kmmLFBqME7Ql0xbJEq6fkOHxnGSYoB3rUHiAwjyLZDgipYYL0eLMaUs/FFPUPxDqa9YY\nfU2CtrEZJmgHgiomJfoG+oX5VpnTFimlLa+Jl2n3SqYtpkgbHreYzVHfl1PljM04QTsQxBRjL96i\nAgs9fXKzFKkz4PNjNilRHxpBhsdF8miFaLLkKzcZJmj7g2p4U4HRCvOt+ANBfH7pxCI1Bn3BmFk2\nhKZoQIK2mDptyVesOW2zHAVraIYJ2sGAGnPf58s9gwCSbYuUGRgMxJzPBtlcRSRPONOO8ZAou0Aa\nm2GCti8QO9Oxya5oIsUGfIFwhhNNeE5b+qCYonjD49ahDaW0gjVhLJZMNyBZLl7uj/meTXZFEyk2\n4AuQZ41eGARwqOU8IJl2ttu2bRtHjx5FURTq6upGvNfY2MjOnTsxm80sWbKEdevWRb1m4cKFnDt3\njm9/+9v4/X4sFgsPP/wwFRUVSWljeHg8VqY9FMz75cAQQzJOpu0Pxpzj0XZFk7XaIlUGxzkqEcA2\n1DclaGevw4cP09raSn19PVu3bmXr1q0j3t+yZQu7d+9m7969HDx4kBMnTsS85nvf+x533XUXTz31\nFMuXL+fJJ59MWjvjZtpD9zvpa8aUUKYd7UlSky1Pn/5AMOYcjwyPi1QKDh3gMN6ctlWCdtZrampi\n2bJlAMybN4/u7m56e3sBaGtro6SkhKqqKgCWLl1KU1MTHo9nzDVer5dNmzaRl5cHQFlZGS0tLUlr\nZ7wtc+UB0djiZtp6ePoMBlX8AXWcOW0ZHhepMxjnJgqh/QOsFhM9ffLgmK3cbjdlZWXh1+Xl5XR3\ndwPgcrkoLy8f8Z7L5Yp6jcvlorCwELPZTCAQ4Omnn+Yv//Ivk9bOE20XgdiFaFo/lDltY4qbaUd7\n+vR6vdjt9qx5+tSePGMNF2mZtgyPi1SIt++4Js9mxitBWzdUVZ3SNYFAgHvuuYebb76ZW265JaHr\nm5ub477ffekyAB0dZ+nuir3/+IWu7rg/L5OyuW0Tlc7PEjdou91uqqurw6+1J0m73R716bOtrY2u\nrq6o1zidToDw06c2lD5V/XHWLebJoSEihRIO2lazZNpZzOFw4Ha7w687OzspLS2N+l5HRwcOhwOr\n1TrmGm3K79vf/jZz587lK1/5SsJtqKmpiflec3MzNTU12BoaAB9XzZ0Ts/jR/uYxFLN13J+XSdpn\nMYJUfJbxHgImXD2e7qfPRJ48Pd5QMB7o76P1vdYx39M19P6p1naam70TaXpK6flJU89tT7Z4Z2lr\n8qxmLnT3EwjG3lNAZE5tbS27d+9m9erVtLS04HA4KCgoAGDWrFl4vV7a29uZOXMmDQ0N7Nixg66u\nrjHX2O12fvnLX2K1Wvna176W9Hb6/KEtc2MNjwPk55mletyg4gbtaE+f2pNkOp4+E3nyPH3uEnCe\nspJi5s6ZNeb7pl3q5+CbxykumU5NzQcS+r2ppucnzUy1PVsfFAbinG+sybMNTdP0+ZhWZEt5u8TE\nLFq0iOrqalavXo2iKGzatInnnnsOj8fD8uXL2bx5M+vXrwdg5cqVOJ1OnE7nmGsAnn76aQYGBli7\ndi0QmibcvHlzUtrpDwQxmRRM4zz45dssXO7pTcrvE9klbtCO9vRpt9uB7Hn6jLduMTynLUOTIgXi\nHRaikaCd/TZs2DDidU9PT/gBdfHixdTX18e9BuBnP/tZahpIKNMeL8uG0GY+fQN+VFWNuR++0Ke4\nQTva0+e+ffsoLi7OmqfP/jjny4aXfMkSCJECAz7toTHe8Hjon5u3bxAoSnWzhEH5A8GYRbeagjwL\nQRUG/cFxN/0R+pPQnPboJ8n58+eH/5wNT58DcfbitZgVTAr0SqYtUiDhTHvo5untlX4oJs+XQCDO\nzwu93z/gl6BtMIbYxlQruIj19KkoClarmR6pHtcNPWzoo5nIki9Aln2JKfH5gxQVWMf9nsijYEvs\neelolkgTQ2xjGm94HEJZTp+s09YFPWzoE2kggf4HI+e0hZgMVVXH3f1RU2CTU+WMyliZ9nh7P1tN\ncjSnTsTa0AeyaztJzaA/8SVfIJm2mLx4G0lp8ocybdkVzXgMEbQTyXRsFjPuAVkjqwexNvSB6NtJ\npmJDn0SWl2nfc/Ld0A5VHs8FWtVLMb+/63LoofHEqTaam2N/XzbL1mV3uSLRUZ3I4XFhLIYI2vEO\nhYfhCvK+AT/2OPNBIrtkYjvJeOvQI9eq//boIaCbK2ZWMruyOOY1xZf6aXxL2y/g/Qm1I5voeW+B\nycq2h5RE7nUwXIjWJxusGI5B5rTH38YU5KQvPZnMhj7jXTOZ7SQnQtuhKvHq8cGUtEMYXyL3Ohie\n0+6XTNtwDBG04y35AjnpS09qa2vZv38/wLgb+vj9fhoaGqitrY15TSq3k9QEAlrQHn/aJX+oEO2y\nBG0xSYnc6wAK8mV43KgMMTye6Jw2SOWuHsTa0Kezs5Oampqs2NAnkm8oaJvj3EjNZhOF+Ra6vRK0\nxeQkmmkXF4R23LvcI33NaAwRtOOt04bhoUl58tSHaBv6aPOL2bChT6RAIDR/Hm94HKCkKI9LPQOp\nbpIwqHCmHSdov/FuaKrotZNu/jblrRLpZIjh8cQK0ULvSaYtks2f4PA4wDS7jW7v4KSK64RIvBBN\nhseNyhBBe2AwgALjnnoTPjRECtFEkoWDdpzsB0KZdiCoysOjmJSBCRai9UkNj+EYImj7AkHMZmXc\n02y0IiDZ91kkm38iw+P20Fxjt8w1iknoT3B43GRSyLeZJdM2IGMEbV8AsyneFpKhJ0+p3BXJFj7f\nOIEjELV9oLu9Mq8tJq4/wbPbYfh4TmEsxgja/uC4Q+MgmbZIHX8gmNB8NkRk2lJBLiYhkaJbTUGe\nhf7BQHhJojAGYwTtBG6aebJGVqRIKGgn9k9pWlEo05YKcjEZia7ThuGtTC/JPc9QjBG0/cG4w+M2\nqxlFkcMaRPL5A2rCQbs0PDwuN1IxcYmcaKgJB23pa4ZinKAdJ9M2KQpF+VbJtEXS+f2JZ9ot2vrZ\nd1ypbJIwqPDmKhPItC9K/YShGCdoJ3ByV3GhTea0RdJNZE5bu5H2SoGQmASfL/E9AbStTC9elqBt\nJIYI2n5/IKGgbS+0ymENIqmCQZVAMPHh8cJ8K4oitRVicrQ9AeJNB8LwVqbnPT0pbZNIL90H7UAg\nSFAFUwKd2F5gZdAfDB8kL8RUDfoTX4IDofWz9gIrl2XER0yCFrTjrZYBKJsWqp9o7/CmtE0ivXQf\ntAf92mENiQ2PgxyNKJJn0JfYsZyRigtt9PT5wkd6CpGo4Uw7gftdkQ2zSaGt83KqmyXSSPdBW7vx\nJTo8DkiWI5Jm+IS5xOa0Yfjh8UJ3X0raJIxL230vgX18MCkKpcV5tHV4CQZlr3ujMEDQTnx4UrtZ\nynyiSJaJDo8DFA89PHZ29aakTcK4tN33xtuyOVL5tHwGfQFcF+UB0SgMELQTz7TbO0NzOy8daU9p\nm0Tu0DLteGdpRyouCj08dnrkRiomxh9IbKWMpqw4H4C2DhkiNwrDBO1ECjMKh5ZA9PTJchuRHIM+\nbYeqiQ+PuyTTFhPkT2DL5kjatrkdHulrRqH7oB0uzEgg09HmtHv6ZHhcJMdkMu1pQ5n2GZcsxRET\n4w8EEzqYRqM9IHZK0DYM3QftCRWiFWhz2lKIJpJjYBJz2tOKbFgtJk6f605Vs4RB+QPqhIbHtamY\n107IDnxGofugrQ1PJtKRiwpCmbbsPy6SRet/EwnaiqIwfVo+7Z3ecCGlEInQCtESVZRvwaQoXOqR\ne55R6D5oD2fa8T+K2aRQlG+hR4K2SJLwkq8JzGkDTC8tIBBU+dkLx1PRLGFQEy1EUxQFe6GcuWAk\n+g/agcQ3VwEoKgjtPy7rFkUyTCbTBphREqrqdXf3J71NwrgmWogGoemYvgG/7ARpEPoP2hOY04bQ\nGtmgqtItJ9+IJNBuhIkclRhpRmkBAG5ZPysmwB+c2Jw2yGoFozFO0E4w07EPzWu7ZTcqkQThoJ3A\n9Eyk8qFMW3ZFExMxmUx7eDMf6WtGoPug7fcnXogGw9WU5y/IU6eYusnOadssZkrsNtwX+1FVmaoR\n8QXV0IlyE820tQLcrksyFWMEug/aEx0eL7GHTr4565aTb8TUhQ8MmeDwOMCMkgIGfAHcF+VmKuIL\nDp0vM9FMuzA/FLQ9ErQNwThBO8FMp1QL2rKxhUiCgUkWosHwvPYpWa8tEhAYKp6dyOYqEFr2BdB1\nWep4jED3QXtwAku+IDQ8rihwzi1BW0zdZKvHAaYPzWufOitBW8Q3tFAm4XudpnBoeNwjKxUMQfdB\ne6LD42aTwrQiG2dcMjwupm44055Y9gMRmfbZS0ltkzCmcKY9weHxgjwLCjI8bhQGCNoTK0SD0BD5\npZ5BvLLhgJii4UK0if9TshdYybOaOS2ZtkiAFrQnWohmUhQK8i10XZagbQQGCNoTW/IFw/Pa7ZJt\niymayvC4oihML8nnrLuH/gE5eU6Mb7KFaBCqIPd0y0oFI9B90PZPcHgchtfItp6TYUkxNQO+ACZF\nmdSNFEJD5KoKreelL4rxTXZ4HKAo38qgP0hPvzwc6p3ug/ZEtzGF0FIbgNMylyimaNAXmNR8tma4\nGE36YjbYtm0bq1atYvXq1bz22msj3mtsbOQzn/kMq1at4tFHH417zY9//GOqq6vp6UlO0etkh8cB\nCocqyD2ymY/uJRS0s7kjT+TAEE3ZtHwUoPl4Z1LaIHLXwGBgQlMzow0Xo8m8dqYdPnyY1tZW6uvr\n2bp1K1u3bh3x/pYtW9i9ezd79+7l4MGDnDhxIuY1v/jFL7hw4QIOhyNp7QtMYXg8fK627Iqme5Z4\n3xDZKU+ePEldXR319fXh97ds2cKePXuorKxkzZo1rFixAo/HE/WaVHTkiVaPA1gtJkqK87jQ3Yeq\nqigTXPcohGbQF8A6iY1VNOXT8jGZFMm0s0BTUxPLli0DYN68eXR3d9PbG9o5sa2tjZKSEqqqqgBY\nunQpTU1NeDyeMdd4vV6WLVuG3W7nV7/6VdLaN9l12gAl9lDQlqWu+hc3aEfryF6vF7vdnhUdOXye\n9gSHKGeU5HPi8gCurj4c5YVJa4/ILQO+wKR2Q9NYzCaurLBz+twlgkF10nPjYurcbjfV1dXh1+Xl\n5XR3h0ZAXC4X5eXlI95ra2ujq6trzDUulwun0zmpNjQ3N8d8Twvaly930/rexFa+9HpDc9lH3zrF\nFYVdk2pbso33WfUmnZ8lbtCO1pFdLhd2uz0tHTneX8YFT6gDnjnTPqG5RSUYWv7QcPAI11yRP+F2\nJYOeO62e255MA74gBXlx/xmN6+orSmjruExnVy8zpxclqWViqiZTaT3V6uyampqY75349UEAysvK\nmDunckI/t3LQz/+82ULQXDTu70iX5ubmrGhHMqTis4x3f53w3SbdHXm8v4zm5mYKi4qBAZxz504o\nSxmgi3fOvkdB6UxqauZNun2TpedOm462b9u2jaNHj6IoCnV1dSxcuDDckRsbG9m5cydms5klS5aw\nbt26mNdAqI7ioYce4vDhwxQVJS8oqqrKoG9qc9oAg0N7DZw62y1BO4McDgdutzv8urOzk9LS0qjv\ndXR04HA4sFqtY66pqKhISfumUoiWb7NgL7By/oIMj+td3LtNtI6sdcpYHXm8a5LNN3RU3USHFcuK\nQ2u1z3TKWu1sk+0FQRptC13rFIP2DKkgzwq1tbXs378fgJaWFhwOBwUFoULBWbNm4fV6aW9vx+/3\n09DQQG1tbdRr7HZ7Sto3lXXaEKogP+vqIRiUtdp6FjfTrq2tZffu3axevXpMp4zsyDNnzqShoYEd\nO3bQ1dUV85pk8/knVwhUogVt2WAl68Sqo4DsKAjSTLaeYrTpQxXk756RCvJMWrRoEdXV1axevRpF\nUdi0aRPPPfccHo+H5cuXs3nzZtavXw/AypUrcTqdOJ3OMdcA/PCHP6SxsRGXy8Xf/d3f8YEPfIB7\n7rlnSu2bSqYNoRMOO7v6ZBpG5+IG7Wgded++fRQXF2dFR/b5g5PKdGwWM0UFVsm0s1CsOgpIX0FQ\nIqayhWmkonwrBXkWTslmPxm3YcOGEa97enrCU0GLFy8esXIm1jUAX/7yl/nyl7+c1LZNZXMVgMry\nQt5pu8hrJ9wStHUsoTnt0Z1y/vz54T9nuiP7/MFJL7kptedxxuWlf8BP/hSLiUTqZKIgKJFCu1eP\nHAWgv7+X1vdap/T7ivKg09PLwaY/kW/L3j2PpAAxc4ZP+Zpc0J4zsxiOwqvHOvn4h+cmsWUinXQf\nqXyBKQTt4lDQPu/p5aqqaUlumZisWDURHo8nbQVB8Qrtmpubufb6BfDrDkqnFTN3zqxJ/y6A9otn\ncV9yUeJwcsO8GVP6Wami5+LJycqmh5SpZtql9jyKC238+R0XgUBwygWUIjN0/1/N2+tj0Bec1LXT\nhnYJ6pCKyqwyXnFPNhQEaQamcFjIaJVDewUcb82ONbQi+0w1aCuKwuxKOz19Pk7LVIxu6T7TDgSD\nmM2T+xjFRUNB29ObzCaJKYpVR9HZ2UlNTU3G6yg0Uznha7SqoTnGN095+Jsp/zRhROHhcWXy/W3m\n9CLePOXhrdMe5s0qTVLLRDrpP2gH1EnP8UyToJ21otVRaEOVma6j0PRrhWhT2BFNU1RgZVqRjaMn\nXLIzmohqqpk2DD8cvnXaw6duvTop7RLppevhcVVVCQTVCR0WEkmCtpgK7Qzsqew9Hmnm9CIGBgOy\nDFFEFZxiIRqE9iCfVmTjrdOeJLVKpJuug7a2R8Bk18nm28wU5JklaItJ0TLtZAXtqhnaEPmFpPw8\nYSzJyLQVRWHBVeW4uvpwX5QTv/RI10HbH5jaZgOKolCYb+WMyzvlJUIi9/QPDmXaSarCjZzXFmK0\nZARtgAVXhfY5kGxbn3QdtId3CJr8xygtzsPnD3L+gmTbYmL6B5KbaZdPyyPPauYtCdoiikCou01p\neBygu2cAgOebTk+tQSIjdB20/VonnsI2klfMCC0Lev2kO853CjGSlmknoxANQiM/M6cXcu5CD12X\n+pPyM4VxBNTkZNqOskJMiiKHh+iUvoP2FPfiBbiyIjQkKUFbTNTwnLY5aT8zPK8tQ5dilKlOB2os\nZhMVZQW4LvaFiymFfug6aAcCU3/yLJ+WT77NzOsn3DKvLSYkXD2exJ2lhue1pRhNjOTzh+5PydgX\nYOb0IlQV3mm7OOWfJdJL10E7nGlPoRMrisKcmdO40N3P2+/JblQiccmuHgdwlBdiMSsyry3G0DLt\npGzmMyO0A9+bp+XhUG90HbS1TNsyxeGia4d2Bjrw57NTbpPIHeHq8SQGbYvZxDWzSjl5ppvefl/S\nfq7QP184aE994x3tlK9jpyVR0RtdB21/EjYbAJg9005RgZWDR8/IELlI2EASd0SLdNOCSoJBlT8e\nOZPUnyv0zR9QUZSpF6JB6DjYaUU23jx1AZ9/cmc3iMzQddDWMu2pnlZjNplYvKASd3e/bKQvEtY3\n4MdmMWFSkrvl6LIPzcFkUvht4yl5iBRhvoCKzWpGSVJ/mzuzmN5+P2++K0PkeqLroO1P0mYDADZr\n6K/i8Jvnp/yzRG7oHwyQZ0v+9v3TSwr4yI1VnDp7iT++2p70ny/0yR9QsSVxpYLzihJA7nl6o+ug\nHUjSEgiA2ZXFKAq8/Po5yW5EQvoH/RTkJe8mGumLn6rGZjWz51ct4blzkdt8fpU8a/Ju2VdUFGG1\nmPjDq+1yz9MRXQft4c1Vpv4x8m0W5s6cxon2btlGUiSkfyA1mfbzTac5cryTG+dN5+LlAXbVH0n6\n7xD64w+o5NmS95BoNpmYM7OYSz2DvNdxOWk/V6SWvoN2EjZXibToegcAP294Jyk/TxjbQAozbYD3\nX1uBxaxw5LhLioUE/b5geJlhsjirpgFwuEWGyPVC10E7kOSgXTWjiGtnl9J8rJOuy7KNpIgtGFQZ\n9AfJT0GmrSnIs/A+53S8fT6Z2xYEgslZox1p7sxpKMDLb5xL6s8VqaProJ3MzQY0M6cXEgyqPP6f\nryftZwrjGRzqe8kcrozmA9dVYFIU9v3hHZl3zGGBoIqqJj9o5+dZmOWw8/Z7FznrlnPc9UDXQTtZ\nR9VFunZ2GSYFjrfKpgMitkFfqO8VpDDTBigutHHN7FLaOrz863+9kdLfJbLXoG9oT4AkbKwy2nVz\nywD4Q7OM5uiBroO2P0lH1UUqyLMwZ+Y0XBf7eO+8rNkW0WlzzKnOtAFunDcdkENtctlw0E7+Lfvq\nK0qwmE386sC7BAJSO5HtdB20w3PaSX76vG5O6Mnz96+0JfXnCuMYHDq8IT8vtZk2QGV5ITNK82k9\nd4lu70DKf5/IPgNDQTsZK2VGs1nNLLiqDG+fjyaZ2856ug7aw0fVJfdjOK+YRr7NzPNNp/H2yf7P\nYqx+XygjOedO/ZnEiqIwf245QRX+eESGMHNRKofHARZeUwHA0/uPyUqFLGeQoJ3cjmwxm/jgdQ56\n+v385uC7Sf3Zwhh6B0I3tvw0DI8DXDu7FEWR0Z9cNTj0kJiK4XGA0uI8brh6Om0dXp78dQvBoBQ9\nZitdB21t+iXZw+MAN8ybjsVs4r/+eFI6sBijTwvaaRgeByjMtzJ35jROtnfTKvvj55xUZ9oAN99Q\nxZUVdn514F2eev6tlP0eMTW6DtqpGh6H0DzPtbNLudzr4/UTUgAkRuodTG+mDXD9UJXv7/70Xtp+\np8gOqZzT1uTZzKy4eS7Timz8/PfvcLxVdobMRroO2gNDcy+2JO7HG2nBVeUA/HT/sXDRmxAQkWmn\neMlXpKuqplFWnMdvm07juSSb/+SSVFaPRyrIs3BHzWxU4MF//5MUPmYhXQdt7cZps6Ym26maUcS8\nWSW8ddrDz38vW5uKYeme04bQDftvV8xnYDDA9392hN5+KZLMFcNz2qkbHtdc6bDzoffNxN3dz/+7\n51DKf5+YGH0H7cEgeVZz0s8zjrTkA1cyoySfn/z2LV55qyNlv0foS7rntDVqUGVOZTGvHu/kC9/Z\nz3d/2szLb5yT9bUGN5CmTFuzaL6DitIC3n6vi7fkAKWsouug3T8YTPnmFoX5Vu5YPAeTovDos0cl\nuxFAaE7bpIDNkt5/QhFJ340AABRJSURBVCaTwsraq/hw9UzybWb+8Go7W588zP/9Ly9wou1iWtsi\n0mcwDXPakUyKwq0fuAKAp184lpbfKRKj66DdNxhMy/BkRWkBi66vwH2xj/9v76tSTS7oGwiSZ7Og\npHCUJxazycRNCyr5/Ir5fOaOa3mfs5yuywPc88gB9r98Wh4sDSgd1eOjXTHDziyHnT+/7ZJRxiyi\n26A94AvgD6RnG0mAm95XycJrZvDyG+d5/uXTafmdInv1pumBcTyKolBZXsjtNbP5i1onNouJR/7j\nKKvve44N33+JlncvZLR9InnCw+MpWCkzno/cWIXFbOK7P22mVbZ1zgq6Ddre3kEA8qzpmVM0m0ws\nmu/AajHx5K9a8Fzql3nEHBUIqvQPpvZYzom6qmoaf3XbNXzwugqqphdx/L0u6n54UM5JNohUb64S\nS0VZIUs/eCXePh/3PvI/PPrsUS5096W1DWIkHQft0BBgOrOdonwrNy2opH8wwP/1nf18tu437H3h\nuAyX55iePh+qCvl5mc20Ryux5/GRhVfwV7ddw6eXzMNqMfHgj//EaydcmW6amKLegdD9zprmGgqA\n+VeVc3vNLAKBIM83nearOxrkMKUM0m3Qvqxl2mkeovzgdRXcNN9BUYEVq8XE0/uP8eCP/0SHpzet\n7RCZo61dzaZMe7RZDjsf/9BcAkGVLf92SDbK0LnOoftLcZEtI7//fc7p/O+/rObW91/B5V4f67//\nEg8/9QrtnZcz0p5clr13nTi0gzzSHbQVReHDN1Tx4Ruq6B/089zB0zS9fo5X3urgr2+/hvdfU0FF\nWQEzpxdFvb713CUuDT1wCH1673zoRlVWnJfhloxvzsxiPv7hObzwciubn3iZbf9Ui/OKkkw3S0zC\n+Qu9mE3pHVkcTVEU3n9tBRaziSNvd/LSkTO8/MZ5Vi+/jrLifN7nLOeKCvu4P+NYq4dz7h6mpanN\nRqTfoD0U+DKZ7eTbLPyv2+bxznsXOfJ2J/Uvvk39i29jMimsXn49q5dfN6K6+On9x9j7wnEA1tw+\ng5pMNVxMyemhvb+nlxRkuCXxzbuylNtrgvz3K218Y+cfmF5awOWeQeZWTeNvP349NfMrw9/77plu\nXjzUSlGBlf+1dB72QhuBQJBH/uPPBFX4sFNqODJBVVU6PL0U5pkyslphtOqrp/M+Zzkn2i/S0NzO\nj58L7VNutZi4+y+rWVnrjNrOt9/r4v/ZdQCAtXfI/W+ydBu0L/dmJtMezaQoXD+3DOeV03jtHTf9\nA37ePdvN0/uP4erqZc0nF1A+LZ+XjrSHAzbA4eNeVn0qgw0Xk3b6XDcA00vzM9ySxMy/qhyb1czh\nN8/TP+CnqMDK261dbH7iZf6i1skXP/U+3n6viy3/doi+gVCV8q/+511WLbuOhub28EPK8VNWPrw4\nmLa1wiLkcq+PvgE/laXZc7tWFIVrZ5dxZYWdtk4vff1+Xj3eyY/+83UOvnaOmxZU8sHrKxgYDPDu\n2W76B/z8dP/weu8Xj3Tz2ZVqVjyE6E1CvWDbtm0cPXoURVGoq6tj4cKF4fcaGxvZuXMnZrOZJUuW\nsG7dupjXnDt3jnvuuYdAIEBFRQUPP/wwNtvk5mi04fH8FG1hOlE2i5mbFoSylvdfV8Gv/uddXjz8\nHn88coY5lXZOnunGajHxV0vn8ccjZ3j7bC/nL/TEHEbPddnY5zSnz12iMM9EYZp3Q5uKq68s4eor\nh4fGXRf7ePFwK785eIo/vXmerssDqCosWzyHy72D/OmtDp789ZsAzL+qjIHBAKfOXmLfH07w2Y9d\nl6mPkXKj+1CkTPW7Dk/ozPaCvOx7WCrMt3L9nNBBNvNmlfD7V9p4/aSb10+6efLXI79XUWD5h+bQ\n0++j8bVznDzTzTWzSjPQan2L2wsOHz5Ma2sr9fX1bN26la1bt454f8uWLezevZu9e/dy8OBBTpw4\nEfOaXbt28bnPfY6nn36auXPn8uyzz0664drxhAX52XfjLC60sWrZ9dy2aBaFeRZOtHfjKCvkr5bO\no6KskBvnTQfg+abTGW1ntsrWPgfgudTP+Qu9OEqtus4SKkoLuOtj17Hwmhm4LvZhs5pZ+ZGruH5u\nGTctqORzH7+e9Z+vYdf62/jYTXO446bZ2PNNPL3/uGErh7O132lFroVZGLQjFRfa+PSSeXxh5QKW\nLZ7NgqvKufGaGXxs8Ww+cctc1n5yAdfNKaOkKFQL8u+/eTPDLdanuBGvqamJZcuWATBv3jy6u7vx\ner3Y7Xba2tooKSmhqqoKgKVLl9LU1ITH44l6zaFDh/jOd74DwO23386//du/8bnPfW7CjX7z1AUO\ntZynqsxKqT07i4HMJoXqq6dTffV0VHXkMNC8WaW8dKSdX//PKapm2JlWZKOyvJD8PDM+X5BBf4BB\nX5CCPAsldhuKohAMqgSC6tD/BwkGVVRtpZky/H+Rv0f7o6IohL+qgPYqMuaM9z2jv+9yX4CuS/0x\nf9ZUmE1KzD4HZKzPBQJBXBf7ePLXLQDcMCf757PjsZhNfPQDV1K78ApMppH/AUvsefQP+Dne2gWE\n6jc+9aEyfvbSBTY98TJfuvMGSovzUJTQFFGonwz/v/a1yPdMpsivDX0dBZNp+FqToqCq4PMH8QUC\nDAyG/ocCVrMJq8WEzx+kfzCA2axgs5ixWkJft1nNmE0KQTX0byMYVAmqavjfivbnAV8Any+I1WIi\nz2Ymz2ZmWpEtar/r7Q0FzEz1O29fKCsFKMqyJYaxFBfauH5uOdfPLY/6/tyZxVjMcOy0h1fe6mB6\nST7+QBCLOfTfQztLwmRSsJhNmE3Z/XDc0x+Y9Glo04psE374jxu03W431dXV4dfl5eW4XC7sdjsu\nl4vy8vIR77W1tdHV1RX1mr6+vvAQ0fTp03G5Jr5+9PCb5/mXoZNnPny9XRfZzug2Wswmrr0yn5bW\nPh75jz9nqFVT9J/nUvaji7tO8L+j9B8gI30O4P7Hm3ht6Fz1a2aXsmheERf8k/pRWWd0wI7F2x/g\n5htm8vIb53nwx39KcavSqyDPzFXBjjF9qLs7VL+QiX73TlsX9+w+gD+gcu3sUmaUGGM/CLPZxILZ\nBbx+uo/v/OvLmW5Ocuyb3P3wk7dcxT995v0TumbCY8uqOvGOE+2aRH9Oc3PziNdmYPPnZkV8RZ9b\nNX62djqfrc10K7LTE0/4OHnyJGVlobmyy5cv09LSEs5yEjGVPgdj+91ffyifv/7QrBFfq7Dps+9N\nVsU1oeU8n1g4K8536tMTT3jG9LuJSna/u2/VlaO+wxh97m8+Mp2/+UimW5EN/GP+m8cTN2g7HA7c\nbnf4dWdnJxUVFVHf6+jowOFwYLVao15TWFhIf38/+fn54e8dT02NLArIRY2NjZSUlIT/+/f29nLb\nbbdht9tpb29PaZ8D6Xe5Kla/g9Tf60D6nUhM3MqG2tpa9u/fD0BLSwsOhwO7PfTEPWvWLLxeL+3t\n7fj9fhoaGqitrY15zUc+8pHw11944QU++tGPpupzCR2TPicyQfqd0ANFTWDsZseOHbzyyisoisKm\nTZt48803KS4uZvny5fzpT39ix44dAHz84x/n7rvvjnrN/Pnz6ezs5N5772VgYIArrriCBx54AKvV\nmtpPKHRJ+pzIBOl3ItslFLSFEEIIkXnZvfBPCCGEEGEStIUQQgidyL7txBI03jaXmfb222/zT//0\nT3zxi19kzZo1Mbc0/OUvf8m///u/YzKZuOuuu/jsZz+Lz+dj48aNnD17FrPZzAMPPMDs2bPT0u7t\n27fT3NyM3+/nH/7hH7jxxht10e50y+a+N1l67bNToZf+rrf+ZqS+lJV9RNWhQ4cOqX//93+vqqqq\nnjhxQr3rrrsy3KJhPT096po1a9T77rtP/clPfqKqqqpu3LhRfe6551RVVdXvfve76k9/+lO1p6dH\n/fjHP65eunRJ7evrU//iL/5C7erqUvft26du3rxZVVVVPXDggPr1r389Le1uampSv/SlL6mqqqoe\nj0ddunSpLtqdbtnc9yZLr312KvTS3/XW34zUl7K1j+hyeHy8bS4zzWaz8cQTT4xYl3no0CE+9rGP\nAaEtDZuamjh69Cg33ngjxcXF5Ofns2jRIl599VWamppYvnw5AB/5yEd49dVX09LuxYsX8/3vfx+A\nadOm0dfXp4t2p1s2973J0mufnQq99He99Tcj9aVs7SO6DNputzu8axGM3OYy0ywWC/n5I49sjLal\nodvtHrMt4uivm0yh83MHBwdT3m6z2UxhYSEAzz77LEuWLNFFu9Mtm/veZOm1z06FXvq73vqbkfpS\ntvYRXQbt0VQdrVqL1daJfj1Vfve73/Hss89y//33J9SObGl3puTC5zTyf3u99Xc9/d1Gk61/r+PJ\ntj6iy6A93taq2Ujb0hCGtz+M9hm0r2tP0j6fD1VVp3z+c6IOHDjAj370I5544gmKi4t10+500lvf\nm6xc+G+vh/5uhP6WjX+vicrGPqLLoD3edoPZKNqWhu9///t5/fXXuXTpEj09Pbz66qvcdNNN1NbW\n8vzzzwPQ0NDAhz/84bS08fLly2zfvp3HHnuM0tJS3bQ73fTW9ybL6P/t9dLfjdDfsvHvNRHZ2kd0\nuyNatK0Ds8Ebb7zBQw89xJkzZ7BYLFRWVrJjxw42btw4ZkvD559/nj179qAoCmvWrOHOO+8kEAhw\n3333cfr0aWw2Gw8++OCETrearPr6enbv3o3T6Qx/7cEHH+S+++7L6nZnQrb2vcnSa5+dCj31dz31\nNyP1pWztI7oN2kIIIUSu0eXwuBBCCJGLJGgLIYQQOiFBWwghhNAJCdpCCCGETkjQFkIIIXRCgnYG\nbN++nbVr13LXXXdxww03sHbtWtauXcsvfvGLhK5//PHH+cMf/pDaRoqs9vnPf57f/e53I77W39/P\n4sWLOXfuXNRr1q5dS2NjI2+99Rb/8i//Mub91tZW7rjjjnF/b19fHy+88AIAL730Ej/84Q8n+QlE\nLpnqPU/zyiuv0N7enqJW6kRSjh0Rk9LW1qZ+9KMfzXQzhA7t27dPXbdu3Yiv/fKXv1TvvvvumNes\nWbNGPXjwYMz3T58+rd5+++3j/t5XXnlFXb9+/cQaK8SQqd7zNm7cqL788stJbJH+6PY8bSPavXs3\n7e3tnD17lnvvvZf+/n527NiBzWajv7+fTZs2UV1dzcaNG6mpqeGWW27hy1/+MrfeeiuvvfYaPT09\nPPbYY1RWVmb6o4gU+8QnPsFDDz1EV1dX+ECJX/ziF3z2s5/lxRdf5F//9V+x2WwEAgG2b9/OrFmz\nwtceOnSI733ve+zdu5dXX32VTZs2UV5eTnV1dfh7Tp48yaZNmzCbzXi9Xr7xjW+wePFi/vmf/5lL\nly6xfft2rrnmGhobG9mxYwdHjx7lwQcfxGKxoCgK999/P9dccw1r167llltu4ciRI5w+fZqvfvWr\n3HnnnWn/+xLZaXBwkM2bN9PW1kZPTw933nknX/ziFzl27BibN2/GarUyMDDAV/7/9u4vpOkuDOD4\n15WjXeQy3M12E8kEodQcjOqmqDDBIVgjb5qiYzEkavRnYQtpyKLtol0ZhKKbP3dhBGGyyiC8cwu6\nUIjIyIso6MKFazJh6fC9CMfr66s3b2/b9PlcbeecPZwzDnv2O7/tnMuXWVpa4tWrV7x//x6Px4PZ\nbM539/NClscLzNevXxkeHubQoUMkk0nu3r3L8PAwbW1tPHr0aEP7ubk5zp07RyQSobq6mhcvXuSh\n1+JP02g0NDQ0EI1GgV/7HX/48IFTp06RSqUIBoMoisKJEyeIRCKbxgkEAty4cYNwOLxuT+tEIsHV\nq1cJh8PcuXOHYDDInj17uHTpEsePH8ftdq+L43a76e7uRlEUOjo68Hq9ubqlpSX6+/vx+XwMDAz8\n5ndCFLOhoSEMBgOKovD48WPGxsb49OkTo6OjNDQ0oCgKfX19LCws0NjYSFVVFbdv396xCRtArrQL\nTG1tLSUlJQBUVFQQCATIZDIsLi6i1Wo3tC8vL8doNAKg1+tJJpN/tL8if6xWK16vl4sXL/Ls2TMs\nFgtqtZqKigpu3brF6uoq8/PzHDlyZNMYs7OzmEwmAI4ePYqiKADodDoCgQDBYJDl5eUt51UqleL7\n9+/U1NQAYDabuXbtWq5+7QNWr9fz48eP/zxusX28efOGRCJBPB4HYGVlhc+fP3P27Fk8Hg9fvnzh\n5MmTsjrzN5K0C0xpaWnusdvtxuv1cuzYMSYnJxkcHNzQfteuXeuer8qutDtGTU0NP3/+ZG5ujrGx\nMR48eMDy8jIul4unT59y4MABRkZGePfu3ZZxVKpfC27ZbDZX1tvbS1NTE1arlY8fP+J0Ojd9/dqX\nzDX/nIO7d+/etE7sbGq1mitXrnDmzJkNdePj48RiMZ48eUI0GiUQCOShh4VHlscLWCKRwGg0ks1m\nefnyZd4OgxeF6/z58zx8+BCNRoPRaCSdTqNSqTAYDGQyGV6/fr3lvKmsrGR6ehqAqampXPna3AN4\n/vx5LoZKpWJlZWVdjL1796LT6ZiZmQEgFotRV1f3W8cptqf6+vrcLb1sNovP5yOVShEKhUgkEpw+\nfRqfz5ebW/82/3YaSdoFzOFw0N7ejtPppKWlhW/fvhEKhfLdLVFAmpubmZiYwGq1ArBv3z4sFgtW\nqxWXy4Xdbicej2/6W4ebN29y7949HA4H6XQ6V97Z2Ynb7cZut2MymdBqtdy/f5/Dhw/z9u1buru7\n18Xx+/34/X5sNhsjIyP09PT8f4MW24bNZqO0tJTW1lYuXLjA/v37KSsr4+DBg7hcLmw2G11dXVy/\nfh34dTSmx+PZ8HfHnURO+RJCCCGKhFxpCyGEEEVCkrYQQghRJCRpCyGEEEVCkrYQQghRJCRpCyGE\nEEVCkrYQQghRJCRpCyGEEEVCkrYQQghRJP4CA/J3dSw7oWgAAAAASUVORK5CYII=\n","text/plain":["<matplotlib.figure.Figure at 0x7f089a217240>"]},"metadata":{"tags":[]}}]},{"metadata":{"id":"yrcpGKbqfzQN","colab_type":"code","colab":{}},"cell_type":"code","source":["TEXT.build_vocab(train_data, max_size=25000, vectors=\"glove.6B.100d\"f)\n","LABEL.build_vocab(train_data)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"jMyUyOJy4vF_","colab_type":"text"},"cell_type":"markdown","source":["##Hierarchical Attention Network"]},{"metadata":{"id":"GoHW3I3W477S","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch\n","import torch.nn as nn\n","import torch.nn.functional as F"],"execution_count":0,"outputs":[]},{"metadata":{"id":"k6iDhhU9gUBW","colab_type":"code","colab":{}},"cell_type":"code","source":["class MLP(nn.Module):\n","    def __init__(self,hidden_size, bidirectional=False):\n","        super().__init__()\n","        if  bidirectional:\n","            bi_flag = 2\n","        else:\n","            bi_flag = 1\n","            \n","        self.feedMLP = nn.Linear(bi_flag*hidden_size,bi_flag*hidden_size)\n","        self.tanh = nn.Tanh()\n","    def forward(self, x):\n","        #x = [N, ..., emd_dim]\n","        e = self.feedMLP(x)                    # e = [N, ..., bi_flag*hidden_size]\n","        u = self.tanh(e)                       # u = [N, ..., bi_flag*hidden_size]\n","        return u"],"execution_count":0,"outputs":[]},{"metadata":{"id":"3aqdwbUoaQsm","colab_type":"code","colab":{}},"cell_type":"code","source":["class SenAtten(nn.Module):\n","    def __init__(self, hidden_size, num_layers, bidirectional=False):\n","        super().__init__()\n","        if bidirectional:\n","            bi_flag = 2\n","        else:\n","            bi_flag = 1\n","        \n","        self.rnn = nn.GRU(input_size = bi_flag*hidden_size,hidden_size=hidden_size,\n","                           num_layers=num_layers, bidirectional=bidirectional, batch_first=True,dropout=0.5)\n","        self.MLP = MLP(hidden_size=hidden_size, bidirectional=bidirectional)\n","        self.fc1 = nn.Linear(bi_flag*hidden_size,1)\n","        self.context = nn.Linear(bi_flag*hidden_size,1,bias = False)\n","    def forward(self,x):\n","        #x = [batch_size, sen_len, num_direction*hidden_size]\n","        s = self.rnn(x)[0]\n","        \n","        u = self.MLP(x)                         # u = [batch_size, sen_len, num_directions*hidden_size]\n","        c = self.context(u)                     # c = [batch_size, sen_len, 1]\n","        \n","        a = F.softmax(c, dim=1)                 # a = [batch_size, sen_len, 1]\n","        \n","        s = s*a                                 # s = [batch_size, sen_len, num_direction*hidden_size]\n","        s = s.sum(dim=-2)                       # s = [batch_size, num_direction*hidden_size]\n","        s = self.fc1(s)                         # s = [batch_size, 1]\n","        \n","        return s   "],"execution_count":0,"outputs":[]},{"metadata":{"id":"Mn3Q5o-JDIIf","colab_type":"text"},"cell_type":"markdown","source":["#### Hierarchy introduced by rehshaping input tensor\n","**Caution** :  In order to used this method the inputs need to be feed into network with same sequence length, either by *padding* or by *clipping* to a defined length. Also the *sentence length* and *num of sentence* pararmeters need to be factors of the define length. "]},{"metadata":{"id":"Fag78x9Hao69","colab_type":"code","colab":{}},"cell_type":"code","source":["class WordAtten(nn.Module):\n","  \n","    sen_len = 50\n","    doc_len = 18\n","    def __init__(self,vocab_size, embedding_dim, hidden_size, num_layers, bidirectional=False):\n","        super().__init__()\n","        if bidirectional:\n","            bi_flag = 2\n","        else:\n","            bi_flag = 1\n","        self.embedding = nn.Embedding(vocab_size,embedding_dim,)\n","        self.rnn = nn.GRU(input_size=embedding_dim, hidden_size=hidden_size,\n","                           num_layers=num_layers, bidirectional=bidirectional, batch_first=True,dropout=0.5)\n","        \n","        self.MLP = MLP(hidden_size=hidden_size, bidirectional=bidirectional)\n","        \n","        self.context = nn.Linear(bi_flag*hidden_size,1,bias = False)\n","        \n","        self.fc1 = nn.Linear(bi_flag*hidden_size,1)\n","        \n","    def forward(self,x):\n","        #x = [sen_len, word_len, batch_size]\n","                \n","        x = x.permute(2,0,1)                    # x = [batch_size, sen_len, word_len]\n","        \n","        embedded = self.embedding(x)            # embedding = [batch_size, sen_len, word_len, emd_dim]\n","        k= embedded.view(-1,50,100)\n","        \n","        h = self.rnn(k)[0]\n","        h_c = h.contiguous()\n","        h_reshaped = h_c.view(-1,18,50,100)     # h = [batch_size, sen_len, word_len, num_directions*hidden_size]\n","        \n","        u = self.MLP(h_reshaped)                # u = [batch_size, sen_len, word_len, num_directions*hidden_size]\n","        c = self.context(u)                     # c [batch_size, sen_len, word_len, 1]\n","        a = F.softmax(c, dim=-2)                # a [batch_size, sen_len, word_len, 1]\n","        \n","        s = h_reshaped*a                        # s = [batch_size, sen_len, word_len, num_direction*hidden_size]\n","        s = s.sum(dim=-2)                       # s = [batch_size, sen_len, num_direction*hidden_size]\n","        \n","        return s  "],"execution_count":0,"outputs":[]},{"metadata":{"id":"_G-8tbMPDZZv","colab_type":"text"},"cell_type":"markdown","source":["#### Hierarchy introduced by spiltting the tensor into fixed sentence sizes using torch.split() function"]},{"metadata":{"id":"kcdxVK5UWV5K","colab_type":"code","colab":{}},"cell_type":"code","source":["class WordAtten2(nn.Module):\n","  \n","    def __init__(self,vocab_size, embedding_dim, sentence_length, hidden_size, num_layers, bidirectional=False):\n","        super().__init__()\n","        if bidirectional:\n","            bi_flag = 2\n","        else:\n","            bi_flag = 1\n","        self.senlen = sentence_length\n","        self.embedding = nn.Embedding(vocab_size,embedding_dim)\n","        self.rnn = nn.GRU(input_size=embedding_dim, hidden_size=hidden_size,\n","                           num_layers=num_layers, bidirectional=bidirectional, batch_first=True,dropout=0.5)\n","        \n","        self.MLP = MLP(hidden_size=hidden_size, bidirectional=bidirectional)\n","        \n","        self.context = nn.Linear(bi_flag*hidden_size,1,bias = False)\n","        \n","        self.fc1 = nn.Linear(bi_flag*hidden_size,1)\n","        \n","    def forward(self,x):\n","        #x = [sen_len, batch_size]\n","                \n","        x = x.permute(1,0)                      #x = [batch_size, sen_length]\n","        \n","        embedded = self.embedding(x)            # embedding = [batch_size, sen_len, emd_dim]\n","        rnn_outputs = []\n","        split_inputs = torch.split(embedded,self.senlen,1)\n","        for inputs in split_inputs:\n","            \n","            h = self.rnn(inputs)[0]\n","            h_c = h.contiguous()\n","            \n","            u = self.MLP(h)                # u = [batch_size, sen_len, word_len, num_directions*hidden_size]\n","            c = self.context(u)                     # c [batch_size, sen_len, word_len, 1]\n","            a = F.softmax(c, dim=-2)                # a [batch_size, sen_len, word_len, 1]\n","        \n","            s = h*a                        # s = [batch_size, sen_len, word_len, num_direction*hidden_size]\n","            s = s.sum(dim=-2)                       # s = [batch_size, sen_len, num_direction*hidden_size]\n","            s = s.unsqueeze(1)\n","            rnn_outputs.append(s)\n","        rnn_outputs = tuple(rnn_outputs)\n","        output = torch.cat(rnn_outputs,dim=1)\n","        return output  "],"execution_count":0,"outputs":[]},{"metadata":{"id":"7knwuhgyioJv","colab_type":"code","colab":{}},"cell_type":"code","source":["class HAN(nn.Module):\n","  def __init__(self, vocab_size, embedding_dim, sentence_length, hidden_size, num_layers, bidirectional = False):\n","      super().__init__()\n","      self.word = WordAtten2(vocab_size, embedding_dim, sentence_length, hidden_size, num_layers, bidirectional)\n","      self.sentence = SenAtten(hidden_size, num_layers, bidirectional)\n","      \n","  def forward(self, x):\n","      w = self.word(x)\n","      s = self.sentence(w)\n","      return s\n","    "],"execution_count":0,"outputs":[]},{"metadata":{"id":"LlxS1RCy6Rnt","colab_type":"text"},"cell_type":"markdown","source":["**Initializing hyper-parameters**"]},{"metadata":{"id":"hufhnGPCltJo","colab_type":"code","outputId":"4e1bd3d1-beee-4360-9c62-2c7cba449a56","executionInfo":{"status":"ok","timestamp":1546294032721,"user_tz":-330,"elapsed":1473,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":72}},"cell_type":"code","source":["INPUT_DIM = len(TEXT.vocab)\n","EMBEDDING_DIM = 100\n","HIDDEN_SIZE = 50\n","NUM_LAYERS = 1\n","BIDIRECTIONAL = True\n","BATCH_SIZE = 128\n","SENTENCE_LENGTH = 50\n","\n","model = HAN(INPUT_DIM, EMBEDDING_DIM, SENTENCE_LENGTH, HIDDEN_SIZE, NUM_LAYERS, BIDIRECTIONAL)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:46: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n","  \"num_layers={}\".format(dropout, num_layers))\n"],"name":"stderr"}]},{"metadata":{"id":"GGrD8kK2nrzx","colab_type":"code","outputId":"25b7a60f-989d-4b74-8768-16eebcda495e","executionInfo":{"status":"ok","timestamp":1546294032726,"user_tz":-330,"elapsed":1092,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":139}},"cell_type":"code","source":["pretrained_embeddings  = TEXT.vocab.vectors\n","model.word.embedding.weight.data.copy_(pretrained_embeddings)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["tensor([[ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n","        [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n","        [-0.0382, -0.2449,  0.7281,  ..., -0.1459,  0.8278,  0.2706],\n","        ...,\n","        [ 0.3581, -0.3348, -0.4465,  ..., -0.7128, -0.1006, -0.2799],\n","        [ 0.3896,  0.0446, -0.3506,  ...,  0.1727,  0.0145, -0.9884],\n","        [ 0.0994, -0.7076, -0.0949,  ..., -0.0898, -1.1580, -0.1628]])"]},"metadata":{"tags":[]},"execution_count":53}]},{"metadata":{"id":"tKaF37VSoDeC","colab_type":"code","colab":{}},"cell_type":"code","source":["'''\n","def binary_accuracy(preds,y):\n","    rounded_preds = torch.round(torch.sigmoid(preds))\n","    correct = (rounded_preds==y).float()\n","    acc = correct.sum()/len(correct)\n","    return acc\n","'''\n","def precision_recall_f1(preds, y):\n","    rounded_preds = torch.round(torch.sigmoid(preds))\n","    tp = y*rounded_preds\n","    fp = (y==0).float()*rounded_preds\n","    tn = (y==0).float()*(rounded_preds==0).float()\n","    fn = y*(rounded_preds==0).float()\n","    \n","    return sum(tp), sum(fp), sum(tn), sum(fn)      \n","\n","  \n","def train(model, iterator, optimizer, criterion):\n","        \n","    epoch_loss= 0\n","    epoch_acc = 0\n","    tp = 0\n","    fp = 0\n","    tn = 0\n","    fn = 0\n","        \n","    model.train()\n","        \n","    for batch in iterator:          \n","        optimizer.zero_grad()\n","            \n","        predictions = model(batch.text).squeeze(1)\n","        loss = criterion(predictions, batch.label)\n","        #acc = binary_accuracy(predictions, batch.label)\n","        a, b, c, d = precision_recall_f1(predictions, batch.label)\n","       \n","        loss.backward() \n","        optimizer.step()\n","            \n","        epoch_loss += loss.item()\n","        #epoch_acc +=acc.item()\n","        tp += a.item()\n","        fp += b.item()\n","        tn += c.item()\n","        fn += d.item()\n","            \n","    return epoch_loss/len(iterator), tp, fp, tn, fn\n","      \n","      \n","def evaluate(model, iterator, criterion):\n","    \n","    epoch_loss = 0\n","    epoch_acc = 0\n","    tp = 0\n","    fp = 0\n","    tn = 0\n","    fn = 0\n","    \n","    model.eval()\n","    \n","    with torch.no_grad():\n","    \n","        for batch in iterator:\n","            predictions = model(batch.text).squeeze(1)\n","            \n","            loss = criterion(predictions, batch.label)\n","            \n","            #acc = binary_accuracy(predictions, batch.label)\n","            a, b, c, d = precision_recall_f1(predictions, batch.label)\n","            \n","            tp += a.item()\n","            fp += b.item()\n","            tn += c.item()\n","            fn += d.item()\n","            epoch_loss += loss.item()\n","            #epoch_acc += acc.item()\n","        \n","    return epoch_loss / len(iterator), tp, fp, tn, fn"],"execution_count":0,"outputs":[]},{"metadata":{"id":"jIJ30ljwnxlI","colab_type":"code","colab":{}},"cell_type":"code","source":["import torch.optim as optim\n","device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","\n","optimizer = optim.Adam(model.parameters(),lr=0.005)\n","criterion = nn.BCEWithLogitsLoss()\n","model = model.to(device)\n","criterion = criterion.to(device)\n","\n","\n","train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits((train_data,valid_data,test_data), batch_size=BATCH_SIZE,device=device)\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"HtrNRhoHoVaU","colab_type":"code","outputId":"853a1911-55c6-4adc-fa3a-048538c64114","executionInfo":{"status":"ok","timestamp":1546294258116,"user_tz":-330,"elapsed":183151,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":156}},"cell_type":"code","source":["N_EPOCHS = 3\n","\n","for epoch in tqdm(range(N_EPOCHS)):\n","    model.word.embedding.weight.requires_grad=True\n","    if epoch>0:\n","        model.word.embedding.weight.requires_grad=False\n","        lr = 0.0002\n","        for param_group in optimizer.param_groups:\n","            param_group['lr'] = lr\n","    train_loss, train_tp, train_fp, train_tn, train_fn = train(model, train_iterator, optimizer, criterion)\n","    valid_loss, valid_tp, valid_fp, valid_tn, valid_fn = evaluate(model, valid_iterator, criterion)\n","    train_acc = (train_tp+train_tn)/(train_tp+train_fp+train_fn+train_tn)\n","    valid_acc = (valid_tp+valid_tn)/(valid_tp+valid_fp+valid_fn+valid_tn)\n","    valid_precision = valid_tp/(valid_tp+valid_fp)\n","    valid_recall = valid_tp/(valid_tp+valid_fn)\n","    valid_f1= (2*valid_precision*valid_recall)/(valid_precision+valid_recall)\n","    print('| Epoch: {} | Train Loss: {:0.2f} | Train Acc: {:0.2f} | Val. Loss: {:0.2f} | Val. Acc: {:0.2f} | Val. F1 {:0.2f} |'.format(epoch+1,train_loss, train_acc*100,valid_loss, valid_acc*100, valid_f1))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["\n","  0%|          | 0/3 [00:00<?, ?it/s]\u001b[A\n"," 33%|███▎      | 1/3 [01:17<02:34, 77.32s/it]\u001b[A"],"name":"stderr"},{"output_type":"stream","text":["| Epoch: 1 | Train Loss: 0.37 | Train Acc: 82.45 | Val. Loss: 0.28 | Val. Acc: 88.12 | Val. F1 0.87 |\n"],"name":"stdout"},{"output_type":"stream","text":["\n"," 67%|██████▋   | 2/3 [02:09<01:09, 69.88s/it]\u001b[A"],"name":"stderr"},{"output_type":"stream","text":["| Epoch: 2 | Train Loss: 0.14 | Train Acc: 95.01 | Val. Loss: 0.25 | Val. Acc: 90.22 | Val. F1 0.90 |\n"],"name":"stdout"},{"output_type":"stream","text":["\n","100%|██████████| 3/3 [03:01<00:00, 64.46s/it]\u001b[A\n","\u001b[A"],"name":"stderr"},{"output_type":"stream","text":["| Epoch: 3 | Train Loss: 0.14 | Train Acc: 95.26 | Val. Loss: 0.26 | Val. Acc: 90.14 | Val. F1 0.90 |\n"],"name":"stdout"}]},{"metadata":{"id":"EhJWIH_AsSFc","colab_type":"code","outputId":"4fff7292-c6b9-4ff8-8fae-197fe97dc85c","executionInfo":{"status":"ok","timestamp":1546294317218,"user_tz":-330,"elapsed":14341,"user":{"displayName":"Gaurav Patel","photoUrl":"https://lh3.googleusercontent.com/-OePEQgv0Daw/AAAAAAAAAAI/AAAAAAAAAaI/dyKmC54H6hc/s64/photo.jpg","userId":"02723724286804741615"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["test_loss, test_tp, test_fp, test_tn, test_fn= evaluate(model, test_iterator, criterion)\n","test_acc = (test_tp+test_tn)/(test_tp+test_fp+test_fn+test_tn)\n","test_precision = test_tp/(test_tp+test_fp)\n","test_recall = test_tp/(test_tp+test_fn)\n","test_f1 = (2*test_precision*test_recall)/(test_precision+test_recall)\n","\n","print('| Test Loss: {:0.2f} | Test Acc: {:0.2f} | Test F1: {:0.2f} |'.format(test_loss, test_acc*100, test_f1))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["| Test Loss: 0.26 | Test Acc: 90.04 | Test F1: 0.90 |\n"],"name":"stdout"}]},{"metadata":{"id":"GxO35wKaob7c","colab_type":"code","colab":{}},"cell_type":"code","source":["\n","\n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"5UaB0y5hgFDb","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]},{"metadata":{"id":"UEnwnweUlD1y","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]}]}