HAN yelp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "torch.set_default_tensor_type('torch.cuda.FloatTensor')\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import itertools\n",
    "import more_itertools\n",
    "import numpy as np\n",
    "import pickle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## The dataset is taken from https://github.com/justmarkham/DAT7/blob/master/data/yelp.csv \n",
    "df=pd.read_csv('yelp.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>date</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>type</th>\n",
       "      <th>user_id</th>\n",
       "      <th>cool</th>\n",
       "      <th>useful</th>\n",
       "      <th>funny</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9yKzy9PApeiPPOUJEtnvkg</td>\n",
       "      <td>2011-01-26</td>\n",
       "      <td>fWKvX83p0-ka4JS3dc6E5A</td>\n",
       "      <td>5</td>\n",
       "      <td>My wife took me here on my birthday for breakf...</td>\n",
       "      <td>review</td>\n",
       "      <td>rLtl8ZkDX5vH5nAx9C3q5Q</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ZRJwVLyzEJq1VAihDhYiow</td>\n",
       "      <td>2011-07-27</td>\n",
       "      <td>IjZ33sJrzXqU-0X6U8NwyA</td>\n",
       "      <td>5</td>\n",
       "      <td>I have no idea why some people give bad review...</td>\n",
       "      <td>review</td>\n",
       "      <td>0a2KyEL0d3Yb1V6aivbIuQ</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6oRAC4uyJCsJl1X0WZpVSA</td>\n",
       "      <td>2012-06-14</td>\n",
       "      <td>IESLBzqUCLdSzSqm0eCSxQ</td>\n",
       "      <td>4</td>\n",
       "      <td>love the gyro plate. Rice is so good and I als...</td>\n",
       "      <td>review</td>\n",
       "      <td>0hT2KtfLiobPvh6cDC8JQg</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>_1QQZuf4zZOyFCvXc0o6Vg</td>\n",
       "      <td>2010-05-27</td>\n",
       "      <td>G-WvGaISbqqaMHlNnByodA</td>\n",
       "      <td>5</td>\n",
       "      <td>Rosie, Dakota, and I LOVE Chaparral Dog Park!!...</td>\n",
       "      <td>review</td>\n",
       "      <td>uZetl9T0NcROGOyFfughhg</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6ozycU1RpktNG2-1BroVtw</td>\n",
       "      <td>2012-01-05</td>\n",
       "      <td>1uJFq2r5QfJG_6ExMRCaGw</td>\n",
       "      <td>5</td>\n",
       "      <td>General Manager Scott Petello is a good egg!!!...</td>\n",
       "      <td>review</td>\n",
       "      <td>vYmM4KTsC8ZfQBg-j5MWkw</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id        date               review_id  stars  \\\n",
       "0  9yKzy9PApeiPPOUJEtnvkg  2011-01-26  fWKvX83p0-ka4JS3dc6E5A      5   \n",
       "1  ZRJwVLyzEJq1VAihDhYiow  2011-07-27  IjZ33sJrzXqU-0X6U8NwyA      5   \n",
       "2  6oRAC4uyJCsJl1X0WZpVSA  2012-06-14  IESLBzqUCLdSzSqm0eCSxQ      4   \n",
       "3  _1QQZuf4zZOyFCvXc0o6Vg  2010-05-27  G-WvGaISbqqaMHlNnByodA      5   \n",
       "4  6ozycU1RpktNG2-1BroVtw  2012-01-05  1uJFq2r5QfJG_6ExMRCaGw      5   \n",
       "\n",
       "                                                text    type  \\\n",
       "0  My wife took me here on my birthday for breakf...  review   \n",
       "1  I have no idea why some people give bad review...  review   \n",
       "2  love the gyro plate. Rice is so good and I als...  review   \n",
       "3  Rosie, Dakota, and I LOVE Chaparral Dog Park!!...  review   \n",
       "4  General Manager Scott Petello is a good egg!!!...  review   \n",
       "\n",
       "                  user_id  cool  useful  funny  \n",
       "0  rLtl8ZkDX5vH5nAx9C3q5Q     2       5      0  \n",
       "1  0a2KyEL0d3Yb1V6aivbIuQ     0       0      0  \n",
       "2  0hT2KtfLiobPvh6cDC8JQg     0       1      0  \n",
       "3  uZetl9T0NcROGOyFfughhg     1       2      0  \n",
       "4  vYmM4KTsC8ZfQBg-j5MWkw     0       0      0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## mark the columns which contains text for classification and target class\n",
    "col_text = 'text'\n",
    "col_target = 'cool'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_arr = np.sort(df[col_target].unique()).tolist()\n",
    "classes = len(cls_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## divide dataset in 80% train 10% validation 10% test as done in the paper\n",
    "length = df.shape[0]\n",
    "train_len = int(0.8*length)\n",
    "val_len = int(0.1*length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = df[:train_len]\n",
    "val = df[train_len:train_len+val_len]\n",
    "test = df[train_len+val_len:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_str(string, max_seq_len):\n",
    "    \"\"\"\n",
    "    adapted from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py\n",
    "    \"\"\"\n",
    "    string = BeautifulSoup(string, \"lxml\").text\n",
    "    string = re.sub(r\"[^A-Za-z0-9(),!?\\\"\\`]\", \" \", string)\n",
    "    string = re.sub(r\"\\\"s\", \" \\\"s\", string)\n",
    "    string = re.sub(r\"\\\"ve\", \" \\\"ve\", string)\n",
    "    string = re.sub(r\"n\\\"t\", \" n\\\"t\", string)\n",
    "    string = re.sub(r\"\\\"re\", \" \\\"re\", string)\n",
    "    string = re.sub(r\"\\\"d\", \" \\\"d\", string)\n",
    "    string = re.sub(r\"\\\"ll\", \" \\\"ll\", string)\n",
    "    string = re.sub(r\",\", \" , \", string)\n",
    "    string = re.sub(r\"!\", \" ! \", string)\n",
    "    string = re.sub(r\"\\(\", \" \\( \", string)\n",
    "    string = re.sub(r\"\\)\", \" \\) \", string)\n",
    "    string = re.sub(r\"\\?\", \" \\? \", string)\n",
    "    string = re.sub(r\"\\s{2,}\", \" \", string)\n",
    "    s =string.strip().lower().split(\" \")\n",
    "    if len(s) > max_seq_len:\n",
    "        return s[0:max_seq_len] \n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## creates a 3D list of format paragraph[sentence[word]]\n",
    "def create3DList(df,col, max_sent_len,max_seq_len):\n",
    "    x=[]\n",
    "    for docs in df[col].as_matrix():\n",
    "        x1=[]\n",
    "        idx = 0\n",
    "        for seq in \"|||\".join(re.split(\"[.?!]\", docs)).split(\"|||\"):\n",
    "            x1.append(clean_str(seq,max_sent_len))\n",
    "            if(idx>=max_seq_len-1):\n",
    "                break\n",
    "            idx= idx+1\n",
    "        x.append(x1)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Fix the maximum length of sentences in a paragraph and words in a sentence\n",
    "max_sent_len = 12\n",
    "max_seq_len = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/root1/anaconda3/lib/python3.6/site-packages/bs4/__init__.py:219: UserWarning: \"b'/'\" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.\n",
      "  ' Beautiful Soup.' % markup)\n",
      "/home/root1/anaconda3/lib/python3.6/site-packages/bs4/__init__.py:282: UserWarning: \"http://stevelerer\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  ' that document to Beautiful Soup.' % decoded_markup\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train: 8000\n",
      "x_val: 1000\n",
      "x_test: 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/root1/anaconda3/lib/python3.6/site-packages/bs4/__init__.py:282: UserWarning: \"http://www\" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
      "  ' that document to Beautiful Soup.' % decoded_markup\n"
     ]
    }
   ],
   "source": [
    "## divides review in sentences and sentences into word creating a 3DList\n",
    "x_train = create3DList(train,col_text, max_sent_len,max_seq_len)\n",
    "x_val = create3DList(val, col_text, max_sent_len,max_seq_len)\n",
    "x_test = create3DList(test, col_text, max_sent_len,max_seq_len)\n",
    "print(\"x_train: {}\".format(len(x_train)))\n",
    "print(\"x_val: {}\".format(len(x_val)))\n",
    "print(\"x_test: {}\".format(len(x_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "stoplist = stopwords.words('english') + list(string.punctuation)\n",
    "stemmer = SnowballStemmer('english')\n",
    "x_train_texts = [[[stemmer.stem(word.lower()) for word in sent  if word not in stoplist] for sent in para]\n",
    "         for para in x_train]\n",
    "x_test_texts = [[[stemmer.stem(word.lower()) for word in sent  if word not in stoplist] for sent in para]\n",
    "         for para in x_test]\n",
    "x_val_texts = [[[stemmer.stem(word.lower()) for word in sent  if word not in stoplist] for sent in para]\n",
    "         for para in x_val]\n",
    "\n",
    "## calculate frequency of words\n",
    "from collections import defaultdict\n",
    "frequency1 = defaultdict(int)\n",
    "for texts in x_train_texts:     \n",
    "    for text in texts:\n",
    "        for token in text:\n",
    "            frequency1[token] += 1\n",
    "for texts in x_test_texts:     \n",
    "    for text in texts:\n",
    "        for token in text:\n",
    "            frequency1[token] += 1\n",
    "for texts in x_val_texts:     \n",
    "    for text in texts:\n",
    "        for token in text:\n",
    "            frequency1[token] += 1\n",
    "            \n",
    "## remove  words with frequency less than 5.\n",
    "x_train_texts = [[[token for token in text if frequency1[token] > 5]\n",
    "         for text in texts] for texts in x_train_texts]\n",
    "\n",
    "x_test_texts = [[[token for token in text if frequency1[token] > 5]\n",
    "         for text in texts] for texts in x_test_texts]\n",
    "x_val_texts = [[[token for token in text if frequency1[token] > 5]\n",
    "         for text in texts] for texts in x_val_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = list(more_itertools.collapse(x_train_texts[:] + x_test_texts[:] + x_val_texts[:],levels=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## train word2vec model on all the words\n",
    "word2vec = Word2Vec(texts,size=200, min_count=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec.save(\"dictonary_yelp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "## convert 3D text list to 3D list of index \n",
    "x_train_vec = [[[word2vec.wv.vocab[token].index for token in text]\n",
    "         for text in texts] for texts in x_train_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_vec = [[[word2vec.wv.vocab[token].index for token in text]\n",
    "         for text in texts] for texts in x_test_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_val_vec = [[[word2vec.wv.vocab[token].index for token in text]\n",
    "         for text in texts] for texts in x_val_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/root1/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `syn0` (Attribute will be removed in 4.0.0, use self.wv.vectors instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "weights = torch.FloatTensor(word2vec.wv.syn0).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_size = len(word2vec.wv.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train[col_target].tolist()\n",
    "y_test = test[col_target].tolist()\n",
    "y_val = val[col_target].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Make the the multiple attention with word vectors.\n",
    "def attention_mul(rnn_outputs, att_weights):\n",
    "    attn_vectors = None\n",
    "    for i in range(rnn_outputs.size(0)):\n",
    "        h_i = rnn_outputs[i]\n",
    "        a_i = att_weights[i]\n",
    "        h_i = a_i * h_i\n",
    "        h_i = h_i.unsqueeze(0)\n",
    "        if(attn_vectors is None):\n",
    "            attn_vectors = h_i\n",
    "        else:\n",
    "            attn_vectors = torch.cat((attn_vectors,h_i),0)\n",
    "    return torch.sum(attn_vectors, 0).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "## The word RNN model for generating a sentence vector\n",
    "class WordRNN(nn.Module):\n",
    "    def __init__(self, vocab_size,embedsize, batch_size, hid_size):\n",
    "        super(WordRNN, self).__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.embedsize = embedsize\n",
    "        self.hid_size = hid_size\n",
    "        ## Word Encoder\n",
    "        self.embed = nn.Embedding(vocab_size, embedsize)\n",
    "        self.wordRNN = nn.GRU(embedsize, hid_size, bidirectional=True)\n",
    "        ## Word Attention\n",
    "        self.wordattn = nn.Linear(2*hid_size, 2*hid_size)\n",
    "        self.attn_combine = nn.Linear(2*hid_size, 2*hid_size,bias=False)\n",
    "    def forward(self,inp, hid_state):\n",
    "        emb_out  = self.embed(inp)\n",
    "\n",
    "        out_state, hid_state = self.wordRNN(emb_out, hid_state)\n",
    "\n",
    "        word_annotation = self.wordattn(out_state)\n",
    "        attn = F.softmax(self.attn_combine(word_annotation),dim=1)\n",
    "\n",
    "        sent = attention_mul(out_state,attn)\n",
    "        return sent, hid_state\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "## The HAN model\n",
    "class SentenceRNN(nn.Module):\n",
    "    def __init__(self,vocab_size,embedsize, batch_size, hid_size,c):\n",
    "        super(SentenceRNN, self).__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.embedsize = embedsize\n",
    "        self.hid_size = hid_size\n",
    "        self.cls = c\n",
    "        self.wordRNN = WordRNN(vocab_size,embedsize, batch_size, hid_size)\n",
    "        ## Sentence Encoder\n",
    "        self.sentRNN = nn.GRU(embedsize, hid_size, bidirectional=True)\n",
    "        ## Sentence Attention\n",
    "        self.sentattn = nn.Linear(2*hid_size, 2*hid_size)\n",
    "        self.attn_combine = nn.Linear(2*hid_size, 2*hid_size,bias=False)\n",
    "        self.doc_linear = nn.Linear(2*hid_size, c)\n",
    "    \n",
    "    def forward(self,inp, hid_state_sent, hid_state_word):\n",
    "        s = None\n",
    "        ## Generating sentence vector through WordRNN\n",
    "        for i in range(len(inp[0])):\n",
    "            r = None\n",
    "            for j in range(len(inp)):\n",
    "                if(r is None):\n",
    "                    r = [inp[j][i]]\n",
    "                else:\n",
    "                    r.append(inp[j][i])\n",
    "            r1 = np.asarray([sub_list + [0] * (max_seq_len - len(sub_list)) for sub_list in r])\n",
    "            _s, state_word = self.wordRNN(torch.cuda.LongTensor(r1).view(-1,batch_size), hid_state_word)\n",
    "            if(s is None):\n",
    "                s = _s\n",
    "            else:\n",
    "                s = torch.cat((s,_s),0)\n",
    "\n",
    "                out_state, hid_state = self.sentRNN(s, hid_state_sent)\n",
    "        sent_annotation = self.sentattn(out_state)\n",
    "        attn = F.softmax(self.attn_combine(sent_annotation),dim=1)\n",
    "\n",
    "        doc = attention_mul(out_state,attn)\n",
    "        d = self.doc_linear(doc)\n",
    "        cls = F.log_softmax(d.view(-1,self.cls),dim=1)\n",
    "        return cls, hid_state\n",
    "    \n",
    "    def init_hidden_sent(self):\n",
    "            return Variable(torch.zeros(2, self.batch_size, self.hid_size)).cuda()\n",
    "    \n",
    "    def init_hidden_word(self):\n",
    "            return Variable(torch.zeros(2, self.batch_size, self.hid_size)).cuda()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "## converting list to tensor\n",
    "y_train_tensor =  [torch.cuda.FloatTensor([cls_arr.index(label)]) for label in y_train]\n",
    "y_val_tensor =  [torch.cuda.FloatTensor([cls_arr.index(label)]) for label in y_val]\n",
    "y_test_tensor =  [torch.cuda.FloatTensor([cls_arr.index(label)]) for label in y_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_seq_len = max([len(seq) for seq in itertools.chain.from_iterable(x_train_vec +x_val_vec + x_test_vec)])\n",
    "max_sent_len = max([len(sent) for sent in (x_train_vec + x_val_vec + x_test_vec)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_sent_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile(np.array([len(seq) for seq in itertools.chain.from_iterable(x_train_vec +x_val_vec + x_test_vec)]),90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile(np.array([len(sent) for sent in (x_train_vec +x_val_vec + x_test_vec)]),90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Padding the input \n",
    "X_train_pad = [sub_list + [[0]] * (max_sent_len - len(sub_list)) for sub_list in x_train_vec]\n",
    "X_val_pad = [sub_list + [[0]] * (max_sent_len - len(sub_list)) for sub_list in x_val_vec]\n",
    "X_test_pad = [sub_list + [[0]] * (max_sent_len - len(sub_list)) for sub_list in x_test_vec]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_data(batch_size, review, targets, sent_attn_model, sent_optimizer, criterion):\n",
    "\n",
    "    state_word = sent_attn_model.init_hidden_word()\n",
    "    state_sent = sent_attn_model.init_hidden_sent()\n",
    "    sent_optimizer.zero_grad()\n",
    "            \n",
    "    y_pred, state_sent = sent_attn_model(review, state_sent, state_word)\n",
    "\n",
    "    loss = criterion(y_pred.cuda(), torch.cuda.LongTensor(targets)) \n",
    "\n",
    "    max_index = y_pred.max(dim = 1)[1]\n",
    "    correct = (max_index == torch.cuda.LongTensor(targets)).sum()\n",
    "    acc = float(correct)/batch_size\n",
    "\n",
    "    loss.backward()\n",
    "    \n",
    "    sent_optimizer.step()\n",
    "    \n",
    "    return loss.data[0],acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "hid_size = 100\n",
    "embedsize = 200 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_attn = SentenceRNN(vocab_size,embedsize,batch_size,hid_size,classes)\n",
    "sent_attn.cuda()\n",
    "sent_attn.wordRNN.embed.from_pretrained(weights)\n",
    "torch.backends.cudnn.benchmark=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 1e-3\n",
    "momentum = 0.9\n",
    "\n",
    "sent_optimizer = torch.optim.SGD(sent_attn.parameters(), lr=learning_rate, momentum= momentum)\n",
    "\n",
    "criterion = nn.NLLLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import math\n",
    "\n",
    "def timeSince(since):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_batch(x,y,batch_size):\n",
    "    k = random.sample(range(len(x)-1),batch_size)\n",
    "    x_batch=[]\n",
    "    y_batch=[]\n",
    "\n",
    "    for t in k:\n",
    "        x_batch.append(x[t])\n",
    "        y_batch.append(y[t])\n",
    "\n",
    "    return [x_batch,y_batch]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validation_accuracy(batch_size, x_val,y_val,sent_attn_model):\n",
    "    acc = []\n",
    "    val_length = len(x_val)\n",
    "    for j in range(int(val_length/batch_size)):\n",
    "        x,y = gen_batch(x_val,y_val,batch_size)\n",
    "        state_word = sent_attn_model.init_hidden_word()\n",
    "        state_sent = sent_attn_model.init_hidden_sent()\n",
    "        \n",
    "        y_pred, state_sent = sent_attn_model(x, state_sent, state_word)\n",
    "        max_index = y_pred.max(dim = 1)[1]\n",
    "        correct = (max_index == torch.cuda.LongTensor(y)).sum()\n",
    "        acc.append(float(correct)/batch_size)\n",
    "    return np.mean(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_early_stopping(batch_size, x_train, y_train, x_val, y_val, sent_attn_model, \n",
    "                         sent_attn_optimiser, loss_criterion, num_epoch,\n",
    "                         print_loss_every = 50, code_test=True):\n",
    "    start = time.time()\n",
    "    loss_full = []\n",
    "    loss_epoch = []\n",
    "    acc_epoch = []\n",
    "    acc_full = []\n",
    "    val_acc = []\n",
    "    epoch_counter = 0\n",
    "    train_length = len(x_train)\n",
    "    for i in range(1, num_epoch + 1):\n",
    "        loss_epoch = []\n",
    "        acc_epoch = []\n",
    "        for j in range(int(train_length/batch_size)):\n",
    "            x,y = gen_batch(x_train,y_train,batch_size)\n",
    "            loss,acc = train_data(batch_size, x, y, sent_attn_model, sent_attn_optimiser, loss_criterion)\n",
    "            loss_epoch.append(loss)\n",
    "            acc_epoch.append(acc)\n",
    "            if (code_test and j % int(print_loss_every/batch_size) == 0) :\n",
    "                print ('Loss at %d paragraphs, %d epoch,(%s) is %f' %(j*batch_size, i, timeSince(start), np.mean(loss_epoch)))\n",
    "                print ('Accuracy at %d paragraphs, %d epoch,(%s) is %f' %(j*batch_size, i, timeSince(start), np.mean(acc_epoch)))\n",
    "        \n",
    "        loss_full.append(np.mean(loss_epoch))\n",
    "        acc_full.append(np.mean(acc_epoch))\n",
    "        torch.save(sent_attn_model.state_dict(), 'sent_attn_model_yelp.pth')\n",
    "        print ('Loss after %d epoch,(%s) is %f' %(i, timeSince(start), np.mean(loss_epoch)))\n",
    "        print ('Train Accuracy after %d epoch,(%s) is %f' %(i, timeSince(start), np.mean(acc_epoch)))\n",
    "\n",
    "        val_acc.append(validation_accuracy(batch_size, x_val, y_val, sent_attn_model)) \n",
    "        print ('Validation Accuracy after %d epoch,(%s) is %f' %(i, timeSince(start), val_acc[-1]))\n",
    "    return loss_full,acc_full,val_acc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/root1/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:19: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss after 1 epoch,(0m 29s) is 3.108306\n",
      "Train Accuracy after 1 epoch,(0m 29s) is 0.486375\n",
      "Validation Accuracy after 1 epoch,(0m 30s) is 0.628125\n",
      "Loss after 2 epoch,(1m 0s) is 2.364582\n",
      "Train Accuracy after 2 epoch,(1m 0s) is 0.625000\n",
      "Validation Accuracy after 2 epoch,(1m 1s) is 0.636458\n",
      "Loss after 3 epoch,(1m 31s) is 1.757459\n",
      "Train Accuracy after 3 epoch,(1m 31s) is 0.624125\n",
      "Validation Accuracy after 3 epoch,(1m 32s) is 0.641667\n",
      "Loss after 4 epoch,(2m 2s) is 1.536340\n",
      "Train Accuracy after 4 epoch,(2m 2s) is 0.626625\n",
      "Validation Accuracy after 4 epoch,(2m 3s) is 0.653125\n",
      "Loss after 5 epoch,(2m 32s) is 1.433756\n",
      "Train Accuracy after 5 epoch,(2m 32s) is 0.627750\n",
      "Validation Accuracy after 5 epoch,(2m 33s) is 0.648958\n",
      "Loss after 6 epoch,(3m 3s) is 1.392134\n",
      "Train Accuracy after 6 epoch,(3m 3s) is 0.623125\n",
      "Validation Accuracy after 6 epoch,(3m 4s) is 0.644792\n",
      "Loss after 7 epoch,(3m 35s) is 1.346634\n",
      "Train Accuracy after 7 epoch,(3m 35s) is 0.628250\n",
      "Validation Accuracy after 7 epoch,(3m 36s) is 0.633333\n",
      "Loss after 8 epoch,(4m 7s) is 1.319090\n",
      "Train Accuracy after 8 epoch,(4m 7s) is 0.625125\n",
      "Validation Accuracy after 8 epoch,(4m 8s) is 0.644792\n",
      "Loss after 9 epoch,(4m 38s) is 1.309785\n",
      "Train Accuracy after 9 epoch,(4m 38s) is 0.636500\n",
      "Validation Accuracy after 9 epoch,(4m 39s) is 0.643750\n",
      "Loss after 10 epoch,(5m 9s) is 1.335532\n",
      "Train Accuracy after 10 epoch,(5m 9s) is 0.620125\n",
      "Validation Accuracy after 10 epoch,(5m 10s) is 0.669792\n",
      "Loss after 11 epoch,(5m 39s) is 1.276084\n",
      "Train Accuracy after 11 epoch,(5m 39s) is 0.629125\n",
      "Validation Accuracy after 11 epoch,(5m 40s) is 0.657292\n",
      "Loss after 12 epoch,(6m 10s) is 1.275306\n",
      "Train Accuracy after 12 epoch,(6m 10s) is 0.630000\n",
      "Validation Accuracy after 12 epoch,(6m 11s) is 0.647917\n",
      "Loss after 13 epoch,(6m 40s) is 1.279388\n",
      "Train Accuracy after 13 epoch,(6m 40s) is 0.622500\n",
      "Validation Accuracy after 13 epoch,(6m 41s) is 0.646875\n",
      "Loss after 14 epoch,(7m 11s) is 1.291924\n",
      "Train Accuracy after 14 epoch,(7m 11s) is 0.619250\n",
      "Validation Accuracy after 14 epoch,(7m 12s) is 0.639583\n",
      "Loss after 15 epoch,(7m 42s) is 1.257676\n",
      "Train Accuracy after 15 epoch,(7m 42s) is 0.633750\n",
      "Validation Accuracy after 15 epoch,(7m 43s) is 0.638542\n",
      "Loss after 16 epoch,(8m 12s) is 1.262287\n",
      "Train Accuracy after 16 epoch,(8m 12s) is 0.630750\n",
      "Validation Accuracy after 16 epoch,(8m 13s) is 0.612500\n",
      "Loss after 17 epoch,(8m 43s) is 1.247166\n",
      "Train Accuracy after 17 epoch,(8m 43s) is 0.636500\n",
      "Validation Accuracy after 17 epoch,(8m 44s) is 0.634375\n",
      "Loss after 18 epoch,(9m 13s) is 1.268824\n",
      "Train Accuracy after 18 epoch,(9m 13s) is 0.629250\n",
      "Validation Accuracy after 18 epoch,(9m 14s) is 0.639583\n",
      "Loss after 19 epoch,(9m 44s) is 1.262401\n",
      "Train Accuracy after 19 epoch,(9m 44s) is 0.620375\n",
      "Validation Accuracy after 19 epoch,(9m 45s) is 0.651042\n",
      "Loss after 20 epoch,(10m 14s) is 1.246277\n",
      "Train Accuracy after 20 epoch,(10m 14s) is 0.636875\n",
      "Validation Accuracy after 20 epoch,(10m 15s) is 0.648958\n",
      "Loss after 21 epoch,(10m 45s) is 1.269834\n",
      "Train Accuracy after 21 epoch,(10m 45s) is 0.620875\n",
      "Validation Accuracy after 21 epoch,(10m 46s) is 0.653125\n",
      "Loss after 22 epoch,(11m 16s) is 1.237262\n",
      "Train Accuracy after 22 epoch,(11m 16s) is 0.633875\n",
      "Validation Accuracy after 22 epoch,(11m 17s) is 0.638542\n",
      "Loss after 23 epoch,(11m 46s) is 1.246036\n",
      "Train Accuracy after 23 epoch,(11m 46s) is 0.631375\n",
      "Validation Accuracy after 23 epoch,(11m 47s) is 0.626042\n",
      "Loss after 24 epoch,(12m 17s) is 1.259667\n",
      "Train Accuracy after 24 epoch,(12m 17s) is 0.625875\n",
      "Validation Accuracy after 24 epoch,(12m 18s) is 0.618750\n",
      "Loss after 25 epoch,(12m 47s) is 1.258397\n",
      "Train Accuracy after 25 epoch,(12m 47s) is 0.626750\n",
      "Validation Accuracy after 25 epoch,(12m 48s) is 0.614583\n",
      "Loss after 26 epoch,(13m 18s) is 1.278833\n",
      "Train Accuracy after 26 epoch,(13m 18s) is 0.617250\n",
      "Validation Accuracy after 26 epoch,(13m 19s) is 0.650000\n",
      "Loss after 27 epoch,(13m 48s) is 1.252806\n",
      "Train Accuracy after 27 epoch,(13m 48s) is 0.627750\n",
      "Validation Accuracy after 27 epoch,(13m 49s) is 0.642708\n",
      "Loss after 28 epoch,(14m 19s) is 1.271457\n",
      "Train Accuracy after 28 epoch,(14m 19s) is 0.613750\n",
      "Validation Accuracy after 28 epoch,(14m 20s) is 0.652083\n",
      "Loss after 29 epoch,(14m 49s) is 1.260413\n",
      "Train Accuracy after 29 epoch,(14m 49s) is 0.628875\n",
      "Validation Accuracy after 29 epoch,(14m 50s) is 0.626042\n",
      "Loss after 30 epoch,(15m 20s) is 1.263353\n",
      "Train Accuracy after 30 epoch,(15m 20s) is 0.627250\n",
      "Validation Accuracy after 30 epoch,(15m 21s) is 0.629167\n",
      "Loss after 31 epoch,(15m 51s) is 1.236014\n",
      "Train Accuracy after 31 epoch,(15m 51s) is 0.629250\n",
      "Validation Accuracy after 31 epoch,(15m 52s) is 0.636458\n",
      "Loss after 32 epoch,(16m 21s) is 1.247655\n",
      "Train Accuracy after 32 epoch,(16m 21s) is 0.627375\n",
      "Validation Accuracy after 32 epoch,(16m 22s) is 0.622917\n",
      "Loss after 33 epoch,(16m 52s) is 1.261320\n",
      "Train Accuracy after 33 epoch,(16m 52s) is 0.626375\n",
      "Validation Accuracy after 33 epoch,(16m 53s) is 0.632292\n",
      "Loss after 34 epoch,(17m 23s) is 1.246664\n",
      "Train Accuracy after 34 epoch,(17m 23s) is 0.623500\n",
      "Validation Accuracy after 34 epoch,(17m 24s) is 0.626042\n",
      "Loss after 35 epoch,(17m 53s) is 1.246657\n",
      "Train Accuracy after 35 epoch,(17m 53s) is 0.628500\n",
      "Validation Accuracy after 35 epoch,(17m 54s) is 0.630208\n",
      "Loss after 36 epoch,(18m 24s) is 1.248078\n",
      "Train Accuracy after 36 epoch,(18m 24s) is 0.625375\n",
      "Validation Accuracy after 36 epoch,(18m 25s) is 0.598958\n",
      "Loss after 37 epoch,(18m 55s) is 1.264729\n",
      "Train Accuracy after 37 epoch,(18m 55s) is 0.620625\n",
      "Validation Accuracy after 37 epoch,(18m 56s) is 0.634375\n",
      "Loss after 38 epoch,(19m 25s) is 1.224138\n",
      "Train Accuracy after 38 epoch,(19m 25s) is 0.634000\n",
      "Validation Accuracy after 38 epoch,(19m 26s) is 0.648958\n",
      "Loss after 39 epoch,(19m 56s) is 1.254944\n",
      "Train Accuracy after 39 epoch,(19m 56s) is 0.616250\n",
      "Validation Accuracy after 39 epoch,(19m 57s) is 0.641667\n",
      "Loss after 40 epoch,(20m 26s) is 1.229572\n",
      "Train Accuracy after 40 epoch,(20m 26s) is 0.628125\n",
      "Validation Accuracy after 40 epoch,(20m 27s) is 0.610417\n",
      "Loss after 41 epoch,(20m 57s) is 1.252779\n",
      "Train Accuracy after 41 epoch,(20m 57s) is 0.624500\n",
      "Validation Accuracy after 41 epoch,(20m 58s) is 0.637500\n",
      "Loss after 42 epoch,(21m 27s) is 1.252964\n",
      "Train Accuracy after 42 epoch,(21m 27s) is 0.620000\n",
      "Validation Accuracy after 42 epoch,(21m 28s) is 0.634375\n",
      "Loss after 43 epoch,(21m 58s) is 1.241271\n",
      "Train Accuracy after 43 epoch,(21m 58s) is 0.618750\n",
      "Validation Accuracy after 43 epoch,(21m 59s) is 0.670833\n",
      "Loss after 44 epoch,(22m 28s) is 1.240037\n",
      "Train Accuracy after 44 epoch,(22m 28s) is 0.626750\n",
      "Validation Accuracy after 44 epoch,(22m 29s) is 0.660417\n",
      "Loss after 45 epoch,(22m 59s) is 1.257659\n",
      "Train Accuracy after 45 epoch,(22m 59s) is 0.621125\n",
      "Validation Accuracy after 45 epoch,(23m 0s) is 0.625000\n",
      "Loss after 46 epoch,(23m 30s) is 1.236918\n",
      "Train Accuracy after 46 epoch,(23m 30s) is 0.629000\n",
      "Validation Accuracy after 46 epoch,(23m 31s) is 0.632292\n",
      "Loss after 47 epoch,(24m 0s) is 1.250551\n",
      "Train Accuracy after 47 epoch,(24m 0s) is 0.628000\n",
      "Validation Accuracy after 47 epoch,(24m 1s) is 0.621875\n",
      "Loss after 48 epoch,(24m 31s) is 1.234213\n",
      "Train Accuracy after 48 epoch,(24m 31s) is 0.626500\n",
      "Validation Accuracy after 48 epoch,(24m 32s) is 0.641667\n",
      "Loss after 49 epoch,(25m 1s) is 1.222193\n",
      "Train Accuracy after 49 epoch,(25m 1s) is 0.631625\n",
      "Validation Accuracy after 49 epoch,(25m 2s) is 0.626042\n",
      "Loss after 50 epoch,(25m 32s) is 1.239464\n",
      "Train Accuracy after 50 epoch,(25m 32s) is 0.629375\n",
      "Validation Accuracy after 50 epoch,(25m 33s) is 0.621875\n",
      "Loss after 51 epoch,(26m 3s) is 1.278126\n",
      "Train Accuracy after 51 epoch,(26m 3s) is 0.620250\n",
      "Validation Accuracy after 51 epoch,(26m 4s) is 0.638542\n",
      "Loss after 52 epoch,(26m 33s) is 1.242736\n",
      "Train Accuracy after 52 epoch,(26m 33s) is 0.627250\n",
      "Validation Accuracy after 52 epoch,(26m 34s) is 0.622917\n",
      "Loss after 53 epoch,(27m 4s) is 1.225011\n",
      "Train Accuracy after 53 epoch,(27m 4s) is 0.625000\n",
      "Validation Accuracy after 53 epoch,(27m 5s) is 0.642708\n",
      "Loss after 54 epoch,(27m 34s) is 1.251219\n",
      "Train Accuracy after 54 epoch,(27m 34s) is 0.629250\n",
      "Validation Accuracy after 54 epoch,(27m 35s) is 0.630208\n",
      "Loss after 55 epoch,(28m 5s) is 1.228548\n",
      "Train Accuracy after 55 epoch,(28m 5s) is 0.636250\n",
      "Validation Accuracy after 55 epoch,(28m 6s) is 0.641667\n",
      "Loss after 56 epoch,(28m 36s) is 1.249303\n",
      "Train Accuracy after 56 epoch,(28m 36s) is 0.624875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy after 56 epoch,(28m 37s) is 0.618750\n",
      "Loss after 57 epoch,(29m 6s) is 1.246333\n",
      "Train Accuracy after 57 epoch,(29m 6s) is 0.626875\n",
      "Validation Accuracy after 57 epoch,(29m 7s) is 0.647917\n",
      "Loss after 58 epoch,(29m 37s) is 1.245983\n",
      "Train Accuracy after 58 epoch,(29m 37s) is 0.621875\n",
      "Validation Accuracy after 58 epoch,(29m 38s) is 0.679167\n",
      "Loss after 59 epoch,(30m 7s) is 1.250815\n",
      "Train Accuracy after 59 epoch,(30m 7s) is 0.627875\n",
      "Validation Accuracy after 59 epoch,(30m 8s) is 0.638542\n",
      "Loss after 60 epoch,(30m 38s) is 1.265061\n",
      "Train Accuracy after 60 epoch,(30m 38s) is 0.616875\n",
      "Validation Accuracy after 60 epoch,(30m 39s) is 0.633333\n",
      "Loss after 61 epoch,(31m 8s) is 1.204380\n",
      "Train Accuracy after 61 epoch,(31m 8s) is 0.639125\n",
      "Validation Accuracy after 61 epoch,(31m 9s) is 0.653125\n",
      "Loss after 62 epoch,(31m 39s) is 1.261976\n",
      "Train Accuracy after 62 epoch,(31m 39s) is 0.615875\n",
      "Validation Accuracy after 62 epoch,(31m 40s) is 0.607292\n",
      "Loss after 63 epoch,(32m 9s) is 1.231365\n",
      "Train Accuracy after 63 epoch,(32m 9s) is 0.627000\n",
      "Validation Accuracy after 63 epoch,(32m 10s) is 0.632292\n",
      "Loss after 64 epoch,(32m 40s) is 1.267271\n",
      "Train Accuracy after 64 epoch,(32m 40s) is 0.617750\n",
      "Validation Accuracy after 64 epoch,(32m 41s) is 0.647917\n",
      "Loss after 65 epoch,(33m 10s) is 1.230974\n",
      "Train Accuracy after 65 epoch,(33m 10s) is 0.629250\n",
      "Validation Accuracy after 65 epoch,(33m 12s) is 0.600000\n",
      "Loss after 66 epoch,(33m 41s) is 1.244095\n",
      "Train Accuracy after 66 epoch,(33m 41s) is 0.632250\n",
      "Validation Accuracy after 66 epoch,(33m 42s) is 0.608333\n",
      "Loss after 67 epoch,(34m 12s) is 1.237985\n",
      "Train Accuracy after 67 epoch,(34m 12s) is 0.629875\n",
      "Validation Accuracy after 67 epoch,(34m 13s) is 0.652083\n",
      "Loss after 68 epoch,(34m 43s) is 1.237092\n",
      "Train Accuracy after 68 epoch,(34m 43s) is 0.622875\n",
      "Validation Accuracy after 68 epoch,(34m 44s) is 0.618750\n",
      "Loss after 69 epoch,(35m 13s) is 1.251232\n",
      "Train Accuracy after 69 epoch,(35m 13s) is 0.628250\n",
      "Validation Accuracy after 69 epoch,(35m 15s) is 0.640625\n",
      "Loss after 70 epoch,(35m 44s) is 1.233484\n",
      "Train Accuracy after 70 epoch,(35m 44s) is 0.628000\n",
      "Validation Accuracy after 70 epoch,(35m 45s) is 0.630208\n",
      "Loss after 71 epoch,(36m 15s) is 1.240702\n",
      "Train Accuracy after 71 epoch,(36m 15s) is 0.625250\n",
      "Validation Accuracy after 71 epoch,(36m 16s) is 0.616667\n",
      "Loss after 72 epoch,(36m 46s) is 1.242893\n",
      "Train Accuracy after 72 epoch,(36m 46s) is 0.626375\n",
      "Validation Accuracy after 72 epoch,(36m 47s) is 0.633333\n",
      "Loss after 73 epoch,(37m 16s) is 1.233552\n",
      "Train Accuracy after 73 epoch,(37m 16s) is 0.630250\n",
      "Validation Accuracy after 73 epoch,(37m 17s) is 0.636458\n",
      "Loss after 74 epoch,(37m 47s) is 1.247546\n",
      "Train Accuracy after 74 epoch,(37m 47s) is 0.625875\n",
      "Validation Accuracy after 74 epoch,(37m 48s) is 0.662500\n",
      "Loss after 75 epoch,(38m 17s) is 1.242872\n",
      "Train Accuracy after 75 epoch,(38m 17s) is 0.628375\n",
      "Validation Accuracy after 75 epoch,(38m 18s) is 0.635417\n",
      "Loss after 76 epoch,(38m 48s) is 1.262877\n",
      "Train Accuracy after 76 epoch,(38m 48s) is 0.619000\n",
      "Validation Accuracy after 76 epoch,(38m 49s) is 0.618750\n",
      "Loss after 77 epoch,(39m 18s) is 1.248989\n",
      "Train Accuracy after 77 epoch,(39m 18s) is 0.628625\n",
      "Validation Accuracy after 77 epoch,(39m 19s) is 0.645833\n",
      "Loss after 78 epoch,(39m 49s) is 1.256209\n",
      "Train Accuracy after 78 epoch,(39m 49s) is 0.624750\n",
      "Validation Accuracy after 78 epoch,(39m 50s) is 0.622917\n",
      "Loss after 79 epoch,(40m 19s) is 1.241953\n",
      "Train Accuracy after 79 epoch,(40m 19s) is 0.628000\n",
      "Validation Accuracy after 79 epoch,(40m 20s) is 0.627083\n",
      "Loss after 80 epoch,(40m 50s) is 1.228316\n",
      "Train Accuracy after 80 epoch,(40m 50s) is 0.626875\n",
      "Validation Accuracy after 80 epoch,(40m 51s) is 0.640625\n",
      "Loss after 81 epoch,(41m 20s) is 1.247978\n",
      "Train Accuracy after 81 epoch,(41m 20s) is 0.629250\n",
      "Validation Accuracy after 81 epoch,(41m 21s) is 0.638542\n",
      "Loss after 82 epoch,(41m 52s) is 1.232032\n",
      "Train Accuracy after 82 epoch,(41m 52s) is 0.628750\n",
      "Validation Accuracy after 82 epoch,(41m 53s) is 0.658333\n",
      "Loss after 83 epoch,(42m 22s) is 1.244150\n",
      "Train Accuracy after 83 epoch,(42m 22s) is 0.626125\n",
      "Validation Accuracy after 83 epoch,(42m 23s) is 0.647917\n",
      "Loss after 84 epoch,(42m 53s) is 1.254482\n",
      "Train Accuracy after 84 epoch,(42m 53s) is 0.623750\n",
      "Validation Accuracy after 84 epoch,(42m 54s) is 0.648958\n",
      "Loss after 85 epoch,(43m 24s) is 1.244959\n",
      "Train Accuracy after 85 epoch,(43m 24s) is 0.626625\n",
      "Validation Accuracy after 85 epoch,(43m 25s) is 0.619792\n",
      "Loss after 86 epoch,(43m 54s) is 1.234997\n",
      "Train Accuracy after 86 epoch,(43m 54s) is 0.629625\n",
      "Validation Accuracy after 86 epoch,(43m 55s) is 0.636458\n",
      "Loss after 87 epoch,(44m 25s) is 1.199930\n",
      "Train Accuracy after 87 epoch,(44m 25s) is 0.640375\n",
      "Validation Accuracy after 87 epoch,(44m 26s) is 0.616667\n",
      "Loss after 88 epoch,(44m 56s) is 1.237022\n",
      "Train Accuracy after 88 epoch,(44m 56s) is 0.630250\n",
      "Validation Accuracy after 88 epoch,(44m 57s) is 0.645833\n",
      "Loss after 89 epoch,(45m 27s) is 1.224268\n",
      "Train Accuracy after 89 epoch,(45m 27s) is 0.636750\n",
      "Validation Accuracy after 89 epoch,(45m 28s) is 0.616667\n",
      "Loss after 90 epoch,(45m 57s) is 1.262825\n",
      "Train Accuracy after 90 epoch,(45m 57s) is 0.621000\n",
      "Validation Accuracy after 90 epoch,(45m 58s) is 0.650000\n",
      "Loss after 91 epoch,(46m 28s) is 1.239101\n",
      "Train Accuracy after 91 epoch,(46m 28s) is 0.623875\n",
      "Validation Accuracy after 91 epoch,(46m 29s) is 0.630208\n",
      "Loss after 92 epoch,(46m 59s) is 1.224268\n",
      "Train Accuracy after 92 epoch,(46m 59s) is 0.635750\n",
      "Validation Accuracy after 92 epoch,(47m 0s) is 0.630208\n",
      "Loss after 93 epoch,(47m 29s) is 1.239489\n",
      "Train Accuracy after 93 epoch,(47m 29s) is 0.624125\n",
      "Validation Accuracy after 93 epoch,(47m 30s) is 0.647917\n",
      "Loss after 94 epoch,(48m 0s) is 1.220662\n",
      "Train Accuracy after 94 epoch,(48m 0s) is 0.639250\n",
      "Validation Accuracy after 94 epoch,(48m 1s) is 0.644792\n",
      "Loss after 95 epoch,(48m 31s) is 1.246375\n",
      "Train Accuracy after 95 epoch,(48m 31s) is 0.627375\n",
      "Validation Accuracy after 95 epoch,(48m 32s) is 0.621875\n",
      "Loss after 96 epoch,(49m 1s) is 1.265341\n",
      "Train Accuracy after 96 epoch,(49m 1s) is 0.618750\n",
      "Validation Accuracy after 96 epoch,(49m 2s) is 0.632292\n",
      "Loss after 97 epoch,(49m 32s) is 1.237104\n",
      "Train Accuracy after 97 epoch,(49m 32s) is 0.625875\n",
      "Validation Accuracy after 97 epoch,(49m 33s) is 0.616667\n",
      "Loss after 98 epoch,(50m 3s) is 1.259590\n",
      "Train Accuracy after 98 epoch,(50m 3s) is 0.627500\n",
      "Validation Accuracy after 98 epoch,(50m 4s) is 0.636458\n",
      "Loss after 99 epoch,(50m 33s) is 1.248772\n",
      "Train Accuracy after 99 epoch,(50m 33s) is 0.621875\n",
      "Validation Accuracy after 99 epoch,(50m 34s) is 0.634375\n",
      "Loss after 100 epoch,(51m 4s) is 1.251115\n",
      "Train Accuracy after 100 epoch,(51m 4s) is 0.623375\n",
      "Validation Accuracy after 100 epoch,(51m 5s) is 0.612500\n",
      "Loss after 101 epoch,(51m 34s) is 1.235592\n",
      "Train Accuracy after 101 epoch,(51m 34s) is 0.629125\n",
      "Validation Accuracy after 101 epoch,(51m 35s) is 0.610417\n",
      "Loss after 102 epoch,(52m 5s) is 1.232759\n",
      "Train Accuracy after 102 epoch,(52m 5s) is 0.632500\n",
      "Validation Accuracy after 102 epoch,(52m 6s) is 0.625000\n",
      "Loss after 103 epoch,(52m 35s) is 1.244601\n",
      "Train Accuracy after 103 epoch,(52m 35s) is 0.621625\n",
      "Validation Accuracy after 103 epoch,(52m 36s) is 0.600000\n",
      "Loss after 104 epoch,(53m 6s) is 1.217572\n",
      "Train Accuracy after 104 epoch,(53m 6s) is 0.631125\n",
      "Validation Accuracy after 104 epoch,(53m 7s) is 0.623958\n",
      "Loss after 105 epoch,(53m 36s) is 1.258042\n",
      "Train Accuracy after 105 epoch,(53m 36s) is 0.621625\n",
      "Validation Accuracy after 105 epoch,(53m 37s) is 0.640625\n",
      "Loss after 106 epoch,(54m 7s) is 1.253184\n",
      "Train Accuracy after 106 epoch,(54m 7s) is 0.623375\n",
      "Validation Accuracy after 106 epoch,(54m 8s) is 0.628125\n",
      "Loss after 107 epoch,(54m 37s) is 1.246893\n",
      "Train Accuracy after 107 epoch,(54m 37s) is 0.629500\n",
      "Validation Accuracy after 107 epoch,(54m 38s) is 0.642708\n",
      "Loss after 108 epoch,(55m 8s) is 1.214182\n",
      "Train Accuracy after 108 epoch,(55m 8s) is 0.635750\n",
      "Validation Accuracy after 108 epoch,(55m 9s) is 0.645833\n",
      "Loss after 109 epoch,(55m 39s) is 1.231330\n",
      "Train Accuracy after 109 epoch,(55m 39s) is 0.627375\n",
      "Validation Accuracy after 109 epoch,(55m 40s) is 0.635417\n",
      "Loss after 110 epoch,(56m 9s) is 1.271385\n",
      "Train Accuracy after 110 epoch,(56m 9s) is 0.622375\n",
      "Validation Accuracy after 110 epoch,(56m 10s) is 0.633333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss after 111 epoch,(56m 40s) is 1.242462\n",
      "Train Accuracy after 111 epoch,(56m 40s) is 0.630000\n",
      "Validation Accuracy after 111 epoch,(56m 41s) is 0.618750\n",
      "Loss after 112 epoch,(57m 10s) is 1.225105\n",
      "Train Accuracy after 112 epoch,(57m 10s) is 0.627000\n",
      "Validation Accuracy after 112 epoch,(57m 11s) is 0.608333\n",
      "Loss after 113 epoch,(57m 41s) is 1.248480\n",
      "Train Accuracy after 113 epoch,(57m 41s) is 0.619625\n",
      "Validation Accuracy after 113 epoch,(57m 42s) is 0.619792\n",
      "Loss after 114 epoch,(58m 11s) is 1.242726\n",
      "Train Accuracy after 114 epoch,(58m 11s) is 0.624250\n",
      "Validation Accuracy after 114 epoch,(58m 12s) is 0.658333\n",
      "Loss after 115 epoch,(58m 42s) is 1.244515\n",
      "Train Accuracy after 115 epoch,(58m 42s) is 0.621750\n",
      "Validation Accuracy after 115 epoch,(58m 43s) is 0.635417\n",
      "Loss after 116 epoch,(59m 12s) is 1.227123\n",
      "Train Accuracy after 116 epoch,(59m 12s) is 0.628000\n",
      "Validation Accuracy after 116 epoch,(59m 13s) is 0.626042\n",
      "Loss after 117 epoch,(59m 43s) is 1.245767\n",
      "Train Accuracy after 117 epoch,(59m 43s) is 0.623875\n",
      "Validation Accuracy after 117 epoch,(59m 44s) is 0.644792\n",
      "Loss after 118 epoch,(60m 14s) is 1.229682\n",
      "Train Accuracy after 118 epoch,(60m 14s) is 0.626500\n",
      "Validation Accuracy after 118 epoch,(60m 15s) is 0.657292\n",
      "Loss after 119 epoch,(60m 44s) is 1.244435\n",
      "Train Accuracy after 119 epoch,(60m 44s) is 0.619750\n",
      "Validation Accuracy after 119 epoch,(60m 45s) is 0.635417\n",
      "Loss after 120 epoch,(61m 15s) is 1.249380\n",
      "Train Accuracy after 120 epoch,(61m 15s) is 0.620500\n",
      "Validation Accuracy after 120 epoch,(61m 16s) is 0.626042\n",
      "Loss after 121 epoch,(61m 45s) is 1.215209\n",
      "Train Accuracy after 121 epoch,(61m 45s) is 0.629500\n",
      "Validation Accuracy after 121 epoch,(61m 46s) is 0.645833\n",
      "Loss after 122 epoch,(62m 16s) is 1.226359\n",
      "Train Accuracy after 122 epoch,(62m 16s) is 0.631750\n",
      "Validation Accuracy after 122 epoch,(62m 17s) is 0.640625\n",
      "Loss after 123 epoch,(62m 46s) is 1.239601\n",
      "Train Accuracy after 123 epoch,(62m 46s) is 0.627500\n",
      "Validation Accuracy after 123 epoch,(62m 47s) is 0.647917\n",
      "Loss after 124 epoch,(63m 17s) is 1.211314\n",
      "Train Accuracy after 124 epoch,(63m 17s) is 0.632125\n",
      "Validation Accuracy after 124 epoch,(63m 18s) is 0.633333\n",
      "Loss after 125 epoch,(63m 47s) is 1.228188\n",
      "Train Accuracy after 125 epoch,(63m 47s) is 0.630250\n",
      "Validation Accuracy after 125 epoch,(63m 48s) is 0.650000\n",
      "Loss after 126 epoch,(64m 18s) is 1.249535\n",
      "Train Accuracy after 126 epoch,(64m 18s) is 0.622875\n",
      "Validation Accuracy after 126 epoch,(64m 19s) is 0.633333\n",
      "Loss after 127 epoch,(64m 48s) is 1.256453\n",
      "Train Accuracy after 127 epoch,(64m 48s) is 0.622375\n",
      "Validation Accuracy after 127 epoch,(64m 49s) is 0.620833\n",
      "Loss after 128 epoch,(65m 19s) is 1.239359\n",
      "Train Accuracy after 128 epoch,(65m 19s) is 0.619750\n",
      "Validation Accuracy after 128 epoch,(65m 20s) is 0.677083\n",
      "Loss after 129 epoch,(65m 50s) is 1.249254\n",
      "Train Accuracy after 129 epoch,(65m 50s) is 0.627500\n",
      "Validation Accuracy after 129 epoch,(65m 51s) is 0.630208\n",
      "Loss after 130 epoch,(66m 20s) is 1.238952\n",
      "Train Accuracy after 130 epoch,(66m 20s) is 0.629750\n",
      "Validation Accuracy after 130 epoch,(66m 21s) is 0.656250\n",
      "Loss after 131 epoch,(66m 51s) is 1.236313\n",
      "Train Accuracy after 131 epoch,(66m 51s) is 0.626625\n",
      "Validation Accuracy after 131 epoch,(66m 52s) is 0.637500\n",
      "Loss after 132 epoch,(67m 22s) is 1.229545\n",
      "Train Accuracy after 132 epoch,(67m 22s) is 0.631250\n",
      "Validation Accuracy after 132 epoch,(67m 23s) is 0.639583\n",
      "Loss after 133 epoch,(67m 53s) is 1.216647\n",
      "Train Accuracy after 133 epoch,(67m 53s) is 0.635125\n",
      "Validation Accuracy after 133 epoch,(67m 54s) is 0.637500\n",
      "Loss after 134 epoch,(68m 23s) is 1.222895\n",
      "Train Accuracy after 134 epoch,(68m 23s) is 0.633500\n",
      "Validation Accuracy after 134 epoch,(68m 24s) is 0.631250\n",
      "Loss after 135 epoch,(68m 54s) is 1.244585\n",
      "Train Accuracy after 135 epoch,(68m 54s) is 0.629375\n",
      "Validation Accuracy after 135 epoch,(68m 55s) is 0.627083\n",
      "Loss after 136 epoch,(69m 25s) is 1.215980\n",
      "Train Accuracy after 136 epoch,(69m 25s) is 0.634750\n",
      "Validation Accuracy after 136 epoch,(69m 26s) is 0.636458\n",
      "Loss after 137 epoch,(69m 55s) is 1.241698\n",
      "Train Accuracy after 137 epoch,(69m 55s) is 0.623875\n",
      "Validation Accuracy after 137 epoch,(69m 56s) is 0.614583\n",
      "Loss after 138 epoch,(70m 26s) is 1.261087\n",
      "Train Accuracy after 138 epoch,(70m 26s) is 0.618750\n",
      "Validation Accuracy after 138 epoch,(70m 27s) is 0.640625\n",
      "Loss after 139 epoch,(70m 57s) is 1.233674\n",
      "Train Accuracy after 139 epoch,(70m 57s) is 0.629000\n",
      "Validation Accuracy after 139 epoch,(70m 58s) is 0.615625\n",
      "Loss after 140 epoch,(71m 28s) is 1.247716\n",
      "Train Accuracy after 140 epoch,(71m 28s) is 0.618750\n",
      "Validation Accuracy after 140 epoch,(71m 29s) is 0.634375\n",
      "Loss after 141 epoch,(71m 58s) is 1.230016\n",
      "Train Accuracy after 141 epoch,(71m 58s) is 0.625000\n",
      "Validation Accuracy after 141 epoch,(71m 59s) is 0.629167\n",
      "Loss after 142 epoch,(72m 29s) is 1.221497\n",
      "Train Accuracy after 142 epoch,(72m 29s) is 0.627375\n",
      "Validation Accuracy after 142 epoch,(72m 30s) is 0.632292\n",
      "Loss after 143 epoch,(72m 59s) is 1.240145\n",
      "Train Accuracy after 143 epoch,(72m 59s) is 0.622625\n",
      "Validation Accuracy after 143 epoch,(73m 0s) is 0.645833\n",
      "Loss after 144 epoch,(73m 30s) is 1.248957\n",
      "Train Accuracy after 144 epoch,(73m 30s) is 0.626500\n",
      "Validation Accuracy after 144 epoch,(73m 31s) is 0.626042\n",
      "Loss after 145 epoch,(74m 0s) is 1.242770\n",
      "Train Accuracy after 145 epoch,(74m 0s) is 0.629000\n",
      "Validation Accuracy after 145 epoch,(74m 1s) is 0.668750\n",
      "Loss after 146 epoch,(74m 31s) is 1.252156\n",
      "Train Accuracy after 146 epoch,(74m 31s) is 0.623375\n",
      "Validation Accuracy after 146 epoch,(74m 32s) is 0.636458\n",
      "Loss after 147 epoch,(75m 1s) is 1.217991\n",
      "Train Accuracy after 147 epoch,(75m 1s) is 0.638375\n",
      "Validation Accuracy after 147 epoch,(75m 2s) is 0.618750\n",
      "Loss after 148 epoch,(75m 32s) is 1.243027\n",
      "Train Accuracy after 148 epoch,(75m 32s) is 0.631250\n",
      "Validation Accuracy after 148 epoch,(75m 33s) is 0.659375\n",
      "Loss after 149 epoch,(76m 2s) is 1.231981\n",
      "Train Accuracy after 149 epoch,(76m 2s) is 0.629250\n",
      "Validation Accuracy after 149 epoch,(76m 3s) is 0.623958\n",
      "Loss after 150 epoch,(76m 33s) is 1.257967\n",
      "Train Accuracy after 150 epoch,(76m 33s) is 0.623000\n",
      "Validation Accuracy after 150 epoch,(76m 34s) is 0.634375\n",
      "Loss after 151 epoch,(77m 4s) is 1.268162\n",
      "Train Accuracy after 151 epoch,(77m 4s) is 0.615750\n",
      "Validation Accuracy after 151 epoch,(77m 5s) is 0.637500\n",
      "Loss after 152 epoch,(77m 35s) is 1.257832\n",
      "Train Accuracy after 152 epoch,(77m 35s) is 0.620375\n",
      "Validation Accuracy after 152 epoch,(77m 36s) is 0.614583\n",
      "Loss after 153 epoch,(78m 6s) is 1.257084\n",
      "Train Accuracy after 153 epoch,(78m 6s) is 0.622000\n",
      "Validation Accuracy after 153 epoch,(78m 7s) is 0.631250\n",
      "Loss after 154 epoch,(78m 36s) is 1.249362\n",
      "Train Accuracy after 154 epoch,(78m 36s) is 0.620000\n",
      "Validation Accuracy after 154 epoch,(78m 37s) is 0.648958\n",
      "Loss after 155 epoch,(79m 7s) is 1.222886\n",
      "Train Accuracy after 155 epoch,(79m 7s) is 0.633250\n",
      "Validation Accuracy after 155 epoch,(79m 8s) is 0.616667\n",
      "Loss after 156 epoch,(79m 38s) is 1.224281\n",
      "Train Accuracy after 156 epoch,(79m 38s) is 0.628250\n",
      "Validation Accuracy after 156 epoch,(79m 39s) is 0.643750\n",
      "Loss after 157 epoch,(80m 9s) is 1.277861\n",
      "Train Accuracy after 157 epoch,(80m 9s) is 0.611500\n",
      "Validation Accuracy after 157 epoch,(80m 10s) is 0.632292\n",
      "Loss after 158 epoch,(80m 39s) is 1.280400\n",
      "Train Accuracy after 158 epoch,(80m 39s) is 0.610500\n",
      "Validation Accuracy after 158 epoch,(80m 40s) is 0.641667\n",
      "Loss after 159 epoch,(81m 10s) is 1.238224\n",
      "Train Accuracy after 159 epoch,(81m 10s) is 0.629750\n",
      "Validation Accuracy after 159 epoch,(81m 11s) is 0.610417\n",
      "Loss after 160 epoch,(81m 41s) is 1.214646\n",
      "Train Accuracy after 160 epoch,(81m 41s) is 0.638500\n",
      "Validation Accuracy after 160 epoch,(81m 42s) is 0.632292\n",
      "Loss after 161 epoch,(82m 11s) is 1.246380\n",
      "Train Accuracy after 161 epoch,(82m 11s) is 0.629250\n",
      "Validation Accuracy after 161 epoch,(82m 12s) is 0.606250\n",
      "Loss after 162 epoch,(82m 42s) is 1.241010\n",
      "Train Accuracy after 162 epoch,(82m 42s) is 0.624000\n",
      "Validation Accuracy after 162 epoch,(82m 43s) is 0.632292\n",
      "Loss after 163 epoch,(83m 13s) is 1.214980\n",
      "Train Accuracy after 163 epoch,(83m 13s) is 0.628875\n",
      "Validation Accuracy after 163 epoch,(83m 14s) is 0.614583\n",
      "Loss after 164 epoch,(83m 43s) is 1.226941\n",
      "Train Accuracy after 164 epoch,(83m 43s) is 0.632375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy after 164 epoch,(83m 44s) is 0.618750\n",
      "Loss after 165 epoch,(84m 14s) is 1.221364\n",
      "Train Accuracy after 165 epoch,(84m 14s) is 0.633250\n",
      "Validation Accuracy after 165 epoch,(84m 15s) is 0.641667\n",
      "Loss after 166 epoch,(84m 45s) is 1.251058\n",
      "Train Accuracy after 166 epoch,(84m 45s) is 0.622000\n",
      "Validation Accuracy after 166 epoch,(84m 46s) is 0.623958\n",
      "Loss after 167 epoch,(85m 15s) is 1.232071\n",
      "Train Accuracy after 167 epoch,(85m 15s) is 0.634375\n",
      "Validation Accuracy after 167 epoch,(85m 16s) is 0.613542\n",
      "Loss after 168 epoch,(85m 46s) is 1.241580\n",
      "Train Accuracy after 168 epoch,(85m 46s) is 0.630000\n",
      "Validation Accuracy after 168 epoch,(85m 47s) is 0.637500\n",
      "Loss after 169 epoch,(86m 17s) is 1.248549\n",
      "Train Accuracy after 169 epoch,(86m 17s) is 0.621125\n",
      "Validation Accuracy after 169 epoch,(86m 18s) is 0.631250\n",
      "Loss after 170 epoch,(86m 48s) is 1.243335\n",
      "Train Accuracy after 170 epoch,(86m 48s) is 0.628625\n",
      "Validation Accuracy after 170 epoch,(86m 49s) is 0.639583\n",
      "Loss after 171 epoch,(87m 18s) is 1.244973\n",
      "Train Accuracy after 171 epoch,(87m 18s) is 0.628875\n",
      "Validation Accuracy after 171 epoch,(87m 19s) is 0.637500\n",
      "Loss after 172 epoch,(87m 49s) is 1.250074\n",
      "Train Accuracy after 172 epoch,(87m 49s) is 0.626750\n",
      "Validation Accuracy after 172 epoch,(87m 50s) is 0.628125\n",
      "Loss after 173 epoch,(88m 20s) is 1.251163\n",
      "Train Accuracy after 173 epoch,(88m 20s) is 0.622750\n",
      "Validation Accuracy after 173 epoch,(88m 21s) is 0.642708\n",
      "Loss after 174 epoch,(88m 50s) is 1.256455\n",
      "Train Accuracy after 174 epoch,(88m 50s) is 0.626125\n",
      "Validation Accuracy after 174 epoch,(88m 51s) is 0.598958\n",
      "Loss after 175 epoch,(89m 21s) is 1.251069\n",
      "Train Accuracy after 175 epoch,(89m 21s) is 0.621250\n",
      "Validation Accuracy after 175 epoch,(89m 22s) is 0.645833\n",
      "Loss after 176 epoch,(89m 52s) is 1.224262\n",
      "Train Accuracy after 176 epoch,(89m 52s) is 0.629500\n",
      "Validation Accuracy after 176 epoch,(89m 53s) is 0.640625\n",
      "Loss after 177 epoch,(90m 22s) is 1.215464\n",
      "Train Accuracy after 177 epoch,(90m 22s) is 0.633375\n",
      "Validation Accuracy after 177 epoch,(90m 23s) is 0.629167\n",
      "Loss after 178 epoch,(90m 53s) is 1.244945\n",
      "Train Accuracy after 178 epoch,(90m 53s) is 0.628125\n",
      "Validation Accuracy after 178 epoch,(90m 54s) is 0.611458\n",
      "Loss after 179 epoch,(91m 24s) is 1.236639\n",
      "Train Accuracy after 179 epoch,(91m 24s) is 0.631000\n",
      "Validation Accuracy after 179 epoch,(91m 25s) is 0.653125\n",
      "Loss after 180 epoch,(91m 55s) is 1.247575\n",
      "Train Accuracy after 180 epoch,(91m 55s) is 0.627750\n",
      "Validation Accuracy after 180 epoch,(91m 56s) is 0.652083\n",
      "Loss after 181 epoch,(92m 25s) is 1.233839\n",
      "Train Accuracy after 181 epoch,(92m 25s) is 0.624000\n",
      "Validation Accuracy after 181 epoch,(92m 26s) is 0.654167\n",
      "Loss after 182 epoch,(92m 56s) is 1.252149\n",
      "Train Accuracy after 182 epoch,(92m 56s) is 0.619125\n",
      "Validation Accuracy after 182 epoch,(92m 57s) is 0.621875\n",
      "Loss after 183 epoch,(93m 27s) is 1.232041\n",
      "Train Accuracy after 183 epoch,(93m 27s) is 0.625500\n",
      "Validation Accuracy after 183 epoch,(93m 28s) is 0.611458\n",
      "Loss after 184 epoch,(93m 57s) is 1.246423\n",
      "Train Accuracy after 184 epoch,(93m 57s) is 0.620500\n",
      "Validation Accuracy after 184 epoch,(93m 58s) is 0.639583\n",
      "Loss after 185 epoch,(94m 28s) is 1.230451\n",
      "Train Accuracy after 185 epoch,(94m 28s) is 0.632125\n",
      "Validation Accuracy after 185 epoch,(94m 29s) is 0.637500\n",
      "Loss after 186 epoch,(94m 59s) is 1.267187\n",
      "Train Accuracy after 186 epoch,(94m 59s) is 0.618625\n",
      "Validation Accuracy after 186 epoch,(95m 0s) is 0.644792\n",
      "Loss after 187 epoch,(95m 30s) is 1.225894\n",
      "Train Accuracy after 187 epoch,(95m 30s) is 0.629125\n",
      "Validation Accuracy after 187 epoch,(95m 31s) is 0.630208\n",
      "Loss after 188 epoch,(96m 0s) is 1.236409\n",
      "Train Accuracy after 188 epoch,(96m 0s) is 0.624375\n",
      "Validation Accuracy after 188 epoch,(96m 1s) is 0.622917\n",
      "Loss after 189 epoch,(96m 31s) is 1.257681\n",
      "Train Accuracy after 189 epoch,(96m 31s) is 0.620875\n",
      "Validation Accuracy after 189 epoch,(96m 32s) is 0.638542\n",
      "Loss after 190 epoch,(97m 2s) is 1.253284\n",
      "Train Accuracy after 190 epoch,(97m 2s) is 0.628125\n",
      "Validation Accuracy after 190 epoch,(97m 3s) is 0.642708\n",
      "Loss after 191 epoch,(97m 32s) is 1.252644\n",
      "Train Accuracy after 191 epoch,(97m 32s) is 0.619875\n",
      "Validation Accuracy after 191 epoch,(97m 33s) is 0.621875\n",
      "Loss after 192 epoch,(98m 3s) is 1.209807\n",
      "Train Accuracy after 192 epoch,(98m 3s) is 0.632750\n",
      "Validation Accuracy after 192 epoch,(98m 4s) is 0.629167\n",
      "Loss after 193 epoch,(98m 34s) is 1.248936\n",
      "Train Accuracy after 193 epoch,(98m 34s) is 0.623000\n",
      "Validation Accuracy after 193 epoch,(98m 35s) is 0.642708\n",
      "Loss after 194 epoch,(99m 4s) is 1.239064\n",
      "Train Accuracy after 194 epoch,(99m 4s) is 0.624125\n",
      "Validation Accuracy after 194 epoch,(99m 5s) is 0.643750\n",
      "Loss after 195 epoch,(99m 35s) is 1.254304\n",
      "Train Accuracy after 195 epoch,(99m 35s) is 0.625500\n",
      "Validation Accuracy after 195 epoch,(99m 36s) is 0.633333\n",
      "Loss after 196 epoch,(100m 6s) is 1.249151\n",
      "Train Accuracy after 196 epoch,(100m 6s) is 0.621875\n",
      "Validation Accuracy after 196 epoch,(100m 7s) is 0.620833\n",
      "Loss after 197 epoch,(100m 36s) is 1.240189\n",
      "Train Accuracy after 197 epoch,(100m 36s) is 0.631250\n",
      "Validation Accuracy after 197 epoch,(100m 37s) is 0.664583\n",
      "Loss after 198 epoch,(101m 7s) is 1.239564\n",
      "Train Accuracy after 198 epoch,(101m 7s) is 0.623750\n",
      "Validation Accuracy after 198 epoch,(101m 8s) is 0.642708\n",
      "Loss after 199 epoch,(101m 38s) is 1.256849\n",
      "Train Accuracy after 199 epoch,(101m 38s) is 0.625750\n",
      "Validation Accuracy after 199 epoch,(101m 39s) is 0.610417\n",
      "Loss after 200 epoch,(102m 9s) is 1.235079\n",
      "Train Accuracy after 200 epoch,(102m 9s) is 0.628375\n",
      "Validation Accuracy after 200 epoch,(102m 10s) is 0.646875\n"
     ]
    }
   ],
   "source": [
    "loss_full, acc_full, val_acc = train_early_stopping(batch_size, X_train_pad, y_train_tensor, X_val_pad,\n",
    "                                y_val_tensor, sent_attn, sent_optimizer, criterion, epoch, 10000, False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(loss_full)\n",
    "plt.ylabel('Training Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.savefig('loss.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xec3HWd+PHXe2Z3Z3svSTabviEECCUhNCmCFBuIWFAPxYbCRc9yKNwp3qHenfXup+J5qAgiCoioQQMRkF5CNiGFJCTZbNpms733Mu/fH9+S2c3M7GyS2YTwfj4e+9idme/MfHbK5/35vD/lK6qKMcYYE0/gaBfAGGPMsc+ChTHGmHFZsDDGGDMuCxbGGGPGZcHCGGPMuCxYGGOMGZcFC2OMMeOyYGGMMWZcFiyMMcaMK+VoF+BIKS4u1lmzZh3tYhhjzBvKmjVrmlW1ZLzjjptgMWvWLKqqqo52MYwx5g1FRHYncpyloYwxxozLgoUxxphxWbAwxhgzLgsWxhhjxmXBwhhjzLiSGixE5AoR2Soi1SJyS4xjPiAim0Vkk4j8dsxtuSKyT0R+ksxyGmOMiS9pU2dFJAjcAVwK1AKrRWS5qm6OOKYSuBU4T1XbRKR0zMN8E3gmWWU0xhiTmGT2LJYC1apao6qDwP3AVWOO+TRwh6q2Aahqo3eDiCwGyoC/JbGMxoyyt7WXv7/ecLSLYcwxJ5nBohzYG3G51r0u0nxgvoi8ICIvi8gVACISAH4A3JzE8hlzkF88V8Nn713LSNjOTW9MpGQGC4ly3dhvYApQCVwEfAj4hYjkAzcBK1R1L3GIyA0iUiUiVU1NTUegyCbZdjX38N6fvkBH79DRLkpU+9r7GBwJ09w9MO6xH//VK/zXo69PQqlMosJhRdUCfTIkM1jUAhURl6cDdVGO+bOqDqnqTmArTvA4B1gmIruA7wMfFZH/GvsEqnqnqi5R1SUlJeNubWKOAetr21m7p50dzd1HuyhR7e/oB5ygMZ6N+zpYv7c92UU6rn3rL5v54d+2HrHHu/KO5/nvJ7YfscczByQzWKwGKkVktoikAdcCy8cc8yfgrQAiUoyTlqpR1Y+o6gxVnQX8M/BrVY06m8q8sXT1DwPQMzB8lEsSnR8s2uIHi3BYaesdoqGrfzKKddx6amsjf9mw/4g8Vv/QCJvqOllnATwpkhYsVHUYWAasBLYAD6rqJhG5XUSudA9bCbSIyGbgKeBmVW1JVpmOlOe2N7Flf2fCx29r6Bo1aDoSVq74n2dZsfHIfEmOFUMJpG+6B47dYNE/NEJrzyAAde19PL+9mX/87dqoaY2u/mFGwkpj5/jpKhNbW+8Qu1p6GBgeOezHqm3rRRVqW3uPQMnMWEldZ6GqK1R1vqrOVdVvu9fdpqrL3b9VVb+kqgtV9RRVvT/KY9ytqsuSWc6JuuUPG/nRk4l3db+/civ/9Lt1fqXT3jvI6/Vdx10K4+4XdnHx959maCQc8xgvSHQPHH7lcKQ1dB7oJdS197F8/T7+umE/XRGB7bHX9vN6fSctPU6Q6B4YPqKBr7Gzn3tf3v2myLuPhJW23kHCCjVNPYf9eLtbnCBR29ZH2CYoHHG2gnuCwmGlobPfb4EmYn1tO10Dw9S7lVF7nzO429ab+GO8Eby6t43O/mGaumK3tr00VHf/sTfAXdd+IFjsa+9ny/4uANp7DpT15oc2cOczNaPeu8Y4/+9E/WbVHr7+p9cSGjOZTB29Q9x03xq2N3T513X1Dx1WOTv6hvBi4raIxz1Ue9wexeBI+Ii+J8ZhwWKCmnsGGA4rHX2JVXYNnf00uKmK7Q3OoG67W9G0uTOCuvqHkjpVs3dweNSXPFm21jvPUd8ZO4/vp6EGj72eRX2nU/HNLs5ib2svW93XzAsMvYPDdPUP09DVT2tEAGmI8/9O1KZ9HQDsaj62UimPvrafFRvruem+tfS57923/7qFD9358iE/ZmSDy/tuHA6vZwGwty3+69fVP8Snf13FXktZJcyCxQQ1dDgV/3i9gh/+bSufvHv1qFTT9kYvWAy5vwdRVS75wTP854otSSox3P3iLt79k+fjpocO18DwCLvcL2tDR5xg4fUsjsExC69nccaMArY2dDE47Lxere577Y1P1Hf00xZR0R3RYFHnjIXtajn8tMyR9PjmBnLSU6hu6uY/3M/qyzUt7G3r9V+niRoVLBoTa8y09w7GHC/c09pLdsjZlCJaEBgcDrN6Vyuqymv7Onl8cwN/enXfIZQcntzSMCkNsGOJBYsJ8iqG9nHWCVTtbuPJ1xu5f/VeggEhNz2F6kavpTrk/+4eGKaxa4B7X94dN31zOGrb+ugfCtOZYG/oUNQ09fi9o3iVZ89g/AHucFiP2oK4+o5+8jJSmVeaPep6ryfo/V+NnQN+AAESft8aOvvZWNsR8/bm7gG/V7ar+dgJFr2Dwzxf3cw1Z0znA4sr+P2avdS197GrxRlQbjzEGWFesJhRmJlwz+J/n9nBe3/6Iv1DB/dMd7f0sHR2IQB7W0enx/a09PL+n73I+3/2Eqt3tfllfm5784TL3T0wzI2/Wcv3Vh65Kb+RfvTkdqp2tSblsQ+HBYsJ8r7MA8NhvzsejfdF+PvrjZxQlsOCqbkHpaHaewdp7h70H++uF3YmpczNbmWWaOosmvbeQerj9Bgic871cWYIdY3Ts7jl4Q1c/dMXjsoA5f6OfqbmpTMtPx0AcZeVtrkpJy8P3jUwTG1bL2kpAdJTAwn3LG59eCOfvGd1zNu9XkVA8HtpR8rTWxtp6R5AVbnul6u496VdCd/3ue3NDAyHuXRhGVecMoX+oTA/e2aHf3u8z0U8Xu/8rNmF7GrpiRoAxqpr76dvaIQNY4JuOKzsbeujsjSbstzQQWmoG+9b46cV97T2+gF+7Z62Cfdyn9vWxOBImLV72hKeiHDX8zv5+p9eG/e4kbDyP09s46E1tRMq02SwYDFBkRVDvFRUS0QX+9SKPCpLs9ne2I2qRqShhmh0H29Kbjr3vrQ7bgA6VN501vbDCBa3/GEjV/7keXoHo3+xtjV0kRIQSnNCcSvPeFNnW7oH+OOr+9hQ28GK1yZ/WvH+jj6m5qVTnp8BwAllOQTkwPsc+X+9vr+Loqw0SnPS/TGpeFq6B3hmWxONXQPueoAOfvPy6FMfb6pzKsClbuV5pKzd08b1v1rNT56qZkdTD89tb+Zbf92ScO/lic0N5KansHR2IefMKSKUEuD+Vw5srrD/EIOF16A6a04RYU1skNtr+Lyyc/QM+/rOfgaHw1QUZlJRkDkqDbW9oYtNdZ184W3zAed99ILFcFh5ecfEZus/vtmZBt/cPcie1l5W1bSMW/YnX29g5ab6cR+7pWeAsDrZgGj2d/Rx42/W0HUUJohYsJigyFaUV+mr6qgctnd56SynS3z6jAIqS7Pp6BuiqXvAr3yGw+oPyv3D2TPoHhhm477YaYpD5QWuQ+1ZhMPKyztbaOwa4Fcv7Ip6zNb6bmYXZ1FRmBm3pXlg6uzBweIPa2sZGlHKckP8vye2j5uOeuy1+iM6QFnf0c+UvAymucFi4bRc8jPT/PcrcobN6/VdFGSmUZYbSigNs2Ljfv//aewc4N6XdnPbn18ble/ftK+TisIMTp2ez56WXv/4rv6hhFrd0agq333M2ZLkmW1NvLTDSbuIwL/8ceO4LWNV5bntzZxfWUJqMEB6apBz5xYxOBJmVlEmcOg9i9aeQbLSglw4v4SstGBC09G9hs8ru9pGXe/NhJpZlElFYeaoyvaRDfsRgfeeXk5eRir7O/po7BqgNCdERmqQ57YnvlXQ8EiYv29t5OTyXACe3dbEJ+5ezZcfXB/3fg2dA7T0DI7bY/aCWKxZZs9ta+bR1+qpGvP/TwYLFhPUEFFhtPcO0tU/xBceWMcZ33qc19yKvrNvmOGwctlJZfzhxnO4+vRyKstyAKhu6B7VwvdaJG9bWAbAur2JfwgSzRV7rbFDHbOoae6mvXeInFAK//fMjqj7Om1v7GJ+WQ5TctPj9yz6o6+zUFXuf2UvS2YW8LV3LmR7YzdPb22M9hCAU2nceN8afvj4toT/j46+oZjjC/1DI7T0DDItL52y3HROLs/lkgVl5GemHkhDdfYTcFNT3QPDFLo9i0QW5v15XR0p7p33d/Q5awF0dEW7qa6Dk6bmMbMoi8GRMPs7nArjI79YNW5lFMvz1c28XNPKgik51DT18Ps1tZTnZ/CFt83nxR0tMVuwtz+ymR89uZ2dzT3Ud/Zz7rwi/7a3LnDOJOBV8ofTsyjISqMkJ8TnL6nkiS2N4+746wWLtbvbRjUm9riNrpmFWVQUZLC/o4+hkTCqyl821HHW7EJKc9OZkptOfccATV0DlBdksHR2IS/VJN6zqNrdRnvvEDdeOI+cUAo/fHwbPYMjbNzX4Y9JRtPQ0c9IWP3vfqwgHRksogUWL722o2nyt8uxYDFBDR39zHRbVO19Q3zxgfUsX1+HqrOeAvAXbBVlp7F4ZiGpwYA/aLq9sdsfs/AuBwQqS3OoKMzg1T2xF+q19gzyhftfZV97H49vbmDpt5/k1y/tOui4mqZu/4vUNzjiT1NNtGdR197He3/6Ak+43e3VbivmW1efTGf/MCs3j+5Od/QNsae1l/llOZTlplPf2R/1yxAOK90xBrhfr++iprmHDyyp4NKFZQQE1scZDH5ySwOqTj5dVRkcDvvP+dq+Dr/y8KgqH/3lKs7+zye56b41o94DgKdedwLTyeV5BAPCXz53Pu9cNJXCiJ5FQ+cA892gDzjBIjd+2g2cgFC1u413nzrNudzZT22bt4DM+d3VP8Sull5OmpbLrGLn87W7pZehkTCb6zp5bFP9IbXgH32tntz0FP77g6cBsKG2g3PmFnHGjAIAqqNUOq/uaeOuF3byv0/v8FMu584t9m+/5MQyMlKDXHhCCVPy0v2gFg4rH7vrFZ6KE+QBvvTgOn7xXA2tPYMUZqUB8PHzZjOnOIuf/L065v2GRsK09Q4xpziL7oHhUbOimt3vXGluiBlFWYTVef1er++ipqmHdy1yXvspeU5jprGrn5LsEPPLstnd0uuMebT28r2Vr/Ope6pibhniDTxfML+Y02cW+OUJBoSH10afWdUzMOwv7GzpHuBHT27nyp+8EPVYL1gMDkffDcEL7tWN3RHX9cZMDx9JFizi2Nl8YNDtU/dUcdfzO6nv7OcEt8Jo6x1k3d52rjljOhmpQXY0OjlgLxdbmBXyH8vr8u5p7aW9d4j8zFTAedMLs0IEA8JpFQUHfUhV1f9SrN3dxp/W1fG5367l3x/ZBMB3Hn2duogua0NnP5f997Pc+WwNwKgPXEfvEK/t6+BLD66LO93xpR0trN3TzqfvreKeF3dRtauNoqw03nHKVIIBOagifrG6GVU4d14RU/JC9A6OjFr13Dc4woqN++kdGvEXYY0NFt6X4IQpOaSnBplVlMW2+i76Bke44n+e5aqfPM8vnqvxj398c6P//22o7eCy/37G72X842/X8tU/bACcVNWjG/fzxJZG1td2cNH8Eh57rZ6fPVMz6vnvW7WH8vwMLpg/ekNKJw3lBNmGrn7mlGSRlRYE8HsWPYMjcQdJX93jBNv3L54OOK1GL83g/d+b3cHtk8vzmFWUBTifv90tvQy7M8R+X3XwJswDwyP88G9bDwp+nr2tvcwuyWbBlBx/LObcuUV+42VH4+hg4aSttpKeGqBvaIQf/72aKbnpfsoJoDw/g1dvu5SLF5QxNS/D71k0ueMyz2yNndZ5ZWcrD6/dx0NramnrPRAs0lICXLyglE11nQzHmOLtfa/efsoUgFEzhrr6h0kLBgilBFg0PQ+AdXvb/RTTZW7PfYrbmGnqGvADy8Cws4jvh49v43+f3sHz1U38y8Mbo6ZBdzb3UpYbIic9lcVuwP3MhXO4cH4Jf3x1X9TeQGT6sql7gHV729m4r8NPoaoq//LHjazZ3epPeAGojZKK8u4T2bO49eGNvPenLyZ91b8FixgerNrLJT94mu+v3EpjVz9PbGngp09X09E3xIIpTrCoa++juXuA2cVZzCnJ8ltp3hhBkftFABARZhRm+sFidrFTIexr76Mkxwkqp1fks7+jf1QL8qmtjbz9/z3HjqZuf7rm2j3t1Lb18b33LSKs8OUH1/uV76qdrQyHlQdW70FVaYoIFu19Qzy5pZGH1+7jxR2xpwzWNHcTDAiXLCjj3x7ZxN821bN4ZgGpwQDT8tPZ2+a0xL772OvsaOrmmW1N5KSncHpFPmW5zkyi+o5+WtznfmD1Hm66b62/4CyUEqB7YBhVZc1uZ0aJ1zqfkufcv7Ism22NXby6t43X67to6RnkW3/dwua6TvoGR3i+uokrTnIqjVse3siull427utgeCRMbVsfr+xqpb6jn5t/v54b73OCx8yiTP7vusVccfIU7lu126/gdzb38Hx1M9eeWUEwMHpn/YLMVH88qqlzgNKcdP9/9MYs4MDgt6ryud+9ylceOpA2Wl/bQWpQWDyrgJz0FDbs7WBoxPliez0LbybUSdNymZKbTiglwK7mHr9SKM0Jcf/qvQdVYC/taOFHf6+OOXtmT2svFQUZiAgXnuAEwnPmFlGYlUZRVtqoFio4n62Xalr4yuULmFeaTffAMOfOLUJk9OuSnhr03y/v8+o1WryeBjjB4YZfV/mNk58+7fQctjV0sa+tj8LMA9+RhdNyGRgOUxNj4N1rdZ9SnkdaSoD9ET26zr4hcjNSEBHmlWSTk57Cmt1trKppZU5xFqXue1aWl05z9wBtvUOU5qQzozDTf522N3Zx3rxivnPNIjbv7+ThtQe/prtaevxgfvXp5Xxo6QyuPLWcq08vZ39HP69EmfIa2fNs6R70X6eX3fTX/o5+frtqD4+s3z8qTRptM0svDeW9b5vqOnhuezNXnVZ+0Ht0pFmwiOIvG+r4ykMbUODZ7U2scdMwXtSfUZRFRmqQjfucL3hFYSbzSrP9VtqBnkXaqMetKMxkT0sv7b2DzHY/cADF2c5xp83IB+Cf7n+VC777FB19Q/6eOfva+vxK64YL5vCPb53L+5dU8M33nMyqnS2872cv0dw9wOqdzod1V0svq3e1+eMV4KSLvF1SH904OpX02r4Oln77CfZ39FHT1MOMwkx+/KHTOXFKLl0DwyyZ5bSiKgqcgLejqZufPr2Dr//pNZ7Z1sRb5hWTEgz4FenX/vQa533n77T1DPrTHL3xmbLcdHoGhnllZyvX/O+LVO1uo8EdD/AC7AllOexq7uHlGuf/+e2nziYjNcjdL+7kmW2N9A+Fue6cmVSWZvs9r72tvdR3OrnhkbDyr3/cSNfAMKdOz6O1Z5DPXVxJSjDAp8+fQ1f/MA+sdlrqv3tlD8GA8MEzI3fUx38P23oHndXbA8OU5oYodQNEYVaqPxjufbF/+fxOHllfx6Mb6/1W5obadhZMySWUEmRqXjpVuw9UKF7PYlNdJ8XZIUpz0wkEhAVTcthQ2+EHi89dUsm+9j5/XMzjBZmntzbRPzTCN/78mh+AhkfC7Gvr8yvEZW+dx48+dDpT85wyz3Vn6EXyFpFeedo0vyd0ztwiYpmal05jVz/DI2F/UaP3W1X590c28bfNDazb286mug6e3trEmbMKCKvTqCqI+I6cNC3P/Z+ipx+9XnJxdoj8jNRRY2ed/cPkpju99UBAOGNGAat3tfLKrlZ/7QU4PQuvAV6SE2Km+9rsaulhR2MP80qzufLUaZxWkc8PH99GOKwMj4T9WYu7mnv8ht6Mokz+872nkJEW5OIFpaSlBKLOeIoMFs3dA35PzPtse+/BzuYemroHKM52Pl9jx5MGhkdo6BxwGjC9Q7R0D/CL53aSlRbkw2fNiPqaHUkWLMYYHgnzvZVbWTg1ly9cMp9tDd08tqmeUErAr8im5KaTn5nKRneMYkZhJvNKstnX3kfv4HDMYDGjMJNdLT30DI4woyjTn8fv9SxOmpZLWjDAqp2t7Gntpbqxy//itfQ4C8HSUgLc+vYF3Hz5AgDet3g6v/r4UrY1dHHnszWs3tXKmbMKyA6l8GDVXr+Xk5ueQkffkD8Yu3Jz/agV3at2ttLYNcBLO1rY2dzDnOIsMtKC/N91i7l4QSlvP3mq/z/sbe3z56y/uKOF/R39XOimb6a4weKVna30DzkrZr0ZXt6Xoiw3RFjxH2NHYzcNnf2U5IRICTofycqyHMIKf1hTy5ySLGYUZfLeM8r507o6vvqHjZTnO4OT51c6zzunJIvatr5RX7AnX29kal46D914LsuXncc1Zzgnajx9RgFLZxXyy+dq6Oof4vdVe7lsYZnf+oyUn5nGwHDY336jLKJnUZgV8luZu1t62N3Sw3cee52irDS6Boapae4mHFY21nb4qZEpeRl+o6M4Oy0iWHRw0rRc/3nPnFXIutp2tuzvoiw3xHluhT12iqaXvlq1s4W7X9zFPS/t5q/ult/7O/oZDqsfLKblZ3ClO24CMK80m2p3OrenprmbvIxUirLS+NBZM/jMBXO44uQpB70unil56YTVSa94LWbv9+ObG/xg9nJNC39eV0dqUPjONYv8+0d+R+aWZBFKCbBpX/QV2gdet5Az8SAi9dbZN0ROeop/efHMAqobu+nqH+asORHBIm90anhafgYBgVU1rfQNjTCvNBsR4ePnzWJ/Rz/ra9v5yVPVvPX7T7O/o4+WnkFmFR9o6HmyQilcUFnM3zY1+K/nzuYetjV0jQoWe1v7/LHDl2taUFV/Jfiulh6auvqZXZxJXkYq+9qdz1zPwDAv7mj2GyReqvTZ7U08sr6Oa5fOIC8jNeprdiRZsHA9s62JLz24jntf3s3ull4+f8k8LnK77Y+sr+PU6fm853SnspmSFxqVy55RmMlcNwdc09RDS7czJdDrqntmFmUy4HbHi7JDfkuoxG1JhFKCfOd9p/D1dy0EnJaF16Vv6R6ktXuQwsy0g7qbF84v4bKFZdz/yh62NnRxQWUJ7zxlKo9u3O+3Yua6U3cbu/pJTw3Q3jvEqpoDLdyd7smIXt3T7gSLEucLUVGYyV3Xn0mFW+FUFGbS3D3A+r3tBASmuWkj7wPspZGyQymkBQM8u73Jbx17ixKnuC1bryu9t62X+s4BP9CAM3YBTprOG4y9/txZDA6HyctI5f4bziY1GODj583iK1ecwHVnz2RgOOyP+Zwzx6lc33N6OanBAIum54963ZZdPI+6jn4++5s1tPUO8ZGzZhJNYZbzHm1tcCqwstyINFRWKqU5IUIpAXa39PLKzlaGRpRvX30yAOv2drCzpcft3Ti9xqkR/+NZs4uobeulf2iE7Y3d/nRMgDNnFzI4HOaJzQ3MLclmRmEmacHAQQPSm/d3UpoTYmhE+YF7EiFvjy4vv+0Fi7G86dyRefIdjc57LyLkpqdy6ztOJCc9dkU01X2/93f0++MwLT2D9LvjHTOLMjmhLIeXdrTw+OYGzp5TxJySbKYXZLiv74FgkRIMsGBKjh9gxvJ7Fjkh8jPSRu2i0Nk/RG5EhblkZoH/91mzD/SMyiJe/9KcdNJSAkzNy/Bn3s0tcb7HF84vIRgQntjSwENraukZHPEHsGcVHRwsAC47aYrb+3PK/9U/bOCm+9bS0DlAZlqQkpwQG/c5n88zZuSzr91p3Hjfg9q2Puo7+inODjG9IIN9bX0MDoe54d4qPvzzVfx5nXPuOK9e+safNxEMCJ94y+yo5TnSLFi4ntnaxMNr9/Hvj2xmbkkWly2cwknTcslJTyGssGRWATddNJevvfNE5pZkU+AOUGeHUijIPLBFxI6mblp7BijMTjvoOSK/tPkZqf5jeD0LgKtPn86Hlzpdytq2Puo6vJ7FIG29o7vtka47Zyad/cOoOhXNRSeU0DM4wtNbG8lNT6E4O0Rn35AzAL5wCplpQR7bdGDh2043T7xyUz0Dw2HmlGRHfR4vaPz99UZmFWXxnfct4saL5vrpmPTUIJctLOMb717IqRV5PLx2H16a3avoprhpHC+I7G3to7Gzf1TLflZRFqlBp3L3gkVlWQ4PfuYc/njTuaOC100XzfNnqHl54GUXO1MbvVTKWOdXFnPGjHxeqG5hVlEm58ZIteS7OfXX670UWiiiZ5FGIOCMRe1u7WV7Y7c7UFtGdiiF9Xvb2eD2PhdVeD0L577F2SHmlmZT39nPproORsLqp2HA6VkA9A2NMLckm5RggNnFWexo7Kaxs5+P/+oVtjV0sbO5h2vPrCA7lMLQiJITSvHL6uW3K2IEiwMz9A70Vmqau5lTHP29j2ZKrvO+13f0j5posamuk437OvjgmRWcN6+YV3a1srO5xx9oPrXCCZ4FmaM/zwun5bF5f2fUwdrmrgHSUwNkpQXJz0wdNbuvs2/Ib3x5jx8QmF5wYN2MU94DnzHvezezKNPvgXuvSX5mGotnFvDrl3b7vb8H3QkGs6P0LADedmIZwYCwcpOTgty0r4Pqxm627O+kLDed4uyQHwivPsP5XL5Q3ez3uEfCyq6WXkpyQpTnZ7CntZdbHt7AC9UtBAR/5uPS2UVkpAbp7B/mpovm+RMXks2ChSusSiglwPmVxXz9XQsJBISUYMBvlSyZVUBRdohPnT8HEfFnM1UUZiIizCzKJBgQqhu7aekZHDUTyjMjYkZJQWaaXxF5OUpPRlrQTVH0st9rrXUP0NozOGrQPNI5c4qYW+JUsKdV5Pt52g21HRRnh8jLSKW1x9leZEahUzk+H7Evjjc24s3ciPWFqHBbhDuaeqgsy+b8yhK+esWCUcfc+dElvH9JBWfOKqTXnbZblJXmD955la03e8zpWfSP+iKnpQT8MpwxM9+/funsQoqyD35tpxc4r+3qna2U5oQ4b14xG//98phBT0T40qUnAPDhs2YQCEQfHPQqsxeqmwmIU9mfX1nMRSeU+C3MmUVZ7G7pYXtDF3OKs0hLCXBKeR7ra9tZs7uNjNQg89xyeC3x6QUZTC/IIKywcpMzPTUyDVWYleZXXHPdXt48d4zh0dfqeWprE1+4fx3gVIyXLixjTkkWHzizgurGboZGwuxp7SUlIP5zjjV2RlRX/xANnQORMSTtAAAgAElEQVTMLY3+3kdT7n4edjb3sL+jn0x3ptiTW5z/6ZTyPM6eU+gPzHvriU5ze1pFYxpVJ03LpaMv+tbnzW4+3/v+tY8ds8g4kIbKCqVw8YKyUWk3cF7XtGAAkQNjhV4jLj8zddT365IFpXT1DxNKCfhTbAG/YTJWYZYTYJ7a2sie1l5/yvrLNS2U5oQozk7zvw8XVpYwvSCDR1+rZ3tDFydOPfDel2SHmF6QyY6mHh5eu48vvm0+Fy8opa13iNSgMCU3nROm5DCjMJPPXDgnalmSIWX8Q94cVJXMtCD3fvKsUddfdlIZq2paWDyjcNT1XkXvVZ6hlCAzCjOpbuymtWdwVHfXM70gAxFQdT6YXs9ibLAAKC/IpKapx5/N1NI9SFvvEOUF0T+oIsI333MyOxq7SU91UmAnlOWwtaHLDxaNfmUdojCrmCe2NFLb1kthVhr7O/o5aVqu3/Lx0lBjRbZST4hYcxDNmbML4ekdFGencUp5Hk+5Uyr9GVNuLndHYzed/cP+zCLPgim57G/vp7I0/vMAflqjZ3DET2GN5y2VxfzhxnP98YRovDTUa/s6uWxhGTnpqeSkp3L3x5f6x8wsyuT56iZ6BkZY7KY/Tq3I5+fP1bC5rpN3Lprqj8VMGRMsAH71wk7mlWZTMea9PXNWIdWN3X6Kc25pNo++tt9fx7B5vzeDKo/z5hUzHFb+tqmewZEwu5p72NPaR3lBhv/cY03JTSc3PYU7n6shLSXAgilOhTWRnkVeRipzirN4dU8bdW7K8PnqZn99xknT8giKIOIEAm9w/YqTp7BqZ4s/s9Cz0A2Ym+s6/QaAp7l70O8NRK6sByfQ5Y5Jl/3iY0sOKq+IUJYXom9wxH9dvEbcvJLsUanKS04s5T8ffZ23nVjG9IIMtjV0MzUv/aD0cqRz5hTx479vZ1XEdiRhdd73gPvY4jY63rloKnc+W4MqXL+wzJ+oUZITIt8NWl+54gRuumgef1hTyxNbGpmWn0EwIPz4Q6cTDEjcshxpSe1ZiMgVIrJVRKpFJOo5tEXkAyKyWUQ2ichv3etOE5GX3Os2iMgHk1lOcN7QQJSpZ+9fPJ1V/3oJeZmjP4j5bn40MrV0Snkeq3a20tA5cNDgNjgBxctZO8HCOSYyDeWZXpDBhtoOf+ZGc88gLd0DFGbGzh+fO7eY686Z5V/2ZrEU56T55XWeL523VDqLrF6sbvEHb692x2RyQin+OMpYRVlpfuuxcpxgsXhmASLO6+KNU8CBCtPT6a7qHhtgb778BO76+JkHTWeNJjMtxW8Vjq1kxitjaozKFA40CsBJ9UUzsyiT/qEw+9r7mF/mVLSnVeQz4g4uf/M9J/vHepXl9IJMPziEFb7//lMP6t1curCUzLSg3+qsLM0mrM742rlziwgGhKIsZ/puemqQ7FCKHyhfr+9iT2tvzPEKcCrOH3/4DPIz0vjqHzZyz0u7gAM9mUQtmVXAqp2ttPQMcsaMfEScyQxT89IpzEojLzOVz144l2Vvneffp6Iwk1987MyDxkO8RY+Rs7TueKqaT91TRUNnv9+wystIZWA4TP/QCAPDI/QPhUeNWcQzNTdjVAPNe43mjumFzi3J5l/fcSJfvLTS76nHGq/wnDWnkLDCr1/a7a6dcnpQThrqQCYhLSXAuxdN87/fZ80u9Afoi7NDfGDJdFZ8/nxuush5zd62sIzUoPifmYrCzFHptcmQtJ6FiASBO4BLgVpgtYgsV9XNEcdUArcC56lqm4iUujf1Ah9V1e0iMg1YIyIrVTVp5yEdUY06T1lEyEw7+GXyKvrI1NJVp01j+XpnECpWuqiiMJO6jn7yR6WhDj52ekEGfe6CwOLsNBo7++nsH445ZhHN2XMKufvFXU7PIiLIlOWGqCzNpiQnxPPVzWS55wA4Z24RJTkhpualx5yzLeJ8YLc2dI3bgs9NT+ULl8xnUUWevzV3WjDg96jgwIwcp1yjg0VFYWbMfHs00wud3LOXGjkSvCA7uziL8yJWMUeaGVGBeAH0vHlFXH16Of/41nmjWrwVhRmU5oRYPLOAqXnO9hMfOWuGX6lEunhBGRu+cZnfAvbSRqrwgSUVnFqRTzg8+nM7rzSbYEDYWt/F3tbeuDOZwBnIPXtOIed/5ykeXruPYEBGfaYTsWRmIQ9WOWsSZhVnUZIdorFrYFRabWyqMpbsUArT8tL9z8RTWxtHbQW+xB3L8dLA7b1DpLhjW5GzoeL58mXzGYyYCegFi7Fb04sIn77ASfOU5KQjArPHCaSnVxSQGhQ21XUyvyyb8yuLWbe3ndKcEMNuKs6bFHLStFxmF2exs9lJ6c4uzmJDbQclOSFCKUG/lwVOcLz58hMO6n1OpmSmoZYC1apaAyAi9wNXAZsjjvk0cIeqtgGoaqP729/wR1XrRKQRKAGSFixUlQQasL7IMQvPBfNLKMpKo6Vn8KBcrGdmUSZr97SRlRbk1Io8FkzJOWiQD0a3jk8uz+NpN4UTrccSy9LZRaQEhGn5GaOm1pXlOsHgLfOKeW57E5Xul2R2cRY3X34CWVGCY6SKwkxqmrvHbWUB/NPbKgFnQRtAdnqKH5wAzpxV4FcMY3scEzW9IIP1e9v99M6RkBIMcNVp07hs4ZSY4xozIz4DXss4Jz3V32IjUmZaCq/869v8yy/ecnHMx/We3zO7OIuAOD2Rc+YW+bPzIoVSgswuzmL5+jpaewbj9iwi73P9ebP47mNbqSjIIJQysdTG4lkHZh5Nzctgan4GjV0DLJwWO70Xz7yyHLY3dtHVP8Q/P7ieBVNyyM9M5eWaVkrc75X3nWnvGyTNfY3GpqFiOWvO6MkMJ07N5fpzZ/HORVNj3icvI5Uff+j0UZMQoslIC3Lq9HyqdrexcGqu3yOZmneg8ef1LkWEfzh7Jg+vraUk25mG7QWLaG64YG5C/1+yJDMNVQ5E7k9Q614XaT4wX0ReEJGXReSKsQ8iIkuBNGBHlNtuEJEqEalqakp858howmESSnd4Tpmex8yiTE6O+PCkBgNceZozoBZtgBvgH86eyb+840REhKtOK+exL1wQtbKIrPBOKT/wHNECSyyFWWn8edl5XHf2zFFddK8LftEJJTR3D/LLF3YyNS+dzLQUPrCkIu6XBpwe1MfOmUVaSuIfHy8QZIWCo4LFkpkHxoLKcg4vWHitromkoRLx/649Pe5rUl7g5JHTUgIJVc6R4gWKsdJTnXGxuSVZUcfEPCdOzWVPay9nzymMORtsrI8snUlmWvCg1nUi5hRn+Y2Y8vwMyvMPtJwPRaXb23yhupmWnkFue9dCvvWek0lPDfjjN16Pr713yE9jRg5wT0RqMMC/XXnSuGmddy2aFnPiRyQvQCyclst5c4v5zjWncMmJpX4Dcmr+gffuk2+ZzV8/fz4iwoKpOaSnBmI2NI+2ZPYson0Lxs6HSwEqgYuA6cBzInKyl24SkanAvcDHVPWgDWNU9U7gToAlS5Yc1sYoYdWoYxaxLJiSyzM3v/Wg6z94ZgX3rdrjt9bHWjQ9n0XTD045jOUNnOemp4zqvcRKb8XitYS8nkVRVppfyb970TRerG7hgaq9LJyb+Bf73adO8zfFS9RUf/1F6qiey8nleaSnui3DQ/yye7z9i2ZOsMI+XKnBAOX5GWSFUibU4DgUt7z9REKp8YP0Vy4/gfecNo2LF5QmvAVEXqYzaO8N6E+EiLB4ZgFPbGmgLC/ENLflfDjBon8ozENraklPDbBkViFpKQGqvnapvy9Xnp+GGvTTxIn2LJLt/MoSfvr0Dk6fUUAgIHzwTGcqvDcOOC0velD6xHmzuWzhlAn37CZLMoNFLRC5f8J0oC7KMS+r6hCwU0S24gSP1SKSC/wV+JqqHvpZ4RMU1gNnRjscC6bk8tq/XT6hVnc0Xut4Wn7GqAAxkTGLSF5LLHItQyAg/Md7TyE/M5WTyw8tZZCoMjdY5LgVamZakN7BEabmpzO9IJOhkfBh723zntPLKc0NRV1hm2yfvmAOmZMwM2W8MQiY+FiPJ3JbjIn6+LmzqCzNJpQS5F2nTmNE9ZDn/1e6kwSefL2Rt8wr9r9L2RE9Um+8r713yB8LSHSAO9nOmVvEE1+6gHljZvHNLs7ijBn5MbdPSU89tJ7dZElmsFgNVIrIbGAfcC3w4THH/An4EHC3iBTjpKVqRCQN+CPwa1X9fRLL6JtozyKeww0U4HxwirOdwebIdQUTGbOI5PUsSsfkQ4MB4dZ3nHjoBU1QTiiFrLQgWSGnQvVSUTmhFM6ZU+TPPz8c6alBLl5QdtiPcyiuOzv6TKk3i3PnFXPuPGcCwGkV+VEH7BPlVbKqo7dGj+SnofqG/EWfx0rPAjgoUIDzmX/4pvOOQmmOjKQFC1UdFpFlwEogCNylqptE5HagSlWXu7ddJiKbgRHgZlVtEZF/AC4AikTkevchr1fVdckqb3iCA9yT4ebL5zM1b3TPIj/O1Nl4vFbX2LUMk0VEOHFqrj9zKDuUQk4oxV8fYownLyOVstwQDZ0DMVfWZ6YFSQs629Z4X9vDTWOa+JL66qrqCmDFmOtui/hbgS+5P5HH/Ab4TTLLNlZYJzbYOBm8XKd3YpPsUMoh5zNTg87q9Fgttcnwm0+d5ef08zNTR6UVjIlUWZpD7+BIzHEPESEvM5X23kGCAaeHnDGJC9TejOzb6jqSaagjLTMthYzU4CGnoDxjV6dPtsjVpv9x9SlHJF1njk//9LZKmroGYq4+BycV1d47RGowQG56StLP5/BmZ8HCNdF1FpOtKDvtkAe3j0WRe+EYM5a3kWI8+ZmptPcNEkoNHDOD28czCxaukfCx27MAZ3Vp4QTWWBhzvMvLcDbbzEgNHlOD28crCxYuZ+rssRssfvYPi4/I1F5jjhcFmalsqhsiK5Rig9uTwF5hl6oSJz161E3m7pLGvBF4Z8vLCqVQmnPsrk84XhzD1ePkirXrrDHm2FRZmkP/UJjqxu6ENxE0h86ChSscY9dZY8yx6f1LpvPlS+cDRD0hljmyLBy7nAHuo10KY0yiRITPXVLJRSeUTngDRzNxFixcamkoY96QTolzpkNz5FgayhVWJWjBwhhjorJg4XLGLI52KYwx5thkwcJls6GMMSY2CxaucFgJ2KthjDFRWfXoOpY3EjTGmKPNgoXL0lDGGBObBQvXsb7rrDHGHE1JDRYicoWIbBWRahG5JcYxHxCRzSKySUR+G3H9x0Rku/vzsWSWE2DE0lDGGBNT0hbliUgQuAO4FKgFVovIclXdHHFMJXArcJ6qtolIqXt9IfANYAmgwBr3vm3JKm84fGzvOmuMMUdTMnsWS4FqVa1R1UHgfuCqMcd8GrjDCwKq2uhefznwuKq2urc9DlyRxLIek+fgNsaYY0Uyg0U5sDficq17XaT5wHwReUFEXhaRKyZw3yNKFf/80MYYY0ZL5t5Q0WpejfL8lcBFwHTgORE5OcH7IiI3ADcAzJgx43DKalNnjTEmjmT2LGqBiojL04G6KMf8WVWHVHUnsBUneCRyX1T1TlVdoqpLSkpKDquwI7bdhzHGxJTMYLEaqBSR2SKSBlwLLB9zzJ+AtwKISDFOWqoGWAlcJiIFIlIAXOZelzS266wxxsQ2brAQkftF5HKZ4FQhVR0GluFU8luAB1V1k4jcLiJXuoetBFpEZDPwFHCzqraoaivwTZyAsxq43b0uaWyA2xhjYktkzOJu4BPAT0TkAeBuVa1O5MFVdQWwYsx1t0X8rcCX3J+x970LuCuR5zkSwqoELFoYY0xU4/YsVPUxVf0gzlTYeuApEXlWRK4TkePm5EnhsKWhjDEmloTGLNxxgw8D1wEbgP8DzgUeS17RJpeloYwxJrZxewYi8iBwCvBb4BpVrXVvuk9EXk1m4SaTTZ01xpjYEkkj/QJnNfVB6xxU9fQjX6SjI6y23YcxxsSSSBpqDuCfEd2dznpD8op0dKgqQduD1xhjokqkevysqrZ7F9y9mm5MXpGODjufhTHGxJZIsAhGXhCRAJCanOIcPSNhG7MwxphYEhmzeFxEfgf8DGd/phuBJ5JaqqMgbNt9GGNMTIkEi5uBm4Av4mzw9zecqbPHFdvuwxhjYhs3WKjqCPBj9+e4FVa1LcqNMSaGRNZZzAW+DSwE0r3rVXV+Ess16SwNZYwxsSUywH038CucFNTbgQdxznp3XLHtPowxJrZEgkWmqq4EUNUdqvo13G3Fjye23YcxxsSWyAD3gLs9+Q4R+SywDyhNbrEmn233YYwxsSUSLL4IZAOfxxm7yMXZsvy4YovyjDEmtrjBQkSCwNWqugrowtl19rjjbXtlwcIYY6KLO2bhTptdOkllOWpGwl6wOMoFMcaYY1QiA9xrReRhEfmQiFzp/STy4CJyhYhsFZFqEbklyu3Xi0iTiKxzfz4Vcdt3RWSTiGwRkR9N9LSuE+HGCjtTnjHGxJDImEUZ0AO8I+I6BZbHu5ObwroDuBSoBVaLyHJV3Tzm0AdUddmY+54LnAcscq96HrgQeDqB8k5Y2E1DWRbKGGOiS2QF96GOUywFqlW1BkBE7geuAsYGi6hPi7MAMA1nfUcq0HCI5Rj/ydyeRdCihTHGRJXICu47o12vquOd06Ic2BtxuRY4K8px14jIBcA24IuquldVXxKRp4D9OMHiJ6q6ZbyyHqqwDXAbY0xciYxZPBnx8wLOGouBBO4XreYde7a9R4BZqroIZyfbewBEZB5wIjAdJ+hc7AaU0U8gcoOIVIlIVVNTUwJFim7E0lDGGBNXImmoByIvi8i9wOMJPHYtUBFxeTpQN+axWyIu/hz4jvv31cDLqtrtPuejwNnAs2PufydwJ8CSJUsOOu1rojTs/LaehTHGRHcoJxKdDcxM4LjVQKWIzBaRNOBaxgyKi8jUiItXAl6qaQ9woYikiEgqzuD2JKShkvUMxhjzxpbImEUbB9JHAaAVOGga7FiqOiwiy4CVOGfbu0tVN4nI7UCVqi4HPu9Owx12H/d69+4PARcDG93nfkxVH5nIPzYRXrCwLcqNMSa6RKbOFkf8HVZvuXMCVHUFsGLMdbdF/H0rcGuU+40An0n0eQ6Xt84iiUs5jDHmDS2RNNQ7gWxVHVFVFZF8EXlXsgs2mWw2lDHGxJdIsLhdVTu8C6raDnwzeUWafDZmYYwx8SUSLKIdk0j66g3D3+7DehbGGBNVontDfVdEZorIDBH5HvBqsgs2mcLeRoLWtTDGmKgSCRbL3OP+jDP1VYGbklmoyaZ+z+LolsMYY45ViSzK6wb+eRLKctSM2AC3McbENW7PQkQeE5H8iMsFIvLX5BZrctmus8YYE18iaagydwYUAKraBkxLXpEmn50pzxhj4kskWIRFZLp3QURmJLE8R4U3G8pWcBtjTHSJTIG9DXhBRP7uXn4rx9kAt51W1Rhj4ktkgPuvIrIUOAdn2/Gvqmpj0ks2iQ6MWVi0MMaYaBLadVZVG1T1T8Ba4BMisj65xZpcaovyjDEmrkRmQ5WKyDIReRHYCmRxYHfY44Jt92GMMfHFDBYi8nER+RvwIs6Ji5YB+1X166p6fK3g9noWFi2MMSaqeGMWd+IEivd7wUFEDvlsdMeyAwPcFiyMMSaaeMGiHPggcIe7KO8BIHVSSjXJ1NJQxhgTV8w0lKo2quqPVfVc4O3AANAqIhvds90dN2zXWWOMiS/R2VC7VfW/VPVUnN5GQrWqiFwhIltFpFpEDjoVq4hcLyJNIrLO/flUxG0zRORvIrJFRDaLyKzE/qWJs+0+jDEmvgmfl0JVNwNfH+84EQkCdwCXArXAahFZ7t4/0gOquizKQ/wa+LaqPi4i2UB4omVNlH8ObosWxhgTVUI9i0O0FKhW1RpVHQTuB65K5I4ishBIUdXHwdn5VlV7k1XQsBuGbDaUMcZEl8xgUQ7sjbhc61431jUiskFEHhKRCve6+UC7iDwsIq+KyPfcnsooInKDiFSJSFVTU9MhF9TWWRhjTHyJLMpbFOVnpoiMd99oVe/YqbePALNUdRHwBHCPe30KcD7OeTTOBOYQZSGgqt6pqktUdUlJScl4/0pMtt2HMcbEl0jP4pfAGpwxhHuBKuCPwHYRuSTO/WqBiojL04G6yANUtUVVB9yLPwcWR9z3VTeFNQz8CTgjgbIeEtvuwxhj4kskWGwHFqvqae5sqMXAOuBy4Adx7rcaqBSR2SKSBlyLc1pWn4hMjbh4JbAl4r4FIuJ1Fy4Gxg6MHzE2wG2MMfElMhvqRFXd4F1Q1Y0icoaqVsdL26jqsIgsA1YCQeAuVd3krtGoUtXlwOdF5EpgGGjFTTWp6oiI/DPwpDhPsgan55EU3gpuixXGGBNdIsFih4j8GGc2EzjrLKpFJIRTycekqiuAFWOuuy3i71uBW2Pc93FgUQLlO2y2KM8YY+JLJA31UZwxhFtwKvY64GM4gSLemMUbhr/dRzLnhhljzBtYIic/6gW+4/6M1XHES3QUWM/CGGPiGzdYiMjZwDeAmZHHq+r8JJZrUh1YZ2HBwhhjoklkzOJXwFdwBplHkluco8MW5RljTHyJBItOVX0k6SU5iqxnYYwx8SUSLP4uIv8JPIyzTTkAkdNp3+j8vaEsWBhjTFSJBIu3jPkNzrYdFxz54hwdtkW5McbEl8hsqPMnoyBHk7fdR9AGLYwxJqqYwUJEPqSqvxORz0e7XVV/lLxiTa4RG7Mwxpi44vUsCtzfh76d6xuEzYYyxpj4YgYLVf2p+3vcs+K90XmL8myLcmOMiS6RRXnFwCeAWYxelHdD8oo1udR6FsYYE1cis6H+DLwMPM/xuijP7VrYALcxxkSXSLDIUtUvJ70kR9GIpaGMMSauRPZZfVRELkt6SY4iS0MZY0x8iQSLzwKPiUi3iLSKSJuItCa7YJPJtvswxpj4EgkWxUAqkIczjbaYBKfTisgVIrJVRKpF5JYot18vIk0iss79+dSY23NFZJ+I/CSR5ztUtkW5McbEF29RXqWqbgdOinFI3L2hRCQI3AFcinPypNUislxVx55L+wFVXRbjYb4JPBPveY6EsJ38yBhj4oo3wH0L8EmcCn+sRPaGWgpUq2oNgIjcD1wFjA0WUYnIYqAMeAxYksh9DpU3G8p6FsYYE128RXmfdH8f6t5Q5cDeiMu1wFlRjrtGRC4AtgFfVNW9IhIAfgBcxyScutXSUMYYE18iU2cRkQXAQiDdu05Vfzve3aJcp2MuPwL8TlUHROSzwD3AxcBNwAo3cMQr1w3ADQAzZswY79+Iybb7MMaY+BJZwf014DJgAbASuBxngd54waIWqIi4PB2oizxAVVsiLv6cA+f5Pgc4X0RuArKBNBHpVtVbxtz/TuBOgCVLlowNRAmz7T6MMSa+RIZ0Pwi8FdivqtcBp5JYj2Q1UCkis0UkDbgWWB55gIhMjbh4JbAFQFU/oqozVHUW8M/Ar8cGiiMpHFZbvW2MMXEkUun3qeqIiAyLSA5QD8wZ706qOiwiy3B6I0HgLlXdJCK3A1Wquhz4vIhcCQwDrcD1h/qPHI6wqqWgjDEmjkSCxasikg/cBVQBncDaRB5cVVcAK8Zcd1vE37cCt47zGHcDdyfyfIcqrJaCMsaYeOIGC3Fq0H9T1XbgDhFZCeSqakLB4o1CrWdhjDFxxR2zUGfTpL9EXK4+3gIFeGkoixbGGBNLIgPcr4jIGUkvyVE0EoagBQtjjIkp3nYfKao6DLwF+LSI7AB6cNZPqKoeNwEkrIrFCmOMiS3emMUrwBnAeyapLEeNqhKwQQtjjIkpXrAQAFXdMUllOWrCalt9GGNMPPGCRYmIfCnWjar6wySU56iwdRbGGBNfvGARxNlq47ivRm02lDHGxBcvWOxX1dsnrSRHUThsaShjjIkn3tTZN03taWkoY4yJL16wSPp5JI4Vtt2HMcbEFzNYqGrrZBbkaHKmzh7tUhhjzLHLqkhgRNVWcBtjTBwWLLB1FsYYMx4LFth2H8YYMx4LFnhblFu0MMaYWCxYYOssjDFmPEkNFiJyhYhsFZFqETnoHNoicr2INInIOvfnU+71p4nISyKySUQ2iMgHk1nOEdtI0Bhj4krktKqHRESCwB3ApUAtsFpElqvq5jGHPqCqy8Zc1wt8VFW3i8g0YI2IrHTP2HfE2ZnyjDEmvmT2LJYC1apao6qDwP3AVYncUVW3qep29+86oBEoSVZBbTaUMcbEl8xgUQ7sjbhc61431jVuqukhEakYe6OILAXSgKRtlW7bfRhjTHzJDBbRql8dc/kRYJaqLgKeAO4Z9QAiU4F7gY+ravigJxC5QUSqRKSqqanpkAtq230YY0x8yQwWtUBkT2E6UBd5gKq2qOqAe/HnwGLvNhHJBf4KfE1VX472BKp6p6ouUdUlJSWHnqUKh5WgdS2MMSamZAaL1UCliMwWkTTgWmB55AFuz8FzJbDFvT4N+CPwa1X9fRLLCFgayhhjxpO02VCqOiwiy4CVOCdSuktVN4nI7UCVqi4HPi8iVwLDQCtwvXv3DwAXAEUi4l13vaquS0ZZnRXcFi2MMSaWpAULAFVdAawYc91tEX/fCtwa5X6/AX6TzLJFcmZDTdazGWPMG4+t4Ma2+zDGmPFYsABGbIDbGGPismCBTZ01xpjxWLDAtvswxpjxWLDAtvswxpjxWLDA1lkYY8x4LFjgDHBbz8IYY2KzYAGopaGMMSYuCxa4aSh7JYwxJiarIrHtPowxZjwWLLDZUMYYMx4LFjg9i6DFCmOMicmCBd7UWYsWxhgTiwULIBy27T6MMSYeCxbYdh/GGDMeCxbAiKWhjDEmLgsWuLOhrGthjDExJTVYiMgVIrJVRKpF5JYot18vIk0iss79+VTEbR8Tke3uz8eSWU5LQxljTHxJO62qiASBO4BLgVpgtYgsV9XNYw59QFWXjblvIfANYAmgwBr3vm3JKKutszDGmIq9uHsAAAncSURBVPiS2bNYClSrao2qDgL3A1cleN/LgcdVtdUNEI8DVySpnLbrrDHGjCOZwaIc2Btxuda9bqxrRGSDiDwkIhUTua+I3CAiVSJS1dTUdMgFHQnbdh/GGBNPMoNFtNpXx1x+BJilqouAJ4B7JnBfVPVOVV2iqktKSkoOuaCq2Dm4jTEmjmQGi1qgIuLydKAu8gBVbVHVAffiz4HFid73SLI0lDHGxJfMYLEaqBSR2SKSBlwLLI88QESmRly8Etji/r0SuExECkSkALjMvS4pbLsPY4yJL2mzoVR1WESW4VTyQeAuVd0kIrcDVaq6HPi8iFwJDAOtwPXufVtF5Js4AQfgdlVtTVZZw2rbfRhjTDxJCxYAqroCWDHmutsi/r4VuDXGfe8C7kpm+TzhsKWhjDEmHlvBjbtFuUULY4yJyYIFloYyxpjxvOmDhaozI9c6FsYYE9ubPliE3dUbNhvKGGNie9MHi5Gw9SyMMWY8b/pgEfbSUBYtjDEmpjd9sFBLQxljzLje9MEibAPcxhgzLgsWfrCwaGGMMbFYsAg7v22dhTHGxGbBwu1ZBC1WGGNMTBYsbDaUMcaM600fLFJTArzzlKnMLMo62kUxxphjVlJ3nX0jyE1P5Y6PnHG0i2GMMce0N33PwhhjzPgsWBhjjBlXUoOFiFwhIltFpFpEbolz3PtEREVkiXs5VUTuEZGNIrJFRKKeIMkYY8zkSFqwEJEgcAfwdmAh8CERWRjluBzg88CqiKvfD4RU9RRgMfAZEZmVrLIaY4yJL5k9i6VAtarWqOogcD9wVZTjvgl8F+iPuE6BLBFJATKAQaAziWU1xhgTRzKDRTmwN+JyrXudT0ROBypU9S9j7vsQ0APsB/YA31fV1iSW1RhjTBzJDBbRVrmpf6NIAPhv4MtRjlsKjADTgNnAl0VkzkFPIHKDiFSJSFVTU9ORKbUxxpiDJDNY1AIVEZenA3URl3OAk4GnRWQXcDaw3B3k/jDwmKoOqWoj8AKwZOwTqOqdqrpEVZeUlJQk6d8wxhgj3jmoj/gDO+MN24BLgH3AauDDqropxvFPA/+sqlUi8lVgAfAJINO977WquiHO8zUBuw+jyMVA82HcP1msXBNzrJYLjt2yWbkm5lgtFxxa2Waq6rit7aSt4FbVYRFZBqwEgsBdqrpJRG4HqlR1eZy73wH8CngNJ531q3iBwn2+w+paiEiVqh7UeznarFwTc6yWC47dslm5JuZYLRckt2xJ3e5DVVcAK8Zcd1uMYy+K+LsbZ/qsMcaYY4Ct4DbGGDMuCxYH3Hm0C/D/27u3ELvqK47j3x9RQ+ql1mgl4CWxpgULNQYRqdWHWrQNrakKNSI0tEKpWKoUREug+OBLLIoEpaI0eMG2UlppXpRIkEip93RGEzSNsXmwThIVbBQlaFx9+K/BnensszM6578Pzu8Dh7NnzWFmzdr/2f99OWftFs5rZkY1Lxjd3JzXzIxqXjDE3IZ2gdvMzD4/fGRhZmad5vxkcajNDivkcbKkJ7Jx4jZJ12X8Zkn/kTSWjxU95bcrGzuOSXo+Y8dJelzSjnz+UuWcvtaoy5ikfZKu76NmktZL2itpayM2bX1UrMsx96Kkod1QpSWv30p6JX/3I5KOzfhiSR806nb3sPIakFvrupP066zZdkkXV87r4UZOuySNZbxazQZsI+qMs4iYsw/KW3p3AqcBRwDjwBk95bIIWJ7LR1M+o3IGcDPl8yd912oXcPyU2K3ATbl8E7C253W5Gzi1j5oBFwDLga1d9QFWAI9S3hZ+LvBM5bwuAg7L5bWNvBY3X9dTzaZdd/m/MA7Mp3R12AnMq5XXlO/fBvymds0GbCOqjLO5fmRxqM0Ohy4iJiJiSy6/C7zMlF5aI2glcH8u3w/8sMdcLgR2RsRn+WDmpxYRTwJT+5e11Wcl8EAUTwPHSlpUK6+I2BgRH+WXT1O6K1TXUrM2K4E/RcT+iPg38Crl/7dqXpIE/Aj44zB+9yADthFVxtlcnyw6mx32QaUd+1l80rb9F3kYub72qZ6GADZKekHSzzJ2YkRMQBnIwJd7yg1gFQf/A49CzdrqM0rj7qeUvc9JSyT9U9JmSef3lNN0625UanY+sCcidjRi1Ws2ZRtRZZzN9cliYLPDPkg6CvgLcH1E7AN+B3wFWEbpwntbT6mdFxHLKfcnuVbSBT3l8X8kHQFcAvw5Q6NSszYjMe4krQE+Ah7K0ARwSkScBfwK+IOkYyqn1bbuRqJmwJUcvFNSvWbTbCNaXzpN7FPXbK5PFl3NDquSdDhlEDwUEX8FiIg9EXEgIj4G7mVIh95dIuKNfN4LPJJ57Jk8rM3nvX3kRpnAtkTEnsxxJGpGe316H3eSVgPfB66KPMGdp3jezuUXKNcFvlozrwHrbhRqdhhwGfDwZKx2zabbRlBpnM31yeI5YKmkJbl3ugoY1LNqaPJc6O+BlyPi9ka8eY7xUkq/rNq5HalyR0MkHUm5QLqVUqvV+bLVwN9q55YO2tsbhZqltvpsAH6c71Y5F/jv5GmEGiR9F7gRuCQi3m/ET1C5wyUqtwRYCrxWK6/8vW3rbgOwStJ8SUsyt2dr5gZ8B3glIl6fDNSsWds2glrjrMZV/FF+UN4x8C/KHsGaHvP4FuUQ8UVgLB8rgAeBlzK+AVjUQ26nUd6JMg5sm6wTsBDYBOzI5+N6yO0LwNvAFxux6jWjTFYTwIeUPbqr2+pDOT1wV465l4CzK+f1KuVc9uQ4uztfe3mu33FgC/CDHmrWuu6ANVmz7cD3auaV8fuAn095bbWaDdhGVBln/gS3mZl1muunoczM7BB4sjAzs06eLMzMrJMnCzMz6+TJwszMOnmyMJsBSQd0cKfbWetUnB1M+/pMiNlAQ70Ht9nn0AcRsazvJMxq85GF2SzIexyslfRsPk7P+KmSNmVjvE2STsn4iSr3khjPxzfzR82TdG/er2CjpAW9/VFmDZ4szGZmwZTTUFc0vrcvIs4B7gTuyNidlDbR36A07FuX8XXA5og4k3LvhG0ZXwrcFRFfB96hfELYrHf+BLfZDEh6LyKOmia+C/h2RLyWzd52R8RCSW9RWlZ8mPGJiDhe0pvASRGxv/EzFgOPR8TS/PpG4PCIuGX4f5nZYD6yMJs90bLc9prp7G8sH8DXFW1EeLIwmz1XNJ6fyuV/ULoZA1wF/D2XNwHXAEia18N9I8xmxHstZjOzQNJY4+vHImLy7bPzJT1D2Qm7MmO/BNZLugF4E/hJxq8D7pF0NeUI4hpKp1OzkeRrFmazIK9ZnB0Rb/Wdi9kw+DSUmZl18pGFmZl18pGFmZl18mRhZmadPFmYmVknTxZmZtbJk4WZmXXyZGFmZp3+B6G93nsyUeaVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e9439acf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(acc_full)\n",
    "plt.ylabel('Training Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.savefig('train_acc.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsvXmYJVd1J/g7EfG23GpXad+gBIhdljFGDWa37A+D3dNgYMY9dveAPYax3bTpAX9t3IM3hsUrcrsBL+waLBtJbgsEZhcgpBJCSFWlkkpVKtWmWrNye1ssZ/64cW7cGy/ivZdZLzLzFfH7vvwyM/JlxH3xbtxzz+93FmJmlChRokSJEv3grPUASpQoUaLE+kdpLEqUKFGixECUxqJEiRIlSgxEaSxKlChRosRAlMaiRIkSJUoMRGksSpQoUaLEQJTGokSJEiVKDERpLEqUKFGixECUxqJEiRIlSgyEt9YDGBW2bt3KV1555VoPo0SJEiXGCvfdd98pZt426HXnjbG48sorsXPnzrUeRokSJUqMFYjo4DCvK2moEiVKlCgxEKWxKFGiRIkSA1EaixIlSpQoMRCFGgsiupGI9hLRPiJ6V85r3kBEu4loFxF9xjj+/vjYHiL6CyKiIsdaokSJEiXyUZjATUQugJsAvArAYQD3EtHtzLzbeM0OAO8GcAMzzxLRBfHxFwG4AcBz4pfeBeCnAHy9qPGWKFGiRIl8FOlZvADAPmbez8xdADcDeF3qNW8BcBMzzwIAM5+IjzOAOoAqgBqACoDjBY61RIkSJUr0QZHG4hIAh4zfD8fHTFwD4Boi+jYR3U1ENwIAM38XwNcAHIu/7mTmPekLENFbiWgnEe08efJkIW+iRIkSJUoUayyyNIZ0D1cPwA4ALwXwJgAfI6KNRPRUAM8AcCmUgXk5Eb2k52TMH2Hm65n5+m3bBuaUlFgD3PXoKTx+ammth1GiRIlzRJHG4jCAy4zfLwVwNOM1tzGzz8wHAOyFMh6/AOBuZl5k5kUAXwDwwgLHWqIgvONzP8BHv7V/rYdRokSJc0SRxuJeADuI6CoiqgJ4I4DbU6+5FcDLAICItkLRUvsBPAHgp4jII6IKlLjdQ0OVWP9o+yG6QbTWwyhRosQ5ojBjwcwBgLcDuBNqof8cM+8iovcS0Wvjl90J4DQR7YbSKN7JzKcB3ALgMQAPAngAwAPM/M9FjbVEcQgiRshp9rFEiRLjhkJrQzHzHQDuSB17j/EzA3hH/GW+JgTwq0WOrcTqIAgZYVQaixIlxh1lBneJwsDM8KMIQWksxgKPnVzE8fn2Wg+jxDpFaSxKFIYwYjADYVgai3HAb3z2frz/i3vXehgl1ilKY1GiMIhHUXoW44GlToClTrDWwyixTlEaixKFwQ9VFFQYldFQ44CQy2CEEvkojUWJwhCEpWcxTogiICo/qxI5KI1FicKQeBblAjQOCCMuDXuJXJTGokRh8EvNYqwQMSMqaagSOSiNRYnCEJSexVghYtbUYYkSaZTGokRh8OOFpzQW44GwzLYv0QelsShRGErNYrwQRlwK3CVyURqLEoWhjIYaL0RcflYl8lEaixKFwY/KPItxQhiVAneJfJTGokRhKD2L8ULEZdHHEvkojUWJwlBqFuOF0liU6IfSWJQoDGIsynDM8UAYlcaiRD4KNRZEdCMR7SWifUT0rpzXvIGIdhPRLiL6THzsZUT0A+OrTUQ/X+RYS4weQRk6OzZgZkRcflYl8lFY8yMicgHcBOBVUL227yWi25l5t/GaHQDeDeAGZp4logsAgJm/BuB58Ws2A9gH4EtFjfVHHb/9Dw9gxwVT+NWfespIzxvEwnapWax/yEdU5lmUyEORnsULAOxj5v3M3AVwM4DXpV7zFgA3MfMsADDziYzz/DsAX2DmZoFj/ZHGvY+fwT0Hzoz8vF3tWZTRUOsd4lGUnkWJPBRpLC4BcMj4/XB8zMQ1AK4hom8T0d1EdGPGed4I4LMFjXHFCMII33r05FoPYyQIQsZ82y/gvKVnMS6QkNnSWJTIQ5HGgjKOpWeiB2AHgJcCeBOAjxHRRn0CoosAPBvAnZkXIHorEe0kop0nT67uwv3NR0/il/7mHjx6fGFVr1sEgijCQnv0TW9Esyizgtc/SmNRYhCKNBaHAVxm/H4pgKMZr7mNmX1mPgBgL5TxELwBwOeZOXPby8wfYebrmfn6bdu2jXDogyGL65ml7qpetwiEERdiLPxSsxgblDRUiUEo0ljcC2AHEV1FRFUoOun21GtuBfAyACCirVC01H7j72/COqSggKRI3lJ3/NtQBlExNJQflHkW4wKRlVZD4P7TLz+Cf7zvcOHXKTFaFGYsmDkA8HYoCmkPgM8x8y4iei8RvTZ+2Z0AThPRbgBfA/BOZj4NAER0JZRn8o2ixngukByCpU64xiM5dwQhY7ETjJwuMntwcxlls64RriINdfsDR/Gve44Xfp0So0VhobMAwMx3ALgjdew9xs8M4B3xV/p/H0evIL5ukBiL88GziMAMLHYDzNQrIzuvbyTjRQy4WSpWiXWB1dQsgiiy5kaJ8UCZwb1CdGOKZfE8MBayQIxat5BoKCDJuSixPhGtomYRReV8GEeUxmKFkJ1RszveNBQz6/eyMGLdwjeMRalbrG8IDbUaVWeDKCpLwIwhSmOxQpwvNJS5ho/as/CNk5cRUesboaEvrca1zI1EifFAaSxWCJns405DmXTAfGu0noVJQ4XlTnJdQ6YBc/F5MWHE5eZhDFEaixVCNItxp6FMOmDknkVYehbjAjNktujw2aA0FmOJ0lisEN3zxrMwjUWpWfyowtQqiv6swogtr7PEeKA0FivE+aJZmA/t/MijoVZvt1ri3GBST0WL3EHEpcA9hiiNxQrhB3EG95gbC3MXOeosbj8qNYtxgWnMi6aIooituVFieLS6If72rgNrUm+tNBYrhPYsxl2ziIrTLAJLsygXh/UMc9NQ5ELEzKVncQ64a98pvPd/7sbuY/Orfu3SWKwQ3fOGhipS4C41i3GBacuL9Czk1KVmsTLIM9X2V3+TWhqLFeJ8DJ0dvcBdRkONC0waqkjPQjYNfjkfVgS5fxKNuZoojcUKYWZwj3ORvLBIGioqPYtxQbRKobM6+a/0LFYEuX+dNbh/pbFYIcSzCCNGZw2s/KggRs+h0Sfl+VZtqGIWoIW2j7sePVXIuX+UYHoTReoJui97qVmsCKVnMYYwP6xxpqJk8m2aqBaalFdUH+5bf3AUv/S338PciA3djxrCVQqdTWio8d1grSW0Z1Eai/FB19g1N8e4p4Xs9DZNVkeuWQRhBIfk52IWoKVOAGagswaC3/mEcJWS8hIaqvQsVoKg9CzGDybFMs6eRaA9iwqWuuFIueQgYtQrLoDiFiDpxlcKpucGc6O/KsaibIi1IohRP++MBRHdSER7iWgfEb0r5zVvIKLdRLSLiD5jHL+ciL5ERHviv19Z5FiXCz9gTFbVQjjOrVVlh7dpogpgtIavG0RoiLEoaGEQo+2PsW60HrBataFM7aqMkFs+wni+d4PV96QL65RHRC6AmwC8CsBhAPcS0e3MvNt4zQ4A7wZwAzPPEtEFxik+AeAPmfnLRDQFYF2tBn4YYeNEFUvd1lh7FrLT2zypjMVCO8DG2HCcK0zPoqiFoRsbu7Lk9bnB1CmKpIjClJAeT48SQyI4TzWLFwDYx8z7mbkL4GYAr0u95i0AbmLmWQBg5hMAQETXAvCY+cvx8UVmbhY41mWjG0bYNKlakI6zZiFC44YJ9V5GKXIHYYRaRU2xosp9iJHolsbinLBataEsY1GK3MtGdJ7SUJcAOGT8fhi9PbWvAXANEX2biO4mohuN42eJ6J+I6H4i+kDsqVggorcS0U4i2nny5MlC3kQe/DDS1M04Z3HLIt7QHsDoJqEfMupesZ6FpqFKwfScYC7iRWoWwSqF6J6v0AL3eZZnQRnH0rPDA7ADwEsBvAnAx4hoY3z8xQB+G8CPA7gawC/3nIz5I8x8PTNfv23bttGNfAj4IWNDQ+3Gx5mGEuNQBF3khxHq4lkUbizKXeq5YLVKlJvnFq92oe3jXf/4w7F+jlYLsrk732iowwAuM36/FMDRjNfcxsw+Mx8AsBfKeBwGcH9MYQUAbgVwXYFjXTb8IPEsmuMscMcPb91TU2GUuz1bsyhmcstDUwrc54ZwlaOhgGSuPXh4Djffewg/PHS2sOueL1jX0VBZ9M+QuBfADiK6ioiqAN4I4PbUa24F8LL4Oluh6Kf98f9uIiJxF14OYDfWEbphhImai6rrYHGMNQt5eItY1JVnUXDorAjcyzj/qPNJ1gKj3oWvhWchxiKI7O8l8rHek/L2xZrBtcs5cewRvB3AnQD2APgcM+8iovcS0Wvjl90J4DQR7QbwNQDvZObTzBxCUVBfIaIHoSitjy7n+kWCmdENI9RcB5M1d6w1C1lstbEYpWcRsqahCtMslulZHJ5t4nnv/TK+/8RsIeNZDTxyfAHP+W93Yt+JhZGdc7VqQ5mbEaGhktyL0jschLUs9zFM6OxzoLyCjxGRA+BvAdzMzAMLqjPzHQDuSB17j/EzA3hH/JX+3y/H1153CCMGM1BxHUxUvbHOs5AyHDVv9NrC6ngWy9Msjs+3EUaMg6eXcN3lmwoZU9E4Pt9GxMCxuTaeesH0SM65WgJ3X8+iFLwHIvEs1mGJcmZeYOaPMvOLAPwXAL8H4BgRfZyInlr4CNchZDde8RxM1byx9iyCHhpqNA+sNLkpPs9ieaGzHV+9br41/p9ZenfJzPjoN/fjbLO77HOuhbFIinHGxQVLGmog1nW5DyJyiei1RPR5AH8O4ENQ0Un/jJTX8KMCWZgqroOpujfyAnyrCdnNST7EqMp9JMJ57FkUFK203NBZ4XrHWbeQzyy9YOw/tYQ/vGMPvrz7+LLPuSaaRUqrKI3FYGgaag2i/4ahoR6F0hM+wMzfMY7fQkQvKWZY6xuyQFVdwoZGBcfn22s8opWjKM8i0FpITG8VtA74y8zgFvd9nA287MTTC4Z4uCsRP81TFZmUZ+dZpDSLMvx5INa9ZsHMi1l/YObfGPF4xgK+4VlsaFTwyPHRCY2rDVl4kuS50UxCWch0baiCxMvlahaykM6PsbHIK/kgEVIrWUisch8R4/++5YeYqnv43dcsK65lIEzxXAx9WrsokY/1Xu7jpjhRDgBARJuI6G8LHNO6hx/EmkVsLMa5l4Kf8gBGJTLKLrFwzSJYJg0lmsUY01B5u8ulOIR7JRRFmobadWwOu47OncMos2GWfQnS0VClwD0Q0XrWLKA8C50tE9dxen5xQ1r/0JqF52CmUcFiJyi0b3GR6M2zGJGxiFI0VMG1oX6UaKg8zULTUP5KaCjbWHSDqJDda1a5DzlWlPd5PmFdC9wAHCLSMYZEtBkFVqsdB6Q1C+bxXXyCiEEEVL3R5kPIPaoVXBtKDPeweRbng8CdJ3JKCHc3XH5YZdpY+CGjvQKjs5zrpI1EWd9rMORerUXo7DCL/ocAfIeIbol/fz2APyxuSOsfYtWFhgKAuZavK7eOE4IwgucQvLil3ahExiS8mOA6VGDzo+UK3GIsxtO4A/m7y6URaRYRK8+Csqq7nSNCqxS6TUMVGYV1vkDs6boUuJn5E0R0H1RZDgLwb82eFD+K0J6FZxuLcUQYMVyH4LmjTcqThcBzHLgOFV51tju0ZqF2ZPNj+nkBRjRUj8AdZh4f7pzJz0HEhRVmDK0MbpuGOh/6cv/xF/ZgY6OK//OlTynk/HmRcKuBoQoJMvMuAJ8DcBuARSK6vNBRrXOYeRYzdWVvx9VY+CGj4jjasxgVFaA9C1d5LUXx0d0VRkONs2ch93aUobOWZxEbiyKoDlPE7vEsxpyGanYD/N23H8dd+4prlyD3b11GQ8UJeY8COADgGwAeB/CFgse1pljqBHi0TzhsshA6mnpaibFodoM1D7sNowium9BQo1rUJdKl4tqeRRQxHjw8uiiblYbOtvxwbMua50VDSfXjFdFQKS3BD3lFQvkg9Cv3Ma591A+eXsKpxQ6+ve80ukFUqPYiRn1dGgsAvw/ghQAeYearALwCwLcLHdUa45N3H8Rr/vIutP3snZWIqVX33Gioj33rAF774bvWNBkpiBieo3QFYJSeRUxDuU7sWajz3rXvFH7uw3fhwKmlEV2Hre+DYO6WF8fUu8jPs1DvrbOC+RSmQme7YYR2AZ6FlWeRCp0d12iot33m+/jfPvY9fGnXkwCKTS409SouMHkyC8MYC5+ZT0NFRTnM/DUAzyt4XGuKkwsddIIIj5/OXtB0Up5H52Qs9j65gLYfoZVjlFYDQcjwHAdEoxWitfflEFzH0ZN8Nq5bNIpopDBiPd6hPQtjtzyuVFSeZnFOAndPNJTaIY9adD4fCwkutgM8/OQCbvn+YQDFJhfatbXWn7E4S0RTAL4J4NNE9OcAxvMpGxKy4zxwMttYmJpFo+Ki4tKKjMX+eHfd6g42Fp+6+yBuvf/Isq8xCEEscAOA59DIREZ58MWziFK74X6LkB9G+C+3PIDDs/3brpsGYrk0FDC+iXl5rTXPqdyHsUtVu1bE5xrtRsY0COdLIUEZNzPgOlToIm4+N6stcg9jLF4HoAngPwH4IoDHAPxckYNaa0jZhP05VIlMhqqrduQryeKOIsbj8fmbQxiLT373IP4x3rmMEmEUwXMTYzEqkdHXmgVZmoUsZP3qDx0608Tndh7G3fvP9L/GioxFcq/H1VgkmoU9b3SexQoW+DCCDpU16adR6xbpsiLW9zHVkKKI8RNXbcYvPP8SvOgpW1aFhgKSyL7VQl9jEXfJu42ZI2YOmPnjzPwXMS01EER0IxHtJaJ9RPSunNe8gYh2E9EuIvqMcTwkoh/EX+kOe4ViITYWeby6WRsKAGYalaFDMf0wwlzTx/GFtqafhjEWcy2/kNhqP9YsAOUFjCwpz8hF8dyE3pIJ3o9ykAVl0ENn7uC6wbCaRaR7d4wrDZWfZ7Hy0Flm1vO51U3+f9S6RWYhwTGvDRVEjKu3TeJPf/F52DhRLfR9RGvoWfTNs2DmkIiaRLSBmZcVwhIbmpsAvAqqp/a9RHS7maNBRDsAvBvADcw8S0QXGKdoMfOaaCNLQxsLtcgux7P4668/ho9/9yA++Pqkr1PLH7xozbX8QqJ3wlizAJRnMapCgvLAeDmeRb9ubMNGx5iL4nI0i61TNRw52xpbY5HoNPb90YUEVyJwR4yKQ+gCloY2as8ii3Mfd80iNKjcikOFRtmZhmi1E/OGyeBuA3iQiL4MQK+eQ1ScfQGAfcy8HwCI6GYoSstM6HsLgJvielNg5hPLGHth0JpFjrHQGdzxDnVDo4LTi8M1nDlytoVTix18buchfczcyeVdT4V6jv5hsjQLl0b2wOpoKMex8izEs+inWQxbsnqlNNS2aWUsxjUxL682VPNcakMxwyEVQm3SGyP3LPoVEhxjz8Kl0T9DWTCfm9UOnx1Gs/gXAL8LJXDfZ3wNwiUADhm/H46PmbgGwDVE9G0iupuIbjT+VieinfHxnx/ieiOD7NDOLHUzu47Jzq3qJsZiWM9CKK4vPvSkPtbMacv6Lz88hnsfP6PPXcSOJYgi7SF5zuhoqMDQdRyinmSiYYzFoEgccwc97APaCSJsnaoCGJ6G+uw9T2Dvk+unDL0ssmaIbBQxlrrDV509eraFj3zzMR1+GUUMxyE4DhXrWcTXq3lObzTUmIbOKs8i9s5HSOXmXUuw2p7FMG1VP571NcS5syrLpO+iB2AHgJcCeBNUn28ph345M18P4M0A/oyIevLnieitsUHZefLk6LImF9o+Lt3UAJAtcpslyoHlGQvxWiJOej3khc5+6Et78T++sV+fuwiOMuzxLEZVG0ryLMjWLIYwFkEOzZJ3DWAZbVWDCBNVDxNVd+jw3d+7fRf+YeehwS9cJWRpFk1jDg2ziNzx4DH80R0P48yS2gyFrOaBS2TlF+XlGq0UYRTBdQgV19GfbzTmnkUYsQ4SqYyQys271kRVrRvrzrMgogNEtD/9NcS5DwO4zPj9UgBHM15zGzP7zHwAwF4o4wFmPhp/3w/g68goi87MH2Hm65n5+m3btg0xpMFgZix2Ajzn0g0AssNn/VBNeFlkNzQqmG/7Q5UpXzT6dV978QyAfIG7G0Y4s9Qp1rMIWdeFGmUNJ9EblGbhGJrFYBoqGpaGio020XI0ixA1z8F03A53mM8sCIsp171ShJqGSuaNUFAODWcsxAi0NC0ITUOZ1WZH/b6F9nQNanLco6HMDZfrOIXSUEEU6U3muvMsAFwP4MfjrxcD+AsAnxri/+4FsIOIriKiKoA3AkhHNd0KVaAQRLQVipbaHzdYqhnHb4CtdRSGlh8iYuAZF87AdShTt/DDhLoBkJQp7wymNRbbAZ59iTJE8j3PWIQR4/RSV3Pr/pARP2l8/v7DeOkHvpa5MAZRpKOhKiOc6PLgVxw7g1tojaE8iyFpqImKu6w8i1rFwXS9ggcOn8Xzf//L+O5j+cF9UcSIeG1KQuchK89CNiGbJqpDLfBiJMRoMDNcBz001Kg9iyiOvqu4pD/fMBo8J9p+iJ/846/gqw8vv7940bCeIbdYgTtioKE9i3UUOgsAzHza+DrCzH8G4OVD/F8A4O0A7gSwB8DnmHkXEb2XiF4bv+xOAKeJaDdUn+93xmG5zwCwk4geiI+/b7Uq3QpNtGmyiss3T2Qai24YaQoKAGbqKot7GMF0sRNgx/YpfOI/vAC/HlembOVoFn7IOLPYPWcaau+Ti3j8dBNnM8YXWLui0XkWYnQqntSGinn2IaKhhi3/IA/lRM1bRrmPCDXPxUzdw8NPLmCu5eNQn+Q/PzXu9YCsDG4Jm900WR3Ss4is72Es0irPwtAsivAsiJQ+FqY8iz5zb7bZxbG5NvbnJMquFWQzYVG5BdJpa+lZDIyGIqLrjF8dKE9jepiTM/MdAO5IHXuP8TMDeEf8Zb7mOwCePcw1Rg3ZoU3XPVy1dTJbswgjLW4DKs8CUOGtl/W8uvf80zUPL7lG0WaeQ308iwgLnQAnFzrquiucHGKMzix1sHmyav0tCJP4+oo7Or5VDJv0ypCFPYuG2nV0DmHEeM6lG62/DfJytLGoukM/OJ1AaKik90i/naAW5n17F/+VPcfx2udeDCqi6cMAZGkWMm83T1Sx78QimLlnbCcXOnjg0Fm88trt2nvQNFT8escpWrNgVbjSiBoa5vMWo7bedA3Z9OhoKMdBGHHm/R/J9ULGxKRattdjBveHjK8/BnAdgDcUOai1hDx0UzVlLB4/tdRD3/gBW57FVVsnAQB37noS/SB6yFQ9sdGNqptrLOTh2X9qEcDKJ4ec/1RGeG/asxhdP4skCMA8b5bA/b4vPIw/+J979O95eQRpJMbCG8r1lw5wNc/Fcy/dgGfGmlG/RSqJ4ko+ozsfehK/efMPsPvY/MBrFoGsqrMSUbdpUhnBrLnyybsP4q2f3IluEGkjIN+jKBG4WwV7Fp4I3OkM7j4blbZO5lw/Hh6QfBaum9BQQHF1m0JmTKxXzYKZX2Z8vYqZ38rMe1djcGsBoaEmY2PR8kMcX2hbr/HDCBUv2TU87cJpvO55F+N/fHM/DuYUHwTU7iiMGFO1ZFc7UXVzd2/yEInrvVIuVCJlsnJBQoNv9VxnZHxrEKlOa65DmZqFuUNc7ATWYjxsKKVkbU9W3aF2nHKNWsXBO179NHz2rS8EMMCzyKChpKzGKEutLwf9NAvxHLMW+ZMLHUSsapFpgbsrngW08GwK3EVoFjIn0v0s+nsWahzrrfWqjN18hoDiwoDDiA3NYp0ZCyL6IyOcFbH4/AfFDmvtsGB4FlfHHkOaJ+2kNAsA+J2ffQYqDuGDX3qkz7mVZmB5FpU+nkU84YQKi3hlnexkQTi91Om9hhEN5Y3Qs/ANeisrGsr01lrdsKdENjCYcpDFcrLmDUXRiaGSch+VODa+3wKUVQ5cFq6Hjq6RsdA9PFjfR9EsNk4oY5G16zy9qD7/ph8kmoVRq8uJjXvRnoVLFG9M0p5F/ucgY1pvuRgyZteoggAU6FkYxkI+41//9H34pb/5XiHXMzEMDfUzzHxWfomzrX+2uCGtLcSzmK57uGpbbCxSuoUf2JoFAGyfqePFO7bh4T7UhD53zaShvD7GQk040SyAlXkXrQE0lO1ZjC6Du2JUs03TUObC0PJDa1c5dAZ3IMbCHYqik2vXPPWwJZRB/v+mtRYg4c8fPLI2NJRV8iEe35KhWQA5xiLOqWh2Qz0n2l2bhvIcsv531MXqRLMw9bEkhDb/c9Ae6Tr3LGSDVBRdFkS9NNRcyx+qvty5Yhhj4UoYKwAQUQNArc/rxxqmZrF9uo5Gxe3JtfDDCFWv99ZtmarqJKdB5xZMVN3M2lBKJOs9x0p0i4SG6vUsrKS8kWoWUZK/4favOtvqhtbvoS4kuDLN4sCpJZxIUYfq2jEN5SV5JTLWNHYdncNSJ8gUuMWz2HNsfuRhkm0/xA8One37mqwy1UvdJIoPMBaSpo+Hn1RGTeZmqxsmobNGwIETC9wmRuVZ7DuxiDNLXZXAFodT6wzucPDnvR5oqIefnO9JvhUD5zhJNJQ6XpxnIUl58tmrnjTFB1oMYyw+BeArRPQfieg/APgygGEyuMcSekGve3AcwlVbJ3EgFpgFJsViYstUDWea3dwF19RDBBM5AnfeLmslEVESDZWlWZg5I2bU0rnCjwwainqrzoYpz8L8XXaag/IsZKyTVdei6N726e/j/V/sldW0Z1FR4yKy4/3N8/7CX30Hn73niRwaKgldffS4PTfOFZ/87kH8L//9O30rAmQVk1vqBGhUXNTj9ybj/ei39uP1//27iCLGqXiz0PJ7NYsorg3lpiJ4RqVZ/PLf3YO/+MqjsVGy9bFhaEcxamtJQ735o9/DX319n3Wsx7NwRLMozljUY89CV3A2MsiLxDAC9/sB/AFU7sMzAfx+fOy8xGInQNV1NFVx1bbJnlyLbiopT7BlsgrmpBtc1rkBRXEJGhU3s/lR3i5rJTsrMUZZXk+63McoPQvTCPXkWRjXaadTE1chAAAgAElEQVSMRRJSOUDgjl/XqKr7KYvPfNvPXGwTzcLVxyqu02OAu0GEbhBhvh0Y47aL68ma+tCR0eoW9x+aRRhxZk0yQVZ9oMVOiMmaq9+bHD/b6mKhE+DYfFvXwmqaArdhvM2qBID63EblWZxZUvlCKoHNiWmo+HNmO4Q2C1Jscy1DZxfbge5BI9DRUGnPoiAaSj6nqufo2mBBlFSOLhLDCNxXAfg6M/82M/9nAN8koiuLHthaYbFth7ZevXUSh2Zb6AYR9p1YwNs+/X2cWermeBaKAsirQJtHQ2V7FvZDsXFCRVCdk2aRJXBHZonyEfazCJPdjms0VUobC92+06ChhJIaNnR2MuWW+2GUydnLgi+7b0AMmX2dhBaJegogAsq4XbyhgamahwdHbCzkfPOt/GoAWZ5FsxtgsuZperQbxv24RV8xIrda3aA3KY8VlWIai+m6NxLPIooYzW6IThDqxc5MyhumNe56CJ0NmXHkbMs+lvIs+vWy7wQhfvPm+3sMznIgGmPNc/Rnb0Y0FolhzNE/ADA/oTA+dl5isRNgspbsPLfP1NVOr9XFd/efwb88eAz7Tiz2CNwAsGVSSTlZ2oCcG0hFQ+UI3OmHYuuUOvdKdnpy/iwjFoRG6OwIi6ApgVsqcZqaReI6A8nO1rzssLWCxCOYiI2v/O6HnGMsej2Lquf06ECm0dECt1kvyY9Qrzi4aENdUzujwNlmF4fOqMWoX5FD0+OSsTa7IRoVV89Lea/y/cEjiQ7SNDULs9wHIWUsKiPxLIRC6sSh414scPspraJvuQ+ZN2ukWTCrfuRHz9paWJDyLCp9QmcfO7GE235wFHfvH6p3XA8k8s2JjYUOFgnZ+tyKwjDGwmNmvcrEP1f7vH6ssdAOrDwIoYyWOqGOOAGQ6VlI6evTOSK30ABpzyJr95Z+cOTcy/UsoojR8kO4DmV22wvi6BRgtLX4zcxwScqTpDgZF5BE45gP17Chs36ocjnEU9A1pYIoMxAgLXADsHa4euxRYnRCw8hJOe+2H6JecZWhGWFo6a6jSXTVfJ/y6UHImgbrGkahFo9JfpdxA8APDc/CpKHafWioqdpoPAsJ6+0EEYJYSDcTNYdJwhQPSPSlU4udQmswpSFT8cxS12opkHgWduhs1nN0xohGGxanFjuJV2V4MVU3mXtBlK2hjhrDXOGkUcsJRPQ6AKeKG9LaYrHjW6GtsrAvtgMstgM4pBKfhBYysWVqsGdRcclarBQNFeiFSCAPhVxHPIvlPiCyI7toQx1Ar56iOqT15kOcK/zQ7O3tIGR7t5/2LMy3NQwtASjNouI6ejct5+/m0VCiWRg0VMWjnkVKCjYKRQaoxULG3A4MYzHCBcuktPp7FpzUB9KeT4i65+i51U15FraxCPRxU7MgQ+B2HcJE1R2JZyGLq9BQnhPnWaRCZvtpVCYNFUaMl3/w6/jsPU+c89iGhbl5M70LMQqyVsuinTV3Jc8pryVBGmeWunj1n34TH7xzrzUG13GsjYqpOxaJYTrl/RqATxPRh6F6VBwC8O8LHdUaYrET4ILpuv5dIpcWOr6q61Sv4I7feLF+WE1sbFTgUL5nsdgOMFXzrJox9YobVzWNdJQDkJSh3j5dx9mmv2JjIXrFZZsmcHhWdenbPqPeHzNb5T4qIxS4/cgufR6GbInEoktoGiojKW/QWPwwQs119G5a7k0QDU9DVZzerHW9iIVseTydQCVjtmMaKgh5pElrDx6Zw0zdw3w76O9ZxOGTzW6o32c7iLChUek1Fn4Siy84s5T83Dbuv+c4ugtNxSXUKk5m8MVyYXoWFHsVFWe5taGS0FkJPjgy28p9/ahhzsUjZ1t46gVTAJJ56xqUK5DtFUueU16zszQ+cKfqN3JsThknXYfKgWUs/HCdaBbM/BgzvxDAtQCuZeYXAVg/bcNGDFnQBaZnsRD/7cINdWzI8Cwch7B5spqZ/Aagpy4UAB0znX4oZcHaHnsEQkN1l1mmXFzeyzarZk5mRFSWOLdcY/TQkTl88ruP9xwPUkl5QWQvrNqz6CY7xvTfhhG4K55j7OZY013D0lCVjBInvqFZmAtYx6Bu6t7oaaiHjszhJ67eAqC/ZxFEqoETYBoF5VlUezyLZF5VYy/sjBHo0PLt3anJvde90XgWkkfU8aMk+c9dXtXZhIpJKMbFIVoCnCs++s39eOzkohWAcdQQuYPUM+Q5fTwLyaBPPevMjL/+xmPYdyJZVh86Moeb71UNt+bjuRCGiWGqea5VlHNdhM4acAG8noj+FcD3CxrPmiO9oItmsdgJFEVV7++MbZms9aWhJqvZxqKZck1lIX/RU7bgJddswzPj3hfL9iz8xLMAbJFbT/RzKPdxy32H8fv/sqfneJZmYYrEolkknkXyv4lmMYCGClR4blJiIRGk+3sWhmaRodPI737E1gIm/y+aRW3ExuLIbAtP2TYVd/HLXwjD0EjMGqhZJOPbMlVFo+pmehbpaKia56BWcUasWYTak7X7WQz+vNtGBre/Ssai7Yf4wzv24F9+eMz2LAyPRqgz0zuXcaZhJkWmj7/vCw/jnx84po/dvf80mIFrL5rRXqYYLN0PxCiX4q5C6GzflS/O1n4tVGvT66BKk/88VD/u8xKLnRzPohPEkVIDjEWfLO7FdtBjbCRHIN3TQh6IK7dM4td+6il4IM7qXa6xkF3MpbFnYUbv9OyK3OU3P+rEOQl+ql5WN4wwXfH0+YMoyiwWaAqsgmFLlEtfkYoOFzWMRZZnoTWLVJ5FRlIeoIRy0+NJjIVqoMTI9mBWgm4s/k5WXczUKwM8C6M+kKFZ1DzHyLNIqB/BlqkqsNi1dKt0NJRQpKP0LBLNIko0i2X2s2hpGir5jBeH7KG+UjR18AWnNAvDswh7nyH1P733LaGhbGMheVxWscrYwF6+eQKPxB6HmS1uJjUGa01DEdGnATwC4NUAPgzgSgCzzPx1Zl67YOcCoUo3R7axiBf3hVjgnhpoLGr5mkWn9/+lzkt6AsnklJ1KP+GsH+RB3T5Th+eQTUNptzY/dPbJuTae/rtfwK6conliAJY6AW657zBueN9XEUWK6zcLCUZsC3vy/uR9r6SQoB+yplYAtbjLbqtfnoVNQ1FPUl6yY4ssQ9IJkh1yPQ5THVVEjuw2G1UX03Wvb56FVfLB0CzqFYOGMmpayRzaMllDo+piNp4DrtG7QmgoxzAWyrMYhbFIDFcY74Kz+lkw5+tUZkSQBCBI0c9X/sk38PHvPD5wHAdOLeFp//UL2H9yuKx7eXailLE4fNb0LJJwVqB/IUERuNPPuhQqtXuqB6h5DjZOVLSXadLGZlLjesjgfhaAWagudw8zcwhgWdtOIrqRiPYS0T4ielfOa95ARLuJaBcRfSb1txkiOhKL64Xj+LwSki6cSQTumqcWhcVOgIUMzSGNLZPV3Nh7RXHZWoemodKaRWohr8Yl0bvL3PnLIjRZ9azYbCDRRSpG1FLEdkXYI2dbaPsRHj+V3U1OzrfYCfDo8QUcOdtCN+b6k92W+i67JSCZ+LpEdqZnMTjPouI6Rmw7D6ShHIK1C6u4To+BNKu6Wp6Fkcg2as1CDOlE1VP9wTsDPItKr2Yhc9U+HuGKLaog5pbJKiaqrt7MbJqo9ERDme1BTV78XLAkxsKXpDzx6JKdcfLesu9nUkjQ0CzaAfwwwr4Tizh4Or/boeCJM010gghPnBn8WsA2ULmeRco7TwoJZhiL2LNIU3v7tWeRHG92QkzW4rkgmoWR05FOalxTz4KZnwvV5GgGwL8S0bcATBPRhcOcmIhcADcB+BkocfxNRHRt6jU7ALwbwA3M/EwAv5U6ze8D+MaQ7+WccTjmIi/e2LCOT9U9LHUCLMVd7vphy2QVC+0g8yFbyPBMGjkCd+JZqI+o6qrXLbc2VNPYsZqionmNfpEc8j7yIjjkIW52Q73TEzpI6CHZrVrx6UbOgjkW8/piMG+9/4jeDZuQviJi7MyQWZMe8sMIn7r7IBbaAWqea0Wjea7TY4DNpLx8zcIZibG448FjODHf1vdmoupiul7pr1kYnoWUfBDPQu6FqVlIc64tU1VMVDwdGbVxoqrLaESsSoebAnet4gxNQx0608RX9mT3x252EhpKyn2YhQRNZyKPepQQcN+IUFvsBLqV8TDJpF2DRhwG2uuNIj1ft03X8ORcO4nYY3tTlzxDvddI8izsz1Zqz1ltcrsBJmJKsu2rea2fV+pNalwNzaLvFZj5YWZ+DzM/DcB/AvAJAPcQ0XeGOPcLAOxj5v1xIt/NAF6Xes1bANwUlz0HM5+QPxDRjwHYDuBLQ7+bc4TsGC7ZlDIWNU/nWQxDQwHZdZiyBHKJaknHXsuirh9eL1kQlwNNb1RcVFyyFsbeSI7eiW4ukFkQY7LYCTSH3I3poErqvEuGQRQKzHzf4tFIOGIQRTiz1MVv/X8/wOfvP9Jzba1ZGDSUrjdk7AbvOXAG//XWh3DbD45YORYA4hDOtGeRPIRWNFQQqhIgEcc01PDtXLPQCUK87TPfx833HrKM+kyjv7FQ0VAJDSW5B2IITeG9E4S4fPMEnnvZRvzYFZv15gRQnkUnRUMlnqzSLLpB1NMpMgufuvsgfuOz92f+zeT+u2GURENFHIdvJ3XE8qjHlpG8KTTUYifQhm+YmmlZEWL9kBiLZG5evLGBIGL9fIdas+iflNf2Qy3ID6NZNDthvHEQGtw3AlJiz8LIU8mqVTdqDG2OmHlnXBvqCihvYBAugcrJEByOj5m4BsA1RPRtIrqbiG4EACJyoNq4vnPY8Y0CUvdFEtgEkzW1G1vqhoNpqJz6UEHYq4cA+TRUoD2Lc9MsEnrD7clWlp/NGk7mtYHEc8hLJJIJLp4XkCxgZp4FkOwwgWRHJjtb87omly3XzSoMKKJ6kmfB1v2RxUEW3tmmb+kVQP/Q2W4YWYaz7Ue6WVC94qDikd7ZrwTtbgRmteiZn5PSLPrQUKHdAMccE6AW+k4QgVmFK09UXdz2thtw47MutPKDNk5UrWg0MxpKPAu5DwPfix/qcaRh7qRb3VDnWQAJxSPCfB71aJb7MGmoxFgM4VmEdtb6ILQMz0Lm5oaGopFlrqfLfeQJ3KaOmdbuHo8pNIuG8sOYklTXW2gHSbkPIlX2P25+FbFdpqUoLNt3YYVhqKGs0afNvwdgB4CXAngTgI/FXfl+HcAdzHwIfUBEbyWinUS08+TJk0MMqT+OzLawdapmJccBqlmRtFYd5FlIPkSaF5VdRTqaSq6VjoYKdEhe3NUtxUUPCzFCE1XPqtGkrmFP9Cy+VSawuaibSIyFQUMFUVyiPKVZdHu9CMuz0NVHE81Adr5ZO23VT9uxQmfN+2MW2ROYCXkytvQuMOkNHVk71k6QlMmoV1zU4pIL6ez7YSHvfbETGJ+TG/PU2Z6FJFKanoXcI3lvtTiz3A9VT5R0xQDBpomK1nmiOBoqEbgJ9fh877ntIfzO5x/s+166sXeTJVCbn/tSJ9SeBSCJj6wNXb7AHc+JyI54kwijYQoMDqKhPnn3QfwfH9+pf8+KhtoYGwt5nkNjtw9AG0GZN2eWunjln3wDdz2q1qetUzVrY3j0bCuhTi3PQtWom2kkxsJkAipx7lKaHSgSRRJdhwFcZvx+KYCjGa+5jZl9Zj4AYC+U8fhJAG8noscBfBDAvyei96UvwMwfYebrmfn6bdu2nfOAj861cMnGes/xqbqHJ+eGMxbPvHgDLt3UwJ/96yPWbuf4vBK9L5i2+0blehapkDwd8bNsgTvQ9ZPSu+h0XRs3i4Ya5Fn4STSUpqHCKC5Rbp9XDGLVS8qKtDLq7CTNjxIxMyuUVDwLbUiNsEoA6ISityRjT3sWVbe3ZIcWuAO2SlB0/CgxFp5reTQrQdu4d3IfGhUPM/UKumGUuQOWtbTmuXBI7ZY7ac/CddDxk1Bl00DaNFRVjyOMWIVkZngWt9x3GLfsPNw3r6FrlEhJw/QoW34Yh87GtGoQxQYt1uQGREOFKe/xyGyz7//ZY+xPqd69/zT+dc9xXRVWR0NxYiw2GIs3YISzUsqziMd46EwT+04s4iPf3A9AJcea+qSI226qHPxSN9TBDoBKzLME7lh/TIzV+qgNtVLcC2AHEV1FRFUAbwRwe+o1twJ4GQAQ0VYoWmo/M/+vzHw5M18J4LcBfIKZM6OpRokjs60evQJQBkJ2MINoqHrFxXtecy0eOb5ohfMdOasmdVo8b+SEzgbpHcsQLUCzINVIpdGPuYuWc6Wvke1ZJA/8iYW27r6mm+90A72YiGaR5nGXjN1zmCr3Yb5nXRsqSsp2zGcYC0nKM8t9mAt32rNwHerRLDI9i9DcxdoCd9uoL5UOU10uWoaxMD2LGSNcO43E4yQtsLfTnkVFtZlNN3uS8ws2amOhdAmzU141zrMAlIHqhhHuejS/JFy//Jb03FaLnbpOOxXOHOYYXl3uI0oZi5g6HsqzyCg3b0J6iHzlYSWd6r7fRkFJqdUmcz1ie1OXDhKRz+uxODz2sk0TVi24A3EY71O2TdqaRSxwm5qFaSwkqVEiytaFZ0FENSJ6MxH9DhG9R74G/R8zBwDeDuBOqPDbzzHzLiJ6r1GY8E4Ap4loN4CvAXgnM6+sfu85guNa9RdvyDAW9d4kvX541bXbcf0Vm3DLfYf1sSNx8bFLU8bIcQiNitsTIRFk7PqJVmAs/FAvEKYoBmSV++ilAtIF5wDgL77yKN7yiZ3W35c6iYDX0Ul6kg2sri8P40TFNQTuZDxCTZmaRVp3MJEWuIMczUJCdl997XZcunHCOkdW6KwYiJ5yHykaKh2mulwknkVoCdzCU2cZSPMzk8qjWZ5FN0g8DtObahgVBDbFC1/bDxHG0VBZnkXNczBd93KjnQA7kTGNbGMRZ5qnEiX9jCgis1pxkNoQJMbi3D2L2Tiz/asPH7fGHUasjcIGTUP51nU1levY3ma6PM9lmxu6FpyMv+Y5uGRjw6ahYs9iRs+FwKKNRX9M50oViWEKCd4GYA7AfQCWVbyfme8AcEfq2HuMnxnAO+KvvHP8PYC/X851V4LTS110gijTszDDZQeV+wBUFuwlmxpWL+Ujsy1UXMK2qd725ZM11+J1AUN8jieB8gyWX+W01Q019WCG2wG9uRxZ3ktiLJJjZ5s+5pp+/PcMGiqOSpJd1oVxwIBEfUzUPEPgNsQ+0SyMzmmyk8/WLCJUXccat3l/kl4PquXoX77p+ZouEFTc3vDXJDO2t5Bg0kDJRTXVlW650J5FN+hJyst7z+aCUfVci64SoyweR1rLABLPouKS1s8UDZUSuL3Es3jRU7Zgql7B1/aeUB5IxsLUz7NYSm2ETIG7x7PIoJPMxT29IZDSG8ujobI/L8ls/97+M1ho+1bCqNz3GW0s7JBvL6XPJbk6ybXqFUcXBG11VWJnsxtiqub15LQ0O4HO5gfUXDBpY/GI0yV7isQwxuJSZr6x8JGsMXTY7MZsGir5ubeAYBbS7TqPnm3hog2NzAdtsuZZvTKAXhoKUDtGf9mFBANNdXnGLvqP7tijH560tmB5FqlezYB62GShk53hmWbXyhoOo6Q2lNzT/SeX4JBaGOQa5kKQVW1WPK5MzSLgHs0iyKShlHeV9UBV3PxOeZJnYda20jRURtG+5ULunSVwV1wjAibDszC0LEmyTNNN1fTxDIG77rl6XrT8UAncjr1xkPO9/BnbMV3z8M8PHMUPj8zheZdt7BmXUDxZ87PVDTFZTTZEnuFZJJ6aHe33DzsP4ejZNn7zlTsMY+hYAjewXBoqjoYyFuXP338YB0838VuvvAazzS6ee9lGPHDoLL716ClNvZoCt/Ys2snfALsKAmAHSQDq2d0yWbNqwW1CYjRqlWTTEkWsGQFhNeZbCQ3lGEmNwXqioQB8h4ieXfhI1hhHchLyADuCaZBmIaikkr2OnG1lGiJAZVeb2c2AKXCnSlOsRLOoJjWaxJu45b7Duh9AMtF7BdusPIu2H+owVfn78bmkxr8sfLKIX7ihDoeAEwsd1DzXKliYVQLENhbq71klu5OkPMmzyKahmt0QEzW35/8BWDV29P+lMrgl4qqHhkq1MF0u5L03OyGavur97rkOZhpDeBauk69ZeDY9ZWoWYiDqVVdH4rX9qCcpr+Y5uO7yTfiVG67Ea597sTYQj53ILpUhG6M8z2LTZNIvzXGSRMrE+KqxyGf/ybsP4tYfHLHu03TdU56FYZCSaKiV0VBffOhJfOZ7T6jQXz/Ci5+6FQBw8HRTzz2z3Mdk1YPnkKahzEQ5ADoLXoyX0FC/+cod+I1XPLWnFlwrNgpVN0mAbAchmJUH7jqEybiwpOVZxEmNaRqsSAxjLP4NgPvish0/JKIHieiHRQ9stSE7lLSmAKQ0i+pwxqKaWtiPnm1lGiIgpqFSnoVE4ViehWcvbE+cbg4M22z7oa4/VTEyuH1jMUnKfWR4Fhnhp/KwNbuhXhwktBhIYtBN/lt6aNQqDhzTWHT7G4slw7NIv1fRLFRNI7vIHGBqFr3VfgUqQoy1ZmX23RZ+3DV28bK41StOTwtTwVzLz8w4T8OOhkroQq1ZZORapDUL02CbeRbdMKGh6hYN5enXNqrq9a04GorIrg01WfPwez/3TGxoVHDBjKJPzM+52Q1wckEx0/3KrDQ7ITYbxkIKCQK9fdGlF8nDxxZ6aKPpugr1zcptydI60kj3+FDjZpxa7OgSPRdtrKPiqq6SSSJgYixchzBV97RnkY4oBOw2wnJffvqZ2/GLP355T0BLy1efu5ktL3+T3vIqSdO3ssUlqTGdj1UkhjEWPwMVzvpqAD8H4DXx9/MKx+baaFRc7WaaMDWLyZwdahpmmKofRjg+387UQ9Q5vR5e1zfoBvOcMuH3nVjESz7wNdxz4EzfcQgFA9iT2NwBpst9+BZPL5M6OSY7PXMxk9BguaY6XzK9xFDKLr2fZ2HSQs1OUubB6ocRc/UNwxCm8yxkUZHdWxaEOz/b9PHyD34d//zDo8bnlhRErFVc27Pw3J5GQ4J3/9MP8ct/f2/m9UyYmsVSJxljP81CxtYvGioJnc2PhmpUXP36VjdM+kykcm6S//MwXfNwwvic/+RLj+DNH71b3YMMjl6w1A105JUae6IzyYKcJOUxHjm+YEVzyfsTOrjd7fXkhvIsQvt8gPrsIk6ilTZNVLGhUcFcy88UuF2HMFXzdE6RNiLGYl1xkurNQhPJ/Uz3rxEayqwGIHNeDLvk3Zjl0GXeyvO55uU+AICZDwLYCGUgfg7AxvjYeYXZpS42T1atukEC8SwalWzeOwumF/DkXBsRIzOHA8jWLLLip82cgENx0t+T83YD+TTMHavnOFakjyDduMXWLLJpKEAtsAKzeKIYvqrxACXGwoVDtmehE7LiBzLK8CwAOzpo38lF+CHjaRdOx9dKEtEEpmcxkeNZyP09vdRBJ4hwcqGjjaUfJQURa55agNuWwJ1tLE4tdPHAobM4Nte/k5ssGBErcVU+p6mqB6IczcL0LMSDSHkWPaGzGXkW9Yqrf+4EcTSUk4TOZvV03jZT054EAN150bwHaWMhQQqbjWZhnkM6+qmpjUWS+fxQ3F42KbNuG4usnJ9h6FmT5hHIuPfGoeAbJyqYaVQw3/KtIouJNpGU/1Hj7d3UuW5SvVkoM5lnDUOzAKA3PMqzSDYPAIzNQwULHd/KvZLzyRyqrAcaioh+E8CnAVwQf32KiP6voge22jjT7GLTZLZ4LZN0WL0C6KU3AOCSVNimYLLq9mgWWfHTprdyKqfrVhqmZ6HyLKJ4p5S8Jh3JkRkNlRK4AeBsK6FaTIZIdkamobvE9CxcsvIs5P5GGZ6FaUTNnfaDcU/pZ16smkJVPCc3dNa8B2nIDlc0kY6f0FDMSSvVekX1duiYNFROnoU89F99+AT6wfSUTi129BidePeapdOY0S86dDbDs+gGZlJehsBdMQTubogoUollSRJo7+KzfbquKzMDim7rpIxE2nDKgmtqFq5DqMdjknBr8X6CkPFQXA5f7qtUD5DnL21gzOv3Q1Y0lFxj75NKizE9C7MicqSNhQojTjK4k92+wNyUyfllnvV4FmIsPPU/UcRJUcla4lnMtwLt3TjUq/msF83iPwL4ibig4HsAvBCqAOB5hdmmrzNa0xBaYFDFWRNmdq9EWl28HM8ii4byEoE6qWA5yFgku2oRc9MPVrqQoK1ZJJNaIJ7FbDO7fpHsjMyxi1dVqzhwKKHDTGORTspT50qua9JeDx2Zw2TVxdVxRdWq68TCe4Zm0c1vWiU7aDFEkiMikFpGSrMIbc8iJ89CFtCv7ulvLEwDfGqhg4lKMsYrtkxk9hAxPYuJeJOR9iwmqi6WukGSw2B6FpXEWCQCd1Y0VO/ScMFMDScMz2Ku5RseRS+9CSTZ25snUsYivrbM+7ohcD94xE74lPkmz6GIw+bzOqj3Sdb5zGN7j8/rc2oayk+ytLVnQcqQ99SGInNTlwjc8l3minzG8tzKRsbcePRoFnEzLLuQoB16vNb9LAQEwFyRQmTXfRprnG12LV7VhITLLs+zSHbp/SKt1PmVZmEKuOmQPHXOxLOQwmTNPiUYALUYy4NZiaOh0sYi0SyyakMlvL95TgCYi+PS0zqPeBZVY+cneo1EQ0mESTeI9H3NErjNBdXyLI7M4ZkXb9C0SaPqoh3YNJSvd6b9PAsxFkneiHmOlh/Cc3sF7n6hs3LP7tp3qqf0vAlz0Tq52LFKcbz8aRfgvoOzPUK5mcEti1qabtqQolGsaCitWThG6Gykk/K0wO1lGIvpGo7Pt/U8nWupBSwysqrFm5bfZeHbmBK408ZCxtj2Q+w5Ng+HFD0XhAn1J5u1ZleVDJGoMWDlmoWM89HjyrPYOFHp0SyiyO4lMZnSLIhghcTbAnc2DSUGr+2HqFcT/ajjR37HnekAACAASURBVJpl6NUsbIEbSPQbbz1oFgD+DsD3iOi/EdF/A3A3gL8pdFRrgNmlrs5oTUNE7WGytwVmldhj821smaz2FCgUTFQ9RGy7x6ruP1kaiilwCw2VTuYzIdE8tsCdLKhP2674/kmtaeRXnZV6+szco1lsmbSN7KL2LLIFbjcugibnkUiliDM8iwwaKggj7D42j2fFfcnlvD2eRSiaRb6xkB3ZYh/PouKodqUdP9LtSyVJ0rxOcs9CXDhTRyeIcN/B2czrArYB9kO2KsK+4hnbETHwjUfsApmmZyHcetsP4VCyQdnQqCDixPu0aSiJhkoE+pavQjWJ8gVuQHVb7ASRpsfE07Pa2QYR/mHnYbzofV9FEEbay+z1LISGsiO2Hjm+iG4Q4WkXzuhzy3OhaSg/RMV19POYjhLMQz8aSqrz1uMgFzsaKrIE7ulUNFQ6x6HiJOPJo6GahsDdMHunh6HV2wSA7m+iw2SNz0k2Cusiz4KZ/wTArwA4A9U571eY+c+KHthqIgjVA5DnWchiNqj/tgmzSmyrG/b936nYGJmF2iQZzITZxvP0YnYjFRNiFGQsKnQ22fW9+Scux2ff8kJcEIe1prNPAbtscstXobKylp+NFwsJi3RIfYm3Y7rGl6SMheJmbeEyXU8HsAVu2f0/dnIJbT/Csy6Z0X9rVN1MGkrKnOcJ3NU0DeXbfbe1ZxELkG3DU8uLhuoEES7frPSprJIdgnQmsWnQnn3JBmydquk6RQLT49zQqGAhTugzmzqJp3ciDnPNyuBuVFw4Mb0mn5cpcGdpFtviIpgnF1TzH9ldd/zIErifONPEyYUOnpxv6894puFBpnMWDSWehYxZQti7QZIAKh5+q6taxUrXyW1TtWXRUJ0MgRtIaK0NjQrm274eW8joFbgNzyL9nJr1xuS7lAGpG8I+M1uahdxLXUOtJqGzXkxPJZ9TOqlxTTULIpqJv28G8DiATwH4JICD8bHzBlITP8+zEMFxWZqFsevsBKFFyaQhhsTcRQch9+zuzHIdef18TSRhe8lu0QwvbVRd/ORTtujXZ3oWgb3LNhc48SxEvJyqeah6jp7sZuz3dL2C6boqa+A6SuDWIZF1W+A2GYWlTqjHJQvvg3G0zLMNz6LuKWPRDSOIM9YxFpq8kGcxaHk0VDuukmrSUGY+A5AlcEeaX+/XaKfthzCD70waynEIL3/6Nnx974ncSsFiFE4udPSYgKQkhYS5mnNPeUXJolWvuNog9wudBaBzZY7Pdyz9qBMm98wM5T0y20r497ikhVxHPAnxQmU8UnJDSv2b+SJTWrNQz5M8j1umqkP2s+j1LMz/kyKBGxoVMCdBD2EUWQL3VK2CZjfUUVJpCsisweaHkWWERf8yN16NqmEsgkhTVJOahlLjkufNcxOBW+b3WledlX7Y9wHYaXzJ7+cNRKjNE7gB4KVP24YXXDW8jZTOdn6oePl0aWwTsus1PYusHUtlmZ5FOrvTi+kfOUfV7Z3kQHanPCBucGNQJ3NxNJTQUNP1Cqquo8eUXnBufOaFuP7KTXAdx2psJMYy0SwiPbZWN8TGiWocSqrOK2HDV2+b0uduVF20/BB+wDoJsRtEhkvfX+Cez6Ghmt0QnhvTUIHiz2Vhyxe4Q20s+pUCafuhRc+kqbIfv3IzFtqBbvcL2HkWYiyOz7ct70EbkcUOKi5Z84iI8PKnXYDrrtikryn31aw6mylwx57FiYW21YyqGyQ1ucyQ3aNzLe21TMSJZ4CEzqqftWcRPx9SzG/LZE2fWwvcRuis5zh6A7BlsrriDO5Ohmcxk9LgzBpMLpE2WoudIOc5TTZ1ZkFNwURVFQ5tx1FepmfRDRLNQmhJed/iySsaSjyL1aOhcrfKzPya+PtVhY9ijSHVUDfmeBYA8OE3X7esc5qaRSeI+noWUzU7QkL+Lz3JJK6embXAnQ65NZEuMuZpGsqmpwS6n4UpcPtJXR/J9BWIkd1seBYdY7Knd1wfeP1zAQC/dfP9ylikaKikgGDSwGepG6iqp7WkIZD0wDYf0nrFQdtXC73kGXTDyEhw6h86a0ZDmYEGtmcR01BGwT7ANghSIVV2g/2MRcsPsWWqqj/LRsqgTRs7afP8gNpd5nkWGwzPIt3sCQD+5pd/XP9sGgvXSaJ6MgVuw7MwjUXbmBd+aHsWF8ZVnCernl4Qnbj1K2BEQ6U8C+k42Y29OSK7X33FI01LbZmqDdeDWyKUItadHM3Px/QsTESchM46TrJ4L3YCBFGUQUOZnkUvQzBR9dCMnydAPIsk50XquYnhls2U3HOrxPt6oKEERPSVYY6NM9KL3ihgahbdIOrZxZuQHdLSAM9C4ucXO4HVq2G+7ePlH/q6VeUWSDyEhFogqxBb2hjJBDxwagkvef/XcORsC50g0lpOsxtaCU1iZOW+TdbUDkneR14JAin3oT2Laq9nYe48VYnsiqahzAgvQb3iotUNtZGVe7U0pGdh0lAmDdeKxdRaxdGFBHtoqCDC7976EN7/xYf15yKROnm9EwD1oMsOWo3Rfk+6jpBva1mAWhxmcjwLufbJhU5fjxZQhlreu51n0ft/UzUPk1UXJ1LGYrFjawBJ+e02FuNzTxgLoueS7hUu/yvjFFFeetlLVV3pIQ8ATT9AxVU0HJHyLCQK61c/uROf/O7jAID//LkH8Fdf32eNTSAtYP0w0sbB1CxMBBHrjYznOIln0Q4QRr0LtVmDTRp0mWhU1VwVr9fSLAKlWZi0qfws1J9nZMCLscjyBEeNXM+CiOoAJgBsJaJNSMJlZwBcXPjIVhGym+lHQy0XZkMek8POwmStl4YymwcJhIYy+3s3uyGeON3E/pNL+P7BWasiaLrbnuc4caJZPMFSC4lc74HDZ/HEmSYeeXIBnSDC9pk6jpxtoRWLcoK5lMA9Va9gtunr43k8qpT7kIdXHgaz3IcsLEvdENtnVJiq7IAlgsREIy7H0Q0j9TDFpTASzjxHs3B6BW5TmG/5kmcR01B+qLOPVbSaWtDu3n8aW6aq+t6KZ9HPWLT8CJsna1oPSRuLrC6KoSGYTsa1nZaMLHggWey6YX/6U90XT2dlW82PvGxDf8FMHcdTNNSSNW8Nz+JsCw4pD2nzZFWPxTXEXlkwa4ZnUa84OkJPBO56xdWfVasbYnLKwxtfcBmu2T6FfXFxwyBi3HPgjC7rfd/BM1aAgWUs/BB1z0HESkyfa/las0wbCzOD23Hs5zWMoxZNVAwqNo+GavmJZ5GuBtDq2gEZ4nnLe3EcGDTU+vAsfhVKn3h6/F2+bgNwU+EjW0UMQ0MtF2YbVKVZ5NeUmtQ0lOlZRD2JNpKUJ+L2TF25s7LjMIu8Ab1lzuW7LD69moX6u7SQnW/76AShvi9SmVNwNuWRTdc8VF1H7+bzPAsJnZWHV7yEyCj3IZ6F6D0SJgqoXWGeZxGEjKqXZDebPcizoGmojngWdq4Gc1yuWyflJdcmSjyYhXaAllGPSR7wQZpFo+rqzz/LAAK2sUhHQwnM+TUVVysFkkU4DxNVTy9CgwRuQOkWJ3s8i2Tedo0aXkdmm3joyByedfEG5UkYmgWgqMO0ZiEUnumZt/0I9TjzX+6HFKf8mWdfZOUHmZqTH7KlT3TDSN9TqZwMJJFXGzM8i8mqa2Vwe04SsqtoqF4GQErayxjSm75G3MeinUlDRXF5mt7ij/OtJCQ9oaHWQegsM/95rFf8NjNfzcxXxV/PZeYPD3NyIroxrla7j4gy26IS0RuIaDcR7SKiz8THriCi+4joB/HxX1vRuxsSs00fXhzxNCqYmsWgHd6UFrgNzSIrfjvukSFlmS/fMoGlTqAf3JPzdm+qpBSBhM7axiK9IMgElCiauZYPP2RtLFopgVsWrs2paChxPvIWHNchREbiliy+SfE1tha/qudgpu5ZnkXaWDQqKinPbLWqNAs7Zj2N3gzuUJcl1/fFcZLeEfGO1BxbJ4iw0PbRMrKmhTbp16xKdrfi9aQNmsnRC7I0C8BOvCMi3Zp1MA1lCNwDakMBqtz80blWrmdhRUOdbWHPkwt49qUb4rHE5UxIjIWblPswabS6Z+20JahAxqSoQTtjGlAlcrqGsVeRVHYotVB0bSPc92nbpzFd8/D0i1TekXlfp+sVuzYUkWYJFuNEud7n1Kah0nql0FBSxsTM4FaahZ0XNJWhWZj3AlidDO6BqyMz/yURPQvAtQDqxvFP9Ps/InKhPJBXATgM4F4iup2Zdxuv2QHg3QBuYOZZIrog/tMxAC9i5g4RTQF4KP7fo8t8f0NBsrezigiuFDKBu3Goaj+BeyJLs8jYkUixPOF1L9s0gcdPNfUkSnsWWshOFQrUxcfSmoVQMvE4TsX0hOy4Wt0wM3U/oaE8633m7XZcUuUQZCFtpDyLMGJrkavGmsVCZwGAeqAaqV7a9YqKsJKErarHsWZh6yJpeBkCt+s4mKi6ScKTqwrfMavXmYZKkgElAEDXY6qoCKpBAnej6uqxDUNDmRncUiZdbUbs/93QUJTgMDSUnN8lSgTuHGNx5ZZJ3P7AUaug4GKKhkqqxarvkjwpY9GehedaGfGC6XrFSFKLDXTF1f+nvL3eeSYdGs0KuOmigTP1Co7Pd9D2Q3TiPiTbZup48P/5af26iaqrvYOpuoe5lm8J3Iln4Wd6FmborKpa3EtDHeoGicCdjobqBtbGVWsWGR6gGOZ1kcFNRL8H4C/jr5cBeD+A1/b9J4UXANjHzPuZuQvgZgCvS73mLQBuYuZZAGDmE/H3LjPLbKwNM85zweySn5tjsVIkDXniPIs+AlQlbmRj5Vlk0VBas1C35rLNqvm7GIsT8x10gwhv+cROPHh4zioPoP5/OM9CcDL2YDbGO62mwbOaC1sicHvW+8z3LBRXnOR7qNfJJjyM2OLga56ri6kB2Z6F/D7f8tX9jJvJSMx6XvOjJCkvpqHipDxzly/RUABwbK5lja3qOjqaqdW1K71WvaSSaBYkwU9oqPR7SuoI2YEPMiaiROSup4yn7I770Z+AXZXArA2VN1+v3jYJ5iTXBeg1Fu1Y5xFIPoy8PynnbY7Z/Hm67llhya04+q2SM7fSFVhlXpleThR7B2b+i2ymaqn3SpR4bdN1T/8vYAvcC+1Al3Y3YSbldYMsGspTnoWhWYhn2NGaRS8NlYQ4J+9fhPq11iwE/w7AKwA8ycy/AuC5UAv4IFwC4JDx++H4mIlrAFxDRN8moruJSLdvJaLL4iZLhwD8v1leBRG9lYh2EtHOkydPpv88NGab3ZGK24BdSLAbRBZNkAWpDyUIcmioiFXHuemah40TqqyD9JI4Pt/GYycX8eXdx7Hz4Bkj6ikJnQWQmweRvp6UFNGahZGUJ/er5jmYqHr4nZ99Ol73vIstzyLfWMAqOyKhqLIbU8bCoKHi0g4y7nbQK3DL6xfaPqquY4TeDvIsks8JkEUkshLkPNfBK56xHf/2+ZfgNc+5GK+//rJkbJ6jjXfb9CwM3SQLqmxKZBmL3mioXhoqXTNsQ0PoJvt/xYgMmnemUSQivOipW/C2lz1FUzJpXL1V5bY8dGROjyFNQ3WNDPapmocr4p+1wE29eornJKHQM42KtdOW++RlUE/mz2K0zD4yMl/F25D7YtJQWV5/YixUwyVd8dVJ5lKiWWQEohhJeelAkg2NCs62fF3XqVG1i1Iude1mXa5Der678SZBaxY5LEERGIakbzFzRERBnNV9AsDVQ/xf1ujTmTMeVGOllwK4FMC3iOhZzHyWmQ8BeA4RXQzgViK6hZmPWydj/giAjwDA9ddfPzgrJwdnmz6u2JJdPnylSOoGhQNDZwHplmcsCiH3RBNJot+xuTa2TFX1hJK+CfPtAHuOzev/T3sWYgzyaCiK22rK/wnVIA9Oyw81p71psoIjZ1v6oX7rS54CwH7w8nhU13EQRUm/gnrVpqGCDBpqsuZpw5unWcg92D5TB8NBNwjR7ARxxnL2/U/fAxG4Zxqu9Zqrtk7iT37xeT3/X/UcrSG1jAAAiXDJMxZmpdgprVnY76kaN4pq+va8ABLaYUOOUUg8i8HzTuASYaZewTt/+um5r79y64Qe//aZGo7Pd3SdJCAJdX36RTM4cGoJz7x4Rs+ZHhrK+AxdJ5l7PZqFH2JDo5JqMWxrSgCsCCRVzDARuOV+S6RU2w97etCbEKMyXfOstqpi1Carbq5mIS1PAbUBSpdO2TJVRbMb4kwcWGPVhgpCNDuhtVkBlNfeMjw2HQ2lmx+tD89iJxFtBPBRqGio7wO4Z4j/OwzgMuP3SwGkvYPDAG5jZp+ZDwDYC2U8NGKPYheAFw9xzRWhCM9Ci24BD0zKA9RuZTFFQ2XlWQAqg3nbdNL8/ZjR//p7+1XnPN/YucuiLQ+FLD5ZD4l5TfEsJmseKi5ZArf2LCq9C5y+Bzk8atqzkIVeaKgoJXDXPMeIDAr0TtOE6VkIrSeaxUTFzdWj0vegE0QIosjyXPrxwaZnEUZsCLaJKJ4FMdiNSqJZpBcIOWYL3DbtIEahnqFZqHEsh4YavOBM1yu6RpR8X0iFfHeCCBdtqGPrVA0/YVQ9MMt9qDHbGwvR1kzNIivPArDDvmV+y2araySeynxN579IAicwyLPwrPalcouku2WmZuEaTcYyaCgpZXJ4VlUiaBjl7ju+yqNKB9uIUU+3E1hXmgUz/3q80/9rKLH6f4/pqEG4F8AOIrqKiKoA3gjg9tRrboXSQUBEW6Foqf1EdCkRNeLjmwDcAGVIRg5mxmyzi405jY9WiqQ/s0ryGmgsUj0tVG2o3gxuAHjk+AKefuGMphDMjmx3Hzit/z/dH7iS4nazxlTJMBY1z9WhqVLPKG/nWnN7H+I0tGaRioYKtShoaxZVI2JIQg7TnoLoHm0/0pqF9AaY6BPlljYWklluLtz9FlEVKtybqFirOH09C9kRNiwaqnecUhpCkO65fO6eRXJNZ8jd6VVxD5ELplW8i01DJcUW7/ytF+NtL3+q/psOnXXzPQsgCcFW51NUUq3i9HSNFOhNkHgWRviscPqahjI8i85QNJSnQ2cdgt50yPOQnWeRdMrrZtBQkogpZVyUHqNydmabquR8OkFYNhRC4fWEzq4lDUVEufUtiOg6Zv5+vxMzc0BEbwdwJwAXwN8y8y4iei+Ancx8e/y3VxPRbqg+Ge9k5tNE9CoAHyIihqKzPsjMDy773Q2Bpa4SukbvWagJkmSo9t/hTdY8KxwxiBgTGVwooOr8P/uSDVq0PbHQ0ZTAwdNqtxKEUdJtryfPIluzAOyF0YxUmYh3uBVX8afi1aQXo6E0C7J3Rbqtqgjc3Bs6KwtpM44i6dEsrNcTwkjyLAKd4JWFrIdsKSUw9uOD0+9Rck9E4M4LnRWDrTSLbBpKHfOs/udpanGgZzFAs0jTUMPg6q2TuOfAGWyLs6xN+tQPGe04N0aysAVmuQ/ApgZVVrL6faZR6aGhzGgowP5M5Pii7tdul0yPjJweoZc6QWTQUL3ve0PD0303Qu4tGCjlZYKQe4ysl+rBnUVDAcpYNAyvt+Y5ulFa+t6Jp+GmWAKdlDfCSM489NMsPhR/rwO4HsADUAv3cwB8D8C/GXRyZr4DwB2pY+8xfmYA74i/zNd8Ob5O4ej4Ia67fKMW4UYF2U3IrmswDeXqiQLEIXcZArfgWZds0JQHM7DjgmktdAPKFU932+uNhuqdYHINqQcFJDRQy1c1ecyWnGkjaBuL7AksC3QrbmKje3/3CZ2VhXSxY5cJF9Stxd2B6yRF2dI1l6zxZhi0MGI0jK51/Sp6pj9XKfY2iIYytY1XPGM7FtpBphfQqLg6oguAFZUDDONZDNikWEJq35dqiGexdVotekJDeQ6h2Q16AhQEutxHqly3urbhWZiaRUxD1b20sejdlCwZArdppE3DYHoWYlCy7vvPPusiTFQ93S8+Yoa5d6tXXLQD1WEwuzp0LHBn0FDiWRyZbVkebNV1NEuQ7hEzkUNDteJeJsN6heeC3OnBzC9j5pcBOAjgOma+npl/DMDzAezL+79xw5apGv7p12/Azzz7opGet6o9iyGNRQYNlVXNUs61Y/uUtRO9YsuEtTgHYWTUhnKs762c0Fkg2bFesWVSH5MWnCLgNiquXpzTi5S8b4nayILsLNtx6Xa5ZhgLk6GRwQ2oRUY8i7mWr8s6mzA9DU1DDeNZGPfYvN2WZ9HnQUwvNIln4aDqufmaheFVXXf5Jrz3dc/KvF+KhupNyjMjhwCb/wdWRkMNm2ckxmLTRBWeQ3remkUJs66blPvopaE8I9FMqhcDkpQXoVG1aSg7dDbWLLTAnWgWgG0YtGYR9Be4X/TUrfidn32GlcNheRZxSfy8DO6kU14GDRV7FmkPuVZxcTTWH+U1AvmcHE1DJRrHapQnB4YTuJ9uUkDM/BCA3rCQEhbS4XzpWO40ptLGIurdscgD9IwLp1FxHWtB2zhR0Ryy/P+gch9ZD4kcu2pbYixqFUc3F2r5ofq90p+G6ld+IInKimIvIDYWDN1YqeraArfQJWeWOvqYiXraWHgOuqESnPtpFm5c3wmwS1NPWJrF8J6FlG2vxaJlbjSUkZDVD42UsUjX+0o8izyBe3hjMSyVcU3cYfGCmbqVH2Q2Bcr0LCopY2GMLe1ZSCkV7al4KYHb+jnWLDpJ5JNdNDDRJ6ZqHogGC9x6XPF1OkFkbSZqMQ2VGQ3lOrqrpJ/BEKiOfOJdmZsiR0cg9tBQ8WYpzRKoWmjFexXAcMZiDxF9jIheSkQ/RUQfBbCn6IGNO1TbTerpBJaHiZj2YYOKyepnASQZseaDvqGRRKkAakeTXljMEgEOZQu3cuwqw7OoeW5Mh4ToxLuhRLPIpqH6hQqLy9z2bc9CJT8lOov2pNxEs5Aiiv09C5VE1w1CPHGmics2Zfc+B+LPKTYGGw1jYedZ9Be4TZiehYwhC2ZCVj/IfReEUWT1fNaaRWp+zeQYkTQszWLIRefKrZP/f3vnHi1JXR/4z7equvu+5s4M82CGGQYGmRGRIIzDRAWMRFBElIg5ArpHRQ9kk3XXx4knKhvimuyD9Wx29cgmh7hEk2g0nvXBZo2P9XiMq4thMKCCIk8VB3kMr3nee7v7u39U/ap+VV3VXd3Tr9v39znnntu3bnX1r6t+Vd/f983nf+8lvOqMTVQDLxYQM1YZ+XzNIh0NVctoFuY8G1NRNfDiREy7kCBkzVDphZnts4DWnIpa4LEQNR+C9nPVCNDFejo6caoSaRaNnDyLaL96U1mqty76RCQ2RdlBDbbQypqhZjM+C/tcjJOwuJowdPWdwLuAu6Ntjg5U/ORG6pxnEUZdmBXQUs6KwaizJiPWXv2unq6wcVWNwBPWz1VTDVsSM1TiKyhyPge+4ElSXA0Sn4WpwZ/2WeQ74ds9YG17a9X34huy3kznhphx2z4Lky2ddejaD0ujWRxYqPP04aXYbFI4HhNZZAU5ZIVPEdlV6VOHl/AkyfoudHBbReTaMVP1OZwpUW7PiyLfRFnNYq6HaCiAXdvWxuY+M8dnrXLn+T6L9DzM+iwCS7OASFjEx/NytYnweJloqEZGs7AEQy3w4gf9QhszlD0uCFfwfsrBHZoYizQLIG5hXMmp4GvCZ1NmqOgazka9wG2MUE/KsVgO/iGZocrUhjoK/Nfox9EFKWHR4aY1K8FnjiyFERhNbXng7jx+jjO2zHP+zg1AelWyerrCBadtZK4W8L0Hn2SpmfSSbs2zqBcKr8ATjputZSqaeqyeqfCjfc8wXQ0FRdyHOjOp4wdCCc3CRFfFmoXa4b7Jw6MWeLEj9skCzSJrhgo8iQsaPsfqqJdHeF4aqe8c+F5cEK5TngWE4Z4HFuo8c3gR0w+7GnipQnY2sYO7gwN6uhq0ZHDbK9ydx6/izK2rOX3zfOp9x89P8evbj+MFVsn63ONXfETCIIleImpqlaRw5FzNT0po5Mz3M7euYde2NXF7ViPgTUiqOc9Gs6j4EkcITkVRQyZxLy/50wRktGoWiX+i6vtxTap2Du742LHPopEKAJiKaoLZ5jODXdhwqdHMnT8mNNYOzDDfKWuCAkuziD7LPhfDSMiD9j24/y76/UMR+UH2ZyijW+ZUfK90NJRRO42ZZSlHvd28epq//9fns2VNuOo3/XwhFDZX7dnGn15xFtXAy2gWaZ9F2GmsSFh4rJ+rxv0YIBQIp6yf5dFnF3jyUNhvYLqar1mUMUOZh9KRjBmqYWkWnlXSoBoknxdrFhmzi+ktHX62pM53J83C3Ny2GarqJw+v9maocFwb58Mb/OkjS7HJsW3orDFDVTubJ7M+C/vhs3a2yi3vOI+TM9+xGnh89ndezDknt28FLCKxIO4lr8u+zvbiJU+zOP2EeT7/e+fG13IqEx1lzvNcrmaRjgbK91kkDu60ZpE2Q01VvJSDu63PwnJw28LUaCd5D+u4l33kaM87vhEI0xmfRfi/1jD+rLCwP6ddAEY/aadZvDP6fekwBjKJVH2JY9A7hTAaYWEqyjZyqlXmMVsLWKgvplfFXpgUlGgW6Wiow4uNwmZMU5WwtIaJGgnH7rE9qgn00P6whEOhg7sLM9TRqEKsmetZp7wZdzXw4h4VppdH9mEkIkwFftzZzgjDwJOUSS0P87Cx+5kYzeLIUucMbggT1O5//FDq3GYd3B//9gN89/793PzWc2IHdyefhWmUo6qISG6fk2NlthaWCvd60CyqmT4ahk7mL7AKC1rhoHYvjqqf9llAeK0W6s2MGSqtWQCpEilH6w0WG0kiqnnQt4uGMpgF22KjGfsLwvFEeRY5SXmJGaqZ2/wIEoGQNkNFwiKnY6eJ6PMzfpuFenpcg6RdD+5Hot8/G8pIJpBK4PFU9PDvqFlEKw3zMMwLnc1jpurz5CFaTChLdpP5zGrsyFKjsIXsH73m+fENBcT2d7M6Vw1v3OkO+h7sEQAAIABJREFUDu52TjfbwW1Kw3sSOrib1pjNisl8xnTVj4VpXhTRVMWLckG8WGhtWzfT0aYb+yysc1jxvVL+l1hYzCemAzPeWiUtLH74y2e4M2p9a5f7aMd0NSyNfnQpLG6YV2DyWAlXrQs9mTPseW370Do51iHRDu1OjvYiphr47D90JL1vHNnX6uy2owkPW4mCC0tpLaJW8ctHQ0X/ytUs6o22Ie6LjWZLMp9hfeTgtnOAEmFRbIayr318LoZQ6gPaZ3AfoLXwH4SJeaqq8zn/c1h04+A2Di9TlC4vdDYPc4OmH3QS5VmkS0OYG0y1eDzGxm2X+hCRuIAchA+4WLPI5llEE77d2G0H98ZYuHhxpixEvRUsMxSEqyvjs8hbkU9XfJ5iKaVZnNLBBGWPNXsOzfZ2Gp65wTfM1Vq2VX0/dtr7nnB4MXG0Hq03Uk1sipixamJNR13b+m2jjs1Qvfgsou8qkr9Kbke2ZHngS0ZYeHF3xOlK2mRlZ0VnQ8KBVAVn2wxlIuWMZiHSfmETaxY50VChEG+0LCayLU/zzVDFDu48M5TR2ryUGarzwqyftNMs8msUO0pjSopDCQf3VIXAk7goXV4hwTxmqqHabpsATMP4ln4WOauSIsxNa4TBTDXghNVT7HvmaHufhd9ZWNgObrO/56V9FqFmkfZ/zNSCOGkpX7MwpgqJ81o6+SsgKXiY1SzKrNyqlqAxGdu2zwLCB43JU4lrFkWtQjth/ADGx1G0Uj0W4kibHh46NWtxYM/xTua1cJ/0w27Hxjk2zSe5QjXr/rGvrfk8gzkfB1OahS0sksinmu9H2vgii41wQdYuGdGMbSEjLMz3PrTQaBGyiW+wuJyI0eynLZ9V4rNo1SzMojCv5MmwHNyl+4hGXezsTnk/H8iIJgj75um00vI8Ye1sNTaz1JtaynE1W/OZj5KYDKEZKlGzzcOwqHlMHrXAj3MFDNs3zLLvmaNpzaLIDFXKZ5HYc/2orEIqdDajWdhmjryHkW3XNu8xvpZ2mNDGNVbobOBJ4n9pcx3M58xPV+LueuacZIXF4cXEHNKpe6Ih29NiEJpFXHeoF5+Fnwh0e06V0SyyeRf/7rIz0sdOCR8vtW9enoUtIGz/hR06a7ou/mz/4bDPTIf7wCxsFhvpNgPGzJaXFJctbJhrhood3DnRUDkm4rkcB3fs6xmSz6LjFRWR14rIvcCDwLeAh4B/GPC4JgJbVS5z86ybrfLEwcUo+7N95rBhphqkVsRgql6GD147gSvdPKbzsVdNVVLCwKzSaymfRYEZqs3YzUps0epPbMIAbT+LuclqOcKiyGdhvpt5j20+KyJbZ8kcI88+niUOnZ0KWpz+5rdpiHR4sUFTwwd+Xm/mPLKtVQfns+gxGirWLOQYNIv8D04vtpKFgP0bkutjCwhbyzC92c0xV02F+SCLjeKowPjYVjSUff+ks8/TxzAP8bgVQBszVLZFr/0/m9kcgR771Ebts7D4Y+BFwP9R1bNF5ALgqsEOazKwJ3SZB8P6uRr7Dy2kspg7cc35p8SmK0NY9bLOUkNTD+3UeEoIi/mpILWSMav06YrPhrkaH7jkNC7J1NSqdaFZ2GPyPaGpSUeywEse1uaYdtG7vIx4I8AqvnDejvW8+8KdHUNHIW1KssdVxsFt1zOaygjQpKFNeD2PZJLGygjspI9HJCwa5cyT3TDTB5+FSdCLt3eoWBC+t735yz7edMYMU7US3fKEZ9YMZXwOvhf6RZ49Wmeprh3vg3ahs/Hn+1nNItKcTSuAnPmzaX6K6y55Hq8+M7l/Es0ix8Ftyn3kVNsduc/CYikqG+6JiKeq3xSRGwY+sgmgW2Gxbq7Kz39+uKVMRzv2bG99GJpkskbG75F6SOdklWZZNRXElWAhcRabBCnTHc/G5B2UyYoFUppFvanxd/c9+8GQ+CwgdKYWVWgNxxCaGt554Y6WffIwN+BszY/CjjUVuttOS8rXLNJCw5hAzAN/od5koaRmEZuhoizuvMJ1x8rcMfgs7ICGSmrlXUazaLXD5x07tW+OZpE319JmqDCM3AiG+akKi/WwyVCna+BbWnDWwR3v05JnER7T+JnyVv4iwjUvPSW1zcyb9bmaRes1CqyF1jAoIyyeFpE54B+BT4nIY0C9w3scdL+SXzdb48lDi1auQW/qZeB5cSc6eyXi56zo27Ft3Wwqe/i0zauo+h6b10wVvsc2SxSREha2ZpHyWSRVRmNhUUmSufKckrVKZ0GVh/mc6Urop6lH5VCMz6jdzXjC6il8T9iyZrolQszuIw2J32Gp0WSpRKtdSFb9hy2fRf/zLHrXLKqWFmVyLkTaX39DnMFdRlhk5pX9APa9KPTait00/gLfE45GeSqxfykK3nji4ELHcZpIrdZoKNvBnj6G+RxjCutk6jJsWj3FmpkKa3N8FkFkWk2boVr9N4OkjLC4DDgKvBt4E7Aa+FCZg4vIxcBHCJsffVxV/1POPm8APkgYpnunqr5RRM4C/gyYJ2yK9O9V9bNlPnOcMKpy6Kwtp1kcXKjH8eK9qpeBL3GnPPsYprjhUqNcWO6Hf/tMLMWCzaunufUDL2ftTKXwPUmeRTnNIjZDGQe3JlpVJTZDhQ8hU9O/qJ5SHF7Z5cPUmAmmKj61SljQsWzo7O6Tj2PvdReydrba4sexzVCqGtuwF+vNlL+mHfk+i/4+HPKcp2VJNMnkehUJ8yxlNYvAun/yzFAQPkztnBaTDDtXC2KfUaIFhvN3/6HFVFJhHnYhQa/ADJUVsuZ8PnUoDPstm2F95Tkn8poXnFB4b9oJi5Cci5FrFiLyMeDTqvpda/Mnyx5YRHzgRsJWrA8Dt4nILap6t7XPDuD9wLmq+lQUcQVwGHizqt4rIicAt4vIV1X16dLfbAwwF73MChKSKIhHnw3DQ3tdQVZ8L6wN1WytdR94HkuNRqkx5Tkpi5L5DGWiofLMUJ5noqGa8d/GcZjkWYTTtSjk1Kz2yp5vg92Mx7bBl3FwA/FKMGuGMg9SIxyM1mR8FmXG2RoN1f+S1DPV3qOh7DBhc53K+CvANKlqra1kqFoanyHPDAXhA3nR+ttoFnO1gKNLYT6FOZ4JC99/cIFt6zoUmTSlOzIanW1my14PU9vKlNMvu/IPfI/V08X7ztT8XDPUOFSdvZewtelDInJDtNrvhj3Afar6gKouAp8h1FJsrgFuVNWnAFT1sej3T1X13uj1PuAxYEOXnz9yzCQpe/OY+GrT8a7XSVCJNIuwjlB2BTbYcLtqCQGZp1kEnoRJeZa/xqzI4oeGqSnUQbMoq/bHY4gyvu1aWxW/XOhs6vMLNIvFejNlzluKy0D0oFmUzOzvhvN3rOeK3Se2NS8WYefVmNdlIv8MU4FXeH7jqLKcCsDZc5cV6IcWohbA1aS0h11HDcIKwXnOZxv7XKc1C8sMljlGLIyiMPhu52MRV79kO5edtSX+uzLgezlL4bdQ1Y+o6ouB3wCeBP5SRH4sIteLyM4Sx94C/ML6++Fom81OYKeIfEdEbo3MVilEZA9QBe4v8ZljRdeaReTYeuxApFn0aG6wy320Rmrkr8z6RdeaRbRfrFloOs/Ctn+b+jhFztOpHn0WFU/im99u/dmtYC30WTQaqexi05ynjBnKfFcTSTUIn8WJx81ww2+f2dOcSDu4Ew2tLFMVv1izCMzxbP9AgWaRmSOHF+tU/KhoYCQs7GCE7GcUURQgku3yZ2OOb3Km+lXo723nbefiMzZZn9vZ5NtPOn6Kqv5MVW9Q1bOBNwKvo1zzo7wzlC0fEgA7gJcRhuN+XETimsoishn4a+BqVW0p3yki14rIXhHZ+/jjj5cY0nAxD8IyDwVI6sUcsxnKiwoJ5ti3s42Q+k2ZEuW2uSOOhspNygtXq8b+baKhinwW2SzfsqyeqcRanXnQVwIrdLbkzTiVNUMZn8VSM46MgagqakE10iyeJ3EvkfC96b4KoyYpbZKUWOlKs8g0NbLJEz52QywbcwxTsfbgQhikYMqR2z4iu6Jyp/sgLwkO0taC7PUIfC/OEof+aRZZhp3B3fFbiEhFRF4jIp8iTMb7KfD6Esd+GDjR+nsrsC9nny+p6pKqPgjcQyg8EJF54H8D/1ZVb837AFW9KeoNvnvDhvGzUsWaRcnJYjSLXz1jzFDHoFnUw3DBouzSTup3r8SaS9t6OzkObi9HWHjpRK/YZ1Fg1puyQme74V0v38knr94DWHkDXjkHt02hGarRaoZarJfzG0FSeRbCkNC5WvmV+6DJM9t1o1nUKl5HzSLlszCaRYuD24Q/h3PkUBQWawr+2T6ilGbR4RrY92CRgztveqyaChJhMaCFWZkS+v2k8FuIyEUicjPhA/1a4MvAc1T1ClX9Yolj3wbsEJHtIlIFrgRuyezzReCC6PPWE5qlHoj2/wLwV6r6uW6/1LgQO/xKxJxD0pvXmKF6XTEEvrAUZUO3xIAPONzOOCzL5llkk/JiYRE1w7FXqcZ+X1SpdbpHM9TqmQrb1oWZ3namcLYAYyeyGdzmQbRQb6bMUN1EQ0EohIywOXi0nqoDNmpsM5SJUOrOZ+EXPuwS4WOZoQrqdZlrbs6N6cJoSonbSZBz1SDpfdKxDE/yusjB7efMj1VTlVhYDMoBnZyL0edZfAD4NPD7qvpktwdW1bqIvAP4KmHo7M2qepeIfAjYq6q3RP97hYjcTRgi+94oAfBfAC8F1onIW6NDvlVV7+h2HKOkW81CRDh+foqfP3k4en+vZigvrDqb40SNzVADUo0BLj1zc26yoKFtUp6lWZx76rpUoTUjLIpWrrtOWsOLT1nHptXdO2oNdgZ6t9EmcV/yjO9isd6Mo3Mg1DTKZA8bZqtBHLN/cKHOXK04dHnYxMIi6M1nceHpx7OqQPglPotW/0CrGcr4LJJjVXyhVvFZWGqwUG/w3OPD2qieFxbePHC0uGNkctx8zaLiJ7kdefNjfiqI53LZ+79bkgoDwzFLtqs6e8GxHlxVv0yokdjbrrdeK/Ce6Mfe52+AvznWzx81sWbRxcXcvn6Wb/009L8ci2bRVNNtb7gOboCPXHl22//nJeV5kc+iGSckCpfv2srlu7bG+xoTQ9HD6LRN8/zttS86prHXrPpS3UZDTWU0i1oUOruQjYaKNIsyWfRAVMuoTrOpHFqsx3b5ccCEB1d9K3S2i4fjey4qjpXJ08zj0NmcPAsgdW6Mz+LgQp0nDy1y8fMT5/D8VCUUFh0d3PbrdM7SVORLyrtPu/GL9Mqwy32Mj6dsAql06eCGUFiYRLheJ5l5X9iJbrg+izLkObiDyAxl97PIMt3BDNUPbDNU0o62WzNUa9XZlBkqzuAu9z3mpyscWFji8FIDVcbWZ5GEivdnfObYdkBDUeismc+2ic5EQz12YIF6U1Pl6ldZrVvb4WcyxW3aJRXafpFBm6HGxsHt6J345ulCWNjNenrWLKL3HVlqXfWUTTQbJHk+C89Lss6z+xg6Obj7genlbWfdH7ODu95MtflcapSvDQXhg+fZI3UOHjWJZmNohuoxz6LtsY1ZyzpeUcMfc61mLUFaCbyUFnrKhuTeMolznRZk9udkFzBTlgk1i61ZDMoMVRRGPCicsBgg3fosIN1/oVefRZDSLDKOwCFPsDzSwiJaHUnawZ0XCRaX+xioZuG1aF9lhXY2z8JEdC3UG3GeBEQO7nqztHZnSmofXAjLR4yVGSrl4O7eZ1Hm2NkKr3Y4dbw9x2dR9SW1sLDvrbKaRaoznZ+vWeTND7uHvdMsHB2p9iIsNtiaRa9mqHDyHF1qLWcdNxQaoRnKvnnsJD47dDbvq89WA07btIrTNg+uo+/zNs9z5pbVAOzctIrnbZ5vW3XW5jkb5tiwqsbJVgmJauC1mKHM6/KaRWhffzbSLIocwqPALiTYiyZd5tj2A/+0Tas4c+vqln1tE5htqjJRS6unK6maZkZYdKNZZGtA1dqYoeZtn8WgHdxjEA3lOEaME66buP/N81Nxe86eCwlaPYBbkvKG4ODuhL1ayzq46200C98TvvKulw50bFfu2caVe7YBcOmZJ3DpmSeUfu+2dTPcdt2FqW3VwGvJszCFIsteg/mpCvWm8sSBMP9mnDQLO0y475pFTm2oK87ZxhXnbGvZ1+59UvHD+mehzyJ87/b1syltxJT86CTYipLywO7elxc6Wz6Xo1cSB7czQy17uq0NBeGD1Djiei8kmPgsWpLyBpzBXYZ0X43EZNPQdLmPSaAWaRZHlhpW6epuNYvwwfNI1H98nPIs7MKLdtXZfpAUJux8PPPAtNu7Ggc3pH2BYJmhOmVwSxthEZTTLAZnhkq08mHghMUASWzf3d08xhHX64rBfO6RpUbLRIqT8gaYZ9GJPM0izOCGRtQoaFKERTXSEg8vNuJOfEaz6FZY7Hv6CDBewiIuUR4ItcDnhtf/GpfvypaA6/HYXWgqduRhYhqTWNDYzm1IHNBl+1lAjoM70xc8ffykUdeg5nKZfiv9ZHxm3QRi23O7IdYsjiHPAkBzEoaCjPN2FOS2VRWh0WwSFZ2dHGHhJ30W5qcCHj+w0JMZChLNYtUYmaGqlhkKyDUR9Uotx2dRRGD5S+zKx4kZai61f+LgLtfPAlqry7ZzcK+yoq3K9PbohWGXKB+fWTeB9OLgBjjrxLXUAo81bZoMtcPWSLIhsuNghvJSeRbJ6sjuZzExwiLwWag3aTSbzFQDqoEXZ2OXdQSbyJpHngk1i9kx0izmpwNW1QI2r57u+7GPm61RCzxOXDvTcd+4nH3gpRzcW9ZME3jC809IB0UYAdxN1dlWzaKdsAhS4xoESddAJyyWPb1Gh1z4vI3c/ocX9WxusFXrIs1ilHkWqWgoP7nhmorl4J4MYVGLHNwLSw2mqz5V3+NQFEZb1vFpVqn7nj6aCu0dB2aqAbd+4OUDCWc+brbK9//woriMSjsC3xYWkc8i8HjhSWv55+svSuU9gB0N1X6epfIsChzcuT6LyOQ4SHNv3CnP5VksfypWhEY3iMgx2aVtQdDaz0JSv0dBKs/C0izqzWZc7mNyNAuPhaUGR5YazFTDsE7j4C770DcPtkefPTpWJijDbC0o7KPdj2OXMeMkZii/pY9MVlDY244lGspk6rfVLAb4II8jG12exfKnl6S8vnyunUhUUJ1zUOF8ZRAJi7DZ4/BEaDZpW+5jOWI0i8OLobCoBl4PDu7wwVZv6lg5t8eJlBnKKkFSxPNPmOeqPds45+TigpeQnqvZOVmLNYvW62gq2w7DDOUc3BNA1vk3LFKaRdYMNQYZ3BCZnRoa39iB1c9ChIGtVIeNcXAfWWwwXQmo+N0Li9mqH1c4Hacci3HCXgRVLZ9FEVMVn/94+a+VOnbghQK/xcHdRrPwPGGuGgzYDOVCZyeG1dMVPAltr8PEnjzZCV5UiG3YmBss1ixMnkWztW/4cmamFvD04SUOL9aZroaROge7jIYSkVi7cJpFPoGlxfe7srJRHLpxcEPotxisGcol5U0Mx89P8bV3v5QLn3f8UD/XLk+RLVWROAJH+0A2N17S/IhYs8iWVVjO7Dl5Lb98+ghPHV6Ko6EOdRkNBYkNfJyKCI4Ttn+w3+Zf8zAuzuDOn6+rpoKBLnyKiioOCicsBsypG1cN3aSS0izG1AzleRJ31YNwXKbcxyRpFhectjF+PV0Jna+RW6arh5kJ9RxHB/c4YOZ1LaVZ9GcexT6LLkqUQ3itBumvnKhCgiJysYjcIyL3icj7CvZ5g4jcLSJ3icinre1fEZGnReTvBznGScS+SVr7WYyHGSrwJDW20MEdahaTEgkFsHXtDKdtCju0GQe3oZtrYITE7Bj1shgn7NBZuwRJf46dr0Gcc/JaXvbcDYWdGS8+YzOvtBou9Zth38sDW6aIiA/cCFxE2Mf7NhG5RVXvtvbZAbwfOFdVnxKRjdYhPgzMAL8zqDFOKvbkyRY5G3Y9mSJ8T1KOf98Lo30mTVgA/OZpG/nJrw7EobOGbladic/CmaHyqKTyLJLX/cDMx6x59NSNq/jE1XsK3/f287b35fOLKDKPDYpBiqQ9wH2q+oCqLgKfAS7L7HMNcKOqPgWgqo+Zf6jqN4ADAxzfxGJHQ2U1i9mqj0j/KoP2iu9J6maOHdyqPZdmH1cuPD30WWUdnt2YSeYjzcKZofKZiUJVZ6KIM+jfitv418bNPGq0zDJJi/1gkDNvC/AL6++HgV/P7LMTQES+A/jAB1X1KwMc04rAju3Orjpet2srp2yYS1XFHAW+CF4m07zZVBqNyfJZAJx94hr+8q3n8OLnrON/3bkv3l7rosCkyQh20VD5/NbZW9i+fpbVM5U4XLVfIeuxZjFm83LXtrV8/M27eeFJa4fyeYOceXlnVnM+fwfwMmAr8G0ROUNVny71ASLXAtcCbNvWvwJmy510Bnf6hpmrBZx76vphD6kF35eUWu+LUI8c3JNmhhKR2NFtr3a7M0OZaCgnLPKw53W1z5pFEqI6XvNSRGKtdRgMUt9/GDjR+nsrsC9nny+p6pKqPgjcQyg8SqGqN6nqblXdvWHDhmMe8KQQZFbs44gvkrqZzaqt3mzt7jdJ9GqGioWFM0N1pN8lbYwZapLnZRkGKSxuA3aIyHYRqQJXArdk9vkicAGAiKwnNEs9MMAxrQjs3IqxFRYZB7cZ52J9soWF0SY86a6Yo3Fwj1NL1XHFLiTYD4oc3CuNgQkLVa0D7wC+CvwY+DtVvUtEPiQir412+yqwX0TuBr4JvFdV9wOIyLeBzwEvF5GHReSVgxrrpJHSLEYc9VSE70nqZjaaxUK9yQTLip4TxkzjpLyieI40/a5/luQCTfDELMFAlymq+mXgy5lt11uvFXhP9JN97/mDHNskY0/qYZUC6Bbf81INmIyq/8yRpYl+IPaaA3DBczdy/aWnt/RlcLRS7XOexbg6uIfNeD5JHMeEiFjN3MdzgvteenVtbsj9BxdY22PTp+VAr2Xrp6s+bztv+4p/YJWh3z6Lcb+XhoUTFhPKsEsBdMuWNdNsXZN0QEuExSJrZ4ZbeHGYjEOJ+Emn/4UEnWYBrkT5xFLxPI7SHHlZjyL++5temPrbCIsDC3XWTLCwiE0kQ+5xspIw57hb7a2IuCPdCndwO2ExoYy7ZpEdl/33ZJuhnGYxaCpxSRvn4O4nbsZOKONSA6os9qptzZD7fwyTfieMOVr5jZ0buOb87Ww7bqbzziVwDu4Qp1lMKOahNK7RUFm8FaJZxN0TnRlqYBw/P8V1rz69b8fzh9w3YlxxM3ZCiUsULBPNIkgJi8nVLEbVl93RO+YWWumahZuxE8pyC/ezfRZrVoJm4cxQywajWax0B7ebsRNKZbmZoWSlaBb97bXgGDxxNNQyWXgNCjdjJ5TlZobyV4gZquqioZYdvhMWgBMWE8uwm7kfK+ZGrAUe00Nq5jIKXJ7F8sMJixA3YyeUSlyDf3lcYmMPnmStAlyexXLEmaFC3IydUGLNYpmZoSbZuQ0udHY54rkMbsAJi4llXLt7FWGExcrRLJbHdXE4zcLghMWE0u+SB4MmFhazE65ZuDyLZYfnhAXghMXEstxWQ94K8Vk4M9TyY7ndS4NioDNWRC4WkXtE5D4ReV/BPm8QkbtF5C4R+bS1/S0icm/085ZBjnMSSco0L48Jbsxmky4skl4LTlgsF1w0VMjAakOJiA/cCFwEPAzcJiK3qOrd1j47gPcD56rqUyKyMdp+HPBHwG5Agduj9z41qPFOGpUxrzqbxWgWk+7gduU+lh/Gse0c3INjD3Cfqj6gqovAZ4DLMvtcA9xohICqPhZtfyXwdVV9Mvrf14GLBzjWicP4KirLJXR2hTi4a67cx7LDNwuvZaKlD4pBztgtwC+svx+OttnsBHaKyHdE5FYRubiL9yIi14rIXhHZ+/jjj/dx6Mufii+ILJ/iZ5vmp6gGHs/dtGrUQxko89MV1sxUOGnd7KiH4iiJa34UMsgS5XlnVnM+fwfwMmAr8G0ROaPke1HVm4CbAHbv3t3y/5VM4HnLRqsA2LR6inv++GJkwm/IqYrPHde/AlU3XZcLsRlqmSy8BsUgnyYPAydaf28F9uXs8yVVXVLVB4F7CIVHmfc62lDxvWU3uSddUNispO+63DFVZ70Vfs0GqVncBuwQke3AL4ErgTdm9vkicBXwCRFZT2iWegC4H/gPIrI22u8VhI5wR0ku37WFk9b1p1OYw7GSefWZm5mqeCs+KGFgwkJV6yLyDuCrgA/crKp3iciHgL2qekv0v1eIyN1AA3ivqu4HEJE/JhQ4AB9S1ScHNdZJ5Iwtqzljy+pRD8PhWPacunGOUzfOjXoYI0cmxXa6e/du3bt376iH4XA4HMsKEbldVXd32m9l61UOh8PhKIUTFg6Hw+HoiBMWDofD4eiIExYOh8Ph6IgTFg6Hw+HoiBMWDofD4eiIExYOh8Ph6MjE5FmIyOPAz47hEOuBJ/o0nH7ixtUd4zouGN+xuXF1x7iOC3ob20mquqHTThMjLI4VEdlbJjFl2Lhxdce4jgvGd2xuXN0xruOCwY7NmaEcDofD0REnLBwOh8PREScsEm4a9QAKcOPqjnEdF4zv2Ny4umNcxwUDHJvzWTgcDoejI06zcDgcDkdHVrywEJGLReQeEblPRN43wnGcKCLfFJEfi8hdIvLOaPsHReSXInJH9HPJiMb3kIj8MBrD3mjbcSLydRG5N/q9ttNx+jym51rn5Q4ReVZE3jWKcyYiN4vIYyLyI2tb7vmRkI9Gc+4HIrJryOP6sIj8JPrsL4jImmj7ySJyxDpvfz6ocbUZW+G1E5H3R+fsHhF55ZDH9VlrTA+JyB3R9qGdszbPiOHMM1VdsT+ETZnuB04BqsCdwOkjGstmYFf0ehXwU+B04IPA74/BuXoIWJ/Z9p+B90Wv3wfcMOJr+SvgpFGcM+ClwC6atw5lAAAFIUlEQVTgR53OD3AJ8A+EveZfBHxvyON6BRBEr2+wxnWyvd+IzlnutYvuhTuBGrA9um/9YY0r8///Alw/7HPW5hkxlHm20jWLPcB9qvqAqi4CnwEuG8VAVPURVf1+9PoA8GNgyyjG0gWXAZ+MXn8S+K0RjuXlwP2qeiyJmT2jqv8IZLs5Fp2fy4C/0pBbgTUisnlY41LVr6lqPfrzVsIe90On4JwVcRnwGVVdUNUHgfsI79+hjkvC5ulvAP52EJ/djjbPiKHMs5UuLLYAv7D+fpgxeECLyMnA2cD3ok3viNTIm4dt6rFQ4GsicruIXBttO15VH4FwIgMbRzQ2CHu82zfwOJyzovMzTvPubYSrT8N2EflnEfmWiJw/ojHlXbtxOWfnA4+q6r3WtqGfs8wzYijzbKULC8nZNtLwMBGZA/4n8C5VfRb4M+A5wFnAI4Qq8Cg4V1V3Aa8C/pWIvHRE42hBRKrAa4HPRZvG5ZwVMRbzTkSuA+rAp6JNjwDbVPVs4D3Ap0VkfsjDKrp2Y3HOgKtIL0qGfs5ynhGFu+Zs6/mcrXRh8TBwovX3VmDfiMaCiFQIJ8GnVPXzAKr6qKo2VLUJ/AUDUr07oar7ot+PAV+IxvGoUWuj34+NYmyEAuz7qvpoNMaxOGcUn5+RzzsReQtwKfAmjQzckYlnf/T6dkK/wM5hjqvNtRuHcxYAlwOfNduGfc7ynhEMaZ6tdGFxG7BDRLZHq9MrgVtGMZDIFvo/gB+r6p9a220b4+uAH2XfO4SxzYrIKvOa0EH6I8Jz9ZZot7cAXxr22CJSq71xOGcRRefnFuDNUbTKi4BnjBlhGIjIxcAfAK9V1cPW9g0i4kevTwF2AA8Ma1zR5xZdu1uAK0WkJiLbo7H90zDHBlwI/ERVHzYbhnnOip4RDGueDcOLP84/hBEDPyVcEVw3wnGcR6gi/gC4I/q5BPhr4IfR9luAzSMY2ymEkSh3AneZ8wSsA74B3Bv9Pm4EY5sB9gOrrW1DP2eEwuoRYIlwRff2ovNDaB64MZpzPwR2D3lc9xHass08+/No39dH1/dO4PvAa0ZwzgqvHXBddM7uAV41zHFF2z8B/MvMvkM7Z22eEUOZZy6D2+FwOBwdWelmKIfD4XCUwAkLh8PhcHTECQuHw+FwdMQJC4fD4XB0xAkLh8PhcHTECQuHowtEpCHpSrd9q1QcVTAdVU6Iw9GWYNQDcDiWGUdU9axRD8LhGDZOs3A4+kDU4+AGEfmn6OfUaPtJIvKNqDDeN0RkW7T9eAl7SdwZ/bwkOpQvIn8R9Sv4mohMj+xLORwWTlg4HN0xnTFDXWH971lV3QN8DPhv0baPEZaJPpOwYN9Ho+0fBb6lqi8g7J1wV7R9B3Cjqj4feJowQ9jhGDkug9vh6AIROaiqcznbHwJ+U1UfiIq9/UpV14nIE4QlK5ai7Y+o6noReRzYqqoL1jFOBr6uqjuiv/8AqKjqnwz+mzkc7XGahcPRP7TgddE+eSxYrxs4v6JjTHDCwuHoH1dYv/9f9Pq7hNWMAd4E/N/o9TeA3wUQEX8EfSMcjq5wqxaHozumReQO6++vqKoJn62JyPcIF2FXRdv+DXCziLwXeBy4Otr+TuAmEXk7oQbxu4SVTh2OscT5LByOPhD5LHar6hOjHovDMQicGcrhcDgcHXGahcPhcDg64jQLh8PhcHTECQuHw+FwdMQJC4fD4XB0xAkLh8PhcHTECQuHw+FwdMQJC4fD4XB05P8DA2yUSM7xktoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e9432fc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(val_acc)\n",
    "plt.ylabel('Validation Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.savefig('val_acc.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_accuracy(batch_size, x_test, y_test, sent_attn_model):\n",
    "    acc = []\n",
    "    test_length = len(x_test)\n",
    "    for j in range(int(test_length/batch_size)):\n",
    "        x,y = gen_batch(x_test,y_test,batch_size)\n",
    "        state_word = sent_attn_model.init_hidden_word()\n",
    "        state_sent = sent_attn_model.init_hidden_sent()\n",
    "        \n",
    "        y_pred, state_sent = sent_attn_model(x, state_sent, state_word)\n",
    "        max_index = y_pred.max(dim = 1)[1]\n",
    "        correct = (max_index == torch.cuda.LongTensor(y)).sum()\n",
    "        acc.append(float(correct)/batch_size)\n",
    "    return np.mean(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.646875"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_accuracy(batch_size, X_test_pad, y_test_tensor, sent_attn)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}