SentimentAnalysisClassificationWithLSTM_GoogleColab.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 238,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4948,
     "status": "ok",
     "timestamp": 1521555309252,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "k2YrqT9OPT8m",
    "outputId": "e0441abc-00a7-4f24-ab22-6f06fe08c389"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[name: \"/device:CPU:0\"\n",
       " device_type: \"CPU\"\n",
       " memory_limit: 268435456\n",
       " locality {\n",
       " }\n",
       " incarnation: 8910045012864862651, name: \"/device:GPU:0\"\n",
       " device_type: \"GPU\"\n",
       " memory_limit: 11297803469\n",
       " locality {\n",
       "   bus_id: 1\n",
       " }\n",
       " incarnation: 4122465078757566685\n",
       " physical_device_desc: \"device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7\"]"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "device_lib.list_local_devices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 343,
     "output_extras": [
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2927,
     "status": "ok",
     "timestamp": 1521555314315,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "2vaAfxgwxbhC",
    "outputId": "24afc89d-2e82-4aa1-ba9f-d09befa384c4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in /usr/local/lib/python3.6/dist-packages\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim)\n",
      "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim)\n",
      "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim)\n",
      "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: bz2file in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: s3transfer<0.2.0,>=0.1.10 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: botocore<1.10.0,>=1.9.11 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: python-dateutil<2.7.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.10.0,>=1.9.11->boto3->smart-open>=1.2.1->gensim)\n",
      "Requirement already satisfied: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.10.0,>=1.9.11->boto3->smart-open>=1.2.1->gensim)\n"
     ]
    }
   ],
   "source": [
    "!pip install gensim\n",
    "import gensim, logging\n",
    "from gensim.models.word2vec import Word2Vec "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 170,
     "output_extras": [
      {
       "item_id": 5
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12597,
     "status": "ok",
     "timestamp": 1521555327479,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "TYWMYqoAVui6",
    "outputId": "372221c2-8add-4879-d415-ad4f878519e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pickle\n",
      "\u001b[31m  Could not find a version that satisfies the requirement pickle (from versions: )\u001b[0m\n",
      "\u001b[31mNo matching distribution found for pickle\u001b[0m\n",
      "title: Documents.pickle, id: 1lUEORk5uwHRm6JkTYoGAK3JaRBd7FU9k\n",
      "downloading to /content/data/Documents.pickle\n",
      "title: model.bin, id: 1xjpW0CDroJyCC_RYI5CroSoZ6MosAZlL\n",
      "downloading to /content/data/model.bin\n",
      "title: y_train.npy, id: 1WV2aWz1yjPOkrnKqpQX2KYN4S5WcjrL2\n",
      "downloading to /content/data/y_train.npy\n"
     ]
    }
   ],
   "source": [
    "!pip install -U -q PyDrive\n",
    "!pip install pickle\n",
    "import os\n",
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "import pickle\n",
    "\n",
    "# 1. Authenticate and create the PyDrive client.\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)\n",
    "\n",
    "# choose a local (colab) directory to store the data.\n",
    "local_download_path = os.path.expanduser('~/data')\n",
    "try:\n",
    "  os.makedirs(local_download_path)\n",
    "except: pass\n",
    "\n",
    "# 2. Auto-iterate using the query syntax\n",
    "#    https://developers.google.com/drive/v2/web/search-parameters\n",
    "file_list = drive.ListFile(\n",
    "    {'q': \"'1S96PP4Pe_LxahlU-Co8foL2DzZE2bKo9' in parents\"}).GetList()\n",
    "\n",
    "for f in file_list:\n",
    "  # 3. Create & download by id.\n",
    "  print('title: %s, id: %s' % (f['title'], f['id']))\n",
    "  fname = os.path.join(local_download_path, f['title'])\n",
    "  print('downloading to {}'.format(fname))\n",
    "  f_ = drive.CreateFile({'id': f['id']})\n",
    "  f_.GetContentFile(fname)\n",
    "  if fname == '/content/data/model.bin':\n",
    "    model = Word2Vec.load(fname) #word2vec model\n",
    "  if fname == '/content/data/Documents.pickle':\n",
    "    with open(fname, 'rb') as f:\n",
    "      Documents = pickle.load(f)\n",
    "  \n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wPeyAm5iymxk"
   },
   "source": [
    "# Load model Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 360,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 584,
     "status": "ok",
     "timestamp": 1521555339165,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "7wlT9XIAmyLS",
    "outputId": "94694921-e0f8-4b49-a86a-735f0cfb6d97"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1.7942369  -0.33382452  0.5339357  -1.4902467   2.1332428  -1.0774956\n",
      " -0.21262288  0.34546664  0.18661968  0.5688681  -0.42535612 -1.4686878\n",
      "  0.93791354 -2.3718512  -0.70141095 -0.9795086  -0.20312025 -1.0589772\n",
      "  0.24206859 -1.1336994   1.6416202  -0.6261653   0.84927803 -0.4769663\n",
      "  0.9985494   2.1509867  -1.1989815  -0.41598585 -0.09528159 -0.7134422\n",
      "  1.7122741   0.5545024   1.277535    1.1946003   0.40057173  0.09237026\n",
      "  0.4424268   1.0867667   0.90509695 -0.19481269  0.91103834  1.1511343\n",
      " -0.06785039 -1.4012144  -1.4653159   2.5017827  -0.34803048  0.43969256\n",
      " -0.88642263  0.0435985   0.38556442 -0.5132431   0.47357002  0.9714809\n",
      " -1.6138848   1.6057731  -1.0455496  -0.12987329 -0.04974201  1.3797334\n",
      " -0.5297085  -1.210395   -0.49806824  2.032588    1.1804183  -0.10756348\n",
      " -0.62049466  0.31814304  0.93198586 -0.41446197 -0.50262237  0.6320385\n",
      "  0.2493732   1.0782746   0.12296637 -0.33538577 -0.5217311  -1.2326243\n",
      " -0.45114192 -1.5398091  -0.46345225  2.1170738   0.02449238 -0.39949843\n",
      "  0.2740543   0.42163128 -0.11416981  1.0085219  -0.1488119  -0.6154405\n",
      " -0.9643602   1.5402884   1.0199872   0.38207138 -0.2976515  -0.10284822\n",
      "  0.42682004  0.0925106  -0.1790923  -0.11463826]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "print(model['good'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Pu3f0AGNz-JS"
   },
   "source": [
    "# Load Document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1182,
     "status": "ok",
     "timestamp": 1521555341676,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "TPnmuhKgz27I",
    "outputId": "3b42aedf-1c34-480f-afa2-13fc33ed8184"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(['This', 'cd', 'storage', 'unit', 'isnt', 'greatest', 'cd', 'rack', 'want', 'something', 'job', 'isnt', 'costly', 'thing', 'made', 'plasti'], 'neg')\n"
     ]
    }
   ],
   "source": [
    "print(Documents[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yPvYyl110Sse"
   },
   "source": [
    "### Prepare our input this time :\n",
    "-  the Training shape will be (M,100,n) where M is number of **exemple** ,100 each word dimension and n is the number of words in each sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 119,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 571,
     "status": "ok",
     "timestamp": 1521555343390,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "CZtElchjm0Zk",
    "outputId": "f3ece198-4abe-4fd3-a35b-7309f6c32ce5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max length sentence is : 506 Min length sentence is : 2\n",
      "Sentence with more than 50 word  8366\n",
      "Sentence with more than 100 word  1373\n",
      "Sentence with more than 200 word  92\n",
      "Sentence with more than 300 word  18\n",
      "Sentence with more than 400 word  6\n"
     ]
    }
   ],
   "source": [
    "def get_Max_Min_length_Sent():\n",
    "    Max = 0\n",
    "    Min = 10\n",
    "    CountSup50 = 0\n",
    "    CountSup100 = 0\n",
    "    CountSup200 = 0\n",
    "    CountSup300 = 0\n",
    "    CountSup400 = 0\n",
    "    for S,t in Documents:\n",
    "        if len(S) > Max:\n",
    "            Max = len(S)\n",
    "        if len(S) < Min :\n",
    "            Min = len(S)\n",
    "        if len(S) > 50:\n",
    "            CountSup50 +=1\n",
    "        if len(S) > 100:\n",
    "            CountSup100 +=1\n",
    "        if len(S) >200 :\n",
    "            CountSup200 +=1\n",
    "        if len(S) > 300:\n",
    "            CountSup300 +=1\n",
    "        if len(S) >400 :\n",
    "            CountSup400 +=1\n",
    "            \n",
    "    return Max,Min,CountSup50,CountSup100,CountSup200,CountSup300,CountSup400\n",
    "\n",
    "Max,Min,CountSup50,CountSup100,CountSup200,CountSup300,CountSup400 = get_Max_Min_length_Sent()\n",
    "print(\"Max length sentence is :\",Max,\"Min length sentence is :\",Min)\n",
    "print(\"Sentence with more than 50 word \",CountSup50)\n",
    "print(\"Sentence with more than 100 word \",CountSup100)\n",
    "print(\"Sentence with more than 200 word \",CountSup200)\n",
    "print(\"Sentence with more than 300 word \",CountSup300)\n",
    "print(\"Sentence with more than 400 word \",CountSup400)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EdvgVOmh0bEi"
   },
   "source": [
    "### fix_sentence_length is a function to fix the size of sentences to a fixed size to feed them into the neural net with specific length sentences with length less than that will be extended with zeros this process don't affect the alogrithme and sentences with length more that the specified length will be truncated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 54,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 560,
     "status": "ok",
     "timestamp": 1521555345133,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "BVqmVN0I0brI",
    "outputId": "239b6c67-11b7-447e-9976-aea994ac266c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 5, 4, 8, 7, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n"
     ]
    }
   ],
   "source": [
    "def fix_sentence_length(length,exemple):\n",
    "    if len(exemple) > length:\n",
    "        exemple = exemple[:length]\n",
    "    elif len(exemple) < length:\n",
    "        for i in range(length - len(exemple)):\n",
    "            listzeros = [0] * 100\n",
    "            exemple.append(listzeros)\n",
    "    return exemple\n",
    "ex = [2,5,4,8,7]\n",
    "ex = fix_sentence_length(7,ex)\n",
    "print(ex) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "AjZc0AIE0dx2"
   },
   "outputs": [],
   "source": [
    "def Sent_Embeding_sequence(words,Documents):\n",
    "    X_SentsEmb = []\n",
    "    y_SentsEmb = []\n",
    "    #on Va récuperer l'embeding de chaque sentence avec l'average des vecteur qui le compose\n",
    "    for S,t in Documents: #S est la phrase , t est la class\n",
    "        #Average of embeding of all words in sentence\n",
    "        Sent = []\n",
    "        Sent = [list(model[m]) for m in S]\n",
    "#        if len(Sent) < 100:\n",
    "#            for i in range(100 - len(Sent)):\n",
    "#                Sent.append(0) # pour avoir une longeur standart de 100 mots par Sent\n",
    "#        else :\n",
    "#            Sent = Sent[:100]\n",
    "        #Add Sentence Vector to list\n",
    "        Sent = fix_sentence_length(100,Sent)\n",
    "        X_SentsEmb.append(Sent)\n",
    "        #Add label to y_SentEmb\n",
    "        if t == 'neg':\n",
    "            y_SentsEmb.append(0)\n",
    "        elif t == 'pos':\n",
    "            y_SentsEmb.append(1)\n",
    "        else:\n",
    "            print('No match label !!! ',t)\n",
    "    return X_SentsEmb,y_SentsEmb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 71,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 40770,
     "status": "ok",
     "timestamp": 1521555387482,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "0xuao2CO0mY2",
    "outputId": "3c3d079f-7b7e-4437-8118-068344bd62ae"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "words = list(model.wv.vocab)\n",
    "X,Y = Sent_Embeding_sequence(words,Documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 33862,
     "status": "ok",
     "timestamp": 1521555425287,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "HzMTSx1H0qEq",
    "outputId": "24325769-0d9d-45d8-cf0f-006da26d617e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X Shape (38252, 100, 100)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "X_a = np.array(X)\n",
    "del X\n",
    "#lista = np.reshape(lista, (len(lista),2, 3))\n",
    "print(\"X Shape\",X_a.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "5TC7flrNM35m"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "#train test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_a, Y, test_size=0.15, random_state=1)\n",
    "#train dev\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.15, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 119,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 987,
     "status": "ok",
     "timestamp": 1521555471777,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "R7puCswONBpw",
    "outputId": "8968a0d3-83b9-4a67-fd1d-9b3c8c8cfe3a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length X_train : 27636\n",
      "length y_train : 27636\n",
      "length X_test : 5738\n",
      "length y_test : 5738\n",
      "length X_val : 4878\n",
      "length y_val : 4878\n"
     ]
    }
   ],
   "source": [
    "print('length X_train :',len(X_train))\n",
    "print('length y_train :',len(y_train))\n",
    "print('length X_test :',len(X_test))\n",
    "print('length y_test :',len(y_test))\n",
    "print('length X_val :',len(X_val))\n",
    "print('length y_val :',len(y_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gKiYDSOI7_Kk"
   },
   "source": [
    "## LSTM\n",
    "#### On Batch Size\n",
    "\n",
    "A benefit of using Keras is that it is built on top of symbolic mathematical libraries such as TensorFlow and Theano for fast and efficient computation. This is needed with large neural networks.\n",
    "\n",
    "A downside of using these efficient libraries is that you must define the scope of your data upfront and for all time. Specifically, the batch size.\n",
    "\n",
    "The batch size limits the number of samples to be shown to the network before a weight update can be performed. This same limitation is then imposed when making predictions with the fit model.\n",
    "\n",
    "Specifically, the batch size used when fitting your model controls how many predictions you must make at a time.\n",
    "\n",
    "This is often not a problem when you want to make the same number predictions at a time as the batch size used during training.\n",
    "## 1st batch learning:\n",
    "- working on the whole dataset to perform learning\n",
    "- Batch gradient descent is a variation of the gradient descent algorithm that calculates the error for each example in the training dataset, but only updates the model after all training examples have been evaluated.\n",
    "-  (+) Fewer updates to the model means this variant of gradient descent is more computationally efficient than stochastic gradient descent\n",
    "-  (-) Model updates, and in turn training speed, may become very slow for large datasets.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 586,
     "status": "ok",
     "timestamp": 1521539730392,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "86CUXRKbPSV-",
    "outputId": "b448d648-dc48-4738-e273-5ac73a0137f4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27636"
      ]
     },
     "execution_count": 26,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 221,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 812,
     "status": "ok",
     "timestamp": 1521555520440,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "k2GqjpPO5N1o",
    "outputId": "83d38db3-8078-4b5a-bab4-49c24c298317"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_1 (LSTM)                (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 80,501\n",
      "Trainable params: 80,501\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding,LSTM\n",
    "from keras.layers.core import Dense,Dropout\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.utils import np_utils\n",
    "from keras.optimizers import SGD,Adam,Nadam\n",
    "from keras import callbacks\n",
    "embed_dim = 100\n",
    "lstm_out = 100\n",
    "\n",
    "\n",
    "modelBatch = Sequential()\n",
    "modelBatch.add(LSTM(lstm_out, input_shape=(X_a.shape[1], X_a.shape[2])))# dropout_U = 0.2, dropout_W = 0.2)\n",
    "modelBatch.add(Dense(1,activation='sigmoid'))\n",
    "modelBatch.compile(loss='mean_squared_error',\n",
    "              optimizer='adam',\n",
    "              metrics=['binary_accuracy'])\n",
    "print(modelBatch.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 751,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 363
      },
      {
       "item_id": 555
      },
      {
       "item_id": 990
      },
      {
       "item_id": 1519
      },
      {
       "item_id": 2025
      },
      {
       "item_id": 2466
      },
      {
       "item_id": 2947
      },
      {
       "item_id": 3429
      },
      {
       "item_id": 3945
      },
      {
       "item_id": 4478
      },
      {
       "item_id": 5000
      },
      {
       "item_id": 5533
      },
      {
       "item_id": 6072
      },
      {
       "item_id": 6599
      },
      {
       "item_id": 7121
      },
      {
       "item_id": 7654
      },
      {
       "item_id": 8167
      },
      {
       "item_id": 8692
      },
      {
       "item_id": 9183
      },
      {
       "item_id": 9474
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2922385,
     "status": "ok",
     "timestamp": 1521558444690,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "sTPHl8fyOT78",
    "outputId": "6ab0c6a3-2395-449f-d9cd-69a1418e146e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/keras/models.py:942: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "27636/27636 [==============================] - 146s 5ms/step - loss: 0.2447 - binary_accuracy: 0.5749\n",
      "Epoch 2/20\n",
      "  992/27636 [>.............................] - ETA: 2:20 - loss: 0.2411 - binary_accuracy: 0.597827636/27636 [==============================] - 146s 5ms/step - loss: 0.2439 - binary_accuracy: 0.5777\n",
      "Epoch 3/20\n",
      " 2944/27636 [==>...........................] - ETA: 2:09 - loss: 0.2417 - binary_accuracy: 0.588727636/27636 [==============================] - 146s 5ms/step - loss: 0.2438 - binary_accuracy: 0.5773\n",
      "Epoch 4/20\n",
      " 3840/27636 [===>..........................] - ETA: 2:05 - loss: 0.2438 - binary_accuracy: 0.575527636/27636 [==============================] - 146s 5ms/step - loss: 0.2351 - binary_accuracy: 0.6013\n",
      "Epoch 5/20\n",
      " 4256/27636 [===>..........................] - ETA: 2:02 - loss: 0.2049 - binary_accuracy: 0.670827636/27636 [==============================] - 146s 5ms/step - loss: 0.1979 - binary_accuracy: 0.6890\n",
      "Epoch 6/20\n",
      " 4448/27636 [===>..........................] - ETA: 2:02 - loss: 0.1881 - binary_accuracy: 0.711327636/27636 [==============================] - 146s 5ms/step - loss: 0.1839 - binary_accuracy: 0.7177\n",
      "Epoch 7/20\n",
      " 4544/27636 [===>..........................] - ETA: 2:01 - loss: 0.1712 - binary_accuracy: 0.740827636/27636 [==============================] - 145s 5ms/step - loss: 0.1720 - binary_accuracy: 0.7394\n",
      "Epoch 8/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:00 - loss: 0.1653 - binary_accuracy: 0.757427636/27636 [==============================] - 144s 5ms/step - loss: 0.1619 - binary_accuracy: 0.7607\n",
      "Epoch 9/20\n",
      " 4608/27636 [====>.........................] - ETA: 2:00 - loss: 0.1515 - binary_accuracy: 0.776527636/27636 [==============================] - 145s 5ms/step - loss: 0.1531 - binary_accuracy: 0.7756\n",
      "Epoch 10/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:01 - loss: 0.1420 - binary_accuracy: 0.791327636/27636 [==============================] - 146s 5ms/step - loss: 0.1447 - binary_accuracy: 0.7896\n",
      "Epoch 11/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:02 - loss: 0.1359 - binary_accuracy: 0.807527636/27636 [==============================] - 147s 5ms/step - loss: 0.1369 - binary_accuracy: 0.8063\n",
      "Epoch 12/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:01 - loss: 0.1271 - binary_accuracy: 0.817127636/27636 [==============================] - 145s 5ms/step - loss: 0.1296 - binary_accuracy: 0.8160\n",
      "Epoch 13/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:02 - loss: 0.1213 - binary_accuracy: 0.833727636/27636 [==============================] - 147s 5ms/step - loss: 0.1242 - binary_accuracy: 0.8257\n",
      "Epoch 14/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:02 - loss: 0.1135 - binary_accuracy: 0.846427636/27636 [==============================] - 146s 5ms/step - loss: 0.1155 - binary_accuracy: 0.8422\n",
      "Epoch 15/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:02 - loss: 0.1072 - binary_accuracy: 0.854227636/27636 [==============================] - 147s 5ms/step - loss: 0.1091 - binary_accuracy: 0.8515\n",
      "Epoch 16/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:00 - loss: 0.1022 - binary_accuracy: 0.861727636/27636 [==============================] - 146s 5ms/step - loss: 0.1022 - binary_accuracy: 0.8616\n",
      "Epoch 17/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:03 - loss: 0.0964 - binary_accuracy: 0.872627636/27636 [==============================] - 148s 5ms/step - loss: 0.0965 - binary_accuracy: 0.8719\n",
      "Epoch 18/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:03 - loss: 0.0811 - binary_accuracy: 0.895327636/27636 [==============================] - 147s 5ms/step - loss: 0.0902 - binary_accuracy: 0.8821\n",
      "Epoch 19/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:01 - loss: 0.0785 - binary_accuracy: 0.903827636/27636 [==============================] - 146s 5ms/step - loss: 0.0838 - binary_accuracy: 0.8929\n",
      "Epoch 20/20\n",
      " 4576/27636 [===>..........................] - ETA: 2:02 - loss: 0.0749 - binary_accuracy: 0.905227636/27636 [==============================] - 147s 5ms/step - loss: 0.0792 - binary_accuracy: 0.8992\n"
     ]
    }
   ],
   "source": [
    "historyBatch = modelBatch.fit(X_train, y_train, nb_epoch = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 24
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7430,
     "status": "ok",
     "timestamp": 1521558462177,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "Rv7ifTkXOfbs",
    "outputId": "0a388a8f-c5a6-4ffa-d411-6ef1afaefcde"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4878/4878 [==============================] - 7s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "scoreBatch = modelBatch.evaluate(np.array(X_val),y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 555,
     "status": "ok",
     "timestamp": 1521558463695,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "V2poPysdbv5W",
    "outputId": "841f8d30-517a-4cce-8d47-990a2169ec87"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.1689953310548795, 0.775932759278717]"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scoreBatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 298,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 927,
     "status": "ok",
     "timestamp": 1521558480344,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "dlEaw1qBWaag",
    "outputId": "60088f1f-0b1c-4c14-e890-abe9db4b045b"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEZCAYAAABrUHmEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmYFMX9x/F39XJfioDCiheK44EX\nlyByGC88Ei+oqGh+XvFCQzyiqFFEUTRqVBSvRKMSjRZRFDxRA0IUFBGIB45HjNeKoiA3InT9/uje\nOG6WZdjZmZ7d/byex4eZ7q6Zz3Y2fOmq7irjvUdERCQbQdIBRESk9lDREBGRrKloiIhI1lQ0REQk\nayoaIiKSNRUNERHJmoqGSA1LpVIDUqmUT6VSbbM83qdSqUHr2Tc1lUrdXrMJRapPRUNERLKmoiEi\nIllrkHQAkUJLpVIeOBE4C9gLeAcYDPwOGAKsAH6bTqfHx8d3AG4F+gPNgVeBYel0en68vytwD7Az\n8Dbwlwrf1wEYA/QFWgD/BM5Jp9MfViP7/wEXAtsDC4G7gOvS6bRPpVLtgDuA/YAmwHzgonQ6PSWV\nSgXA6Pjn2wwoA25Op9NjNzaD1G+60pD66jfACcB2QHtgGjAd2ByYSPSXfLnHgYZERaEU+BqYmEql\ngvgv48eAN4C2wEnA0Arf9SSwCkgBHYDP4s/cKKlUaiBRkbgQaEVU+C6J/wS4Jt7eCdgUeAD4ayqV\nagAcC/wK2Jeo8P0fcG0qldptY3NI/aaiIfXVo+l0+uN0Ov0VMANYnE6nH0mn02uASUD7VCrVIpVK\n7QH0An6XTqcXpdPppcClwA5Ad6AHsC0wKp1Or4qvPu4t/5L4KqRH3H5JOp1eRvSX/q6pVKr7RmY+\nA3gsnU4/n06n16bT6WnA34kKAkSFYg2wMt5/O9AxnU6vjfetA5an02mfTqdfAVqn0+m3NjKD1HMq\nGlJffZbxeiXwRYX3EHXxdALWZHYlpdPpT4n+ct4e6Bjv/zyj/TsZr3eM//wklUqtTqVSq4GvgJCo\n2GyMTsC7FbZ9GOcAuA7YE/gilUr9NZVKHQeUxPv+BnwMfJpKpZ5KpVK/ISokIhtFYxpSX4UbeF+u\nMWAq2W4AH++v+I+vzPer4s9unk6n11UjZ8UslfEA6XT6zVQq1Qk4ADiMqIttaCqV6p9OpxcDfVOp\nVA/gcKLxnMtSqVTPdDr9SY65pB7RlYZI1T4CGqZSqVT5hlQqtSPRGMcHRFcoDVKpVGlGm8xxgg+I\n/n+2e0Z7k0qltq1mlopjEF3i7yCVSm0KkE6nn02n0+cAewN9gD1SqVTjVCrVMp1Oz0qn0yPiPMuB\nY6qRQ+oxFQ2Rqr0BvAVcl0qlNkmlUq2JuoHmAW8CrwHfApemUqmmqVSqC9FgOADpdPpdYCpwcyqV\nKk2lUk2Ay4EZ8euNcR9wdCqVOiCVSjVIpVIHAEfH2wFmAqNSqVTzeIB+b+B74BOiq47H4zu5AHYC\nWgPvb2QGqedUNESqkE6nPfALoq7cD4luY10DDIwHlFcDPye6K+lbor/AR1f4mBPife8BC4B+wMFx\n243J8hjRIPoYYDFwM3BaOp2eEB8yGOgJfAl8B5wPHJ1Op78FLiYaS3krlUqtJLp769p0Ov3UxmQQ\nMVq5T0REsqUrDRERyZqKhoiIZE1FQ0REsqaiISIiWavrD/dplF9EpHoqe6i1MEXDWnsz0fw9Hhjm\nnJuVsW8/olsU1wFp4DSiWxLH8+N0DG8558611m4FjCOaGuFL4ETn3PdVfXdZWVm1c5eWlubUPt+U\nLzfKlxvly00x5ystLV3vvrx3T1lr+wOdnXO9gVP56eyhEE0pPcg51wdoCQyMt7/snBsQ/3duvO0q\nYKxzri/RPfOn5Du/iIj8qBBjGvsDTwA45+YDra21rTL2d3POlU/2thBoU8VnDSCathqimUgPqNmo\nIiJSlUJ0T7UHZme8XxhvWwrgnFsKYK3tABxENMXCbsAu1tqJRAvGjHTOvQA0z+iO+ppobYIqVXWZ\nlY1c2+eb8uVG+XKjfLkp9nyVSWIg/H8GV6y1mxNdOZztnPvWWvsBMBJwRNNBT7HW7rChz6mMxjSS\no3y5Ub7cKF/1VVXMClE0yoiuLMqVEg1iAxB3VT0LXOacmwzgnPsCeDQ+5CNr7QJgS2C5tbapc25V\n/L44z7iISB1ViDGNycAgAGttV6DMObcsY/9NwM3OuefKN1hrh1hrL4xftwe2IJqC+kV+nMr5GOA5\nRESkYAoyYaG19jqi22hDovWT9wKWAM8TzdY5I+Pwh4lWGXuYaGWxRkRjGs/E4x4PEq2o9glwsnPu\nhyq+2qt7KjnKlxvly43yVV/cPZXccxrOueEVNs3LeL2+1ch+XsnnfAkcWFO51sevW4efMI7FDUoI\nPdCsBTRrjmnWHJo2h+Ytoj/j96akZIOfKSJSF9T1J8KrZ/lS/EuTWL72pxcx670ma9L0vwWEZs2h\nWQtM0+bQoAEYAyaAoPzPAIxh2r8/o98O20b7K+zDBFBSgtmrN2Mfe4JjjjmGDh02eKOYiEjeqWhU\nwmzSmuDGB2jXMGDhJ/+BlcvxK1fAyhWwajmsiP7877aVy6M/F38DZZ+C91XOX7Lgh5B/fPM9ff89\np8ocftLfOPuwX2LatavRn09EpLpUNNbDNG9Bo9JSTKNm0fss2/kwhNUroyISroMwBO8h9OCj17fd\ndDPvLfqYAz9azgG9erJg4TfccN653PDAX1m4eDGrv1/Dr/rvQ6/5Mzlv7J849+WXmL7lTqxs2JjP\nPvuMsrIyhg4dylFHHZW/EyAiUol6XTTC8X/Bz35lvfvLSkpYt27dRn2m6daHYPDJ0TjIehx70ilM\nmDCB7bbbjk8//ZQxf/ozixcvpvt++zNw4EDKysq48sor2eeW2zGnngxffYH/96d81bo9o++4m1lz\n5zFx4kQVDREpuHpdNIrBTjvtBEDLli1Jp9M89dRTBEHA0qVLMc1awBZbEvx8INz/J7osWUA48je0\n3f8YVqxYkXByEamP6nXRCAafDINPXu/+QtwS17BhQwBefPFFli5dypgxY1i2bBlnnHHGf48xnVKY\n/ofQ4JP34av3Ce+/Ff9Dc8KVy/OaTUSkIi3ClABjzP90ey1dupQOHToQBAHTpk1j7dq1P23ToAGm\na2+C4dfD5h3w3y1iwVm/xP9rFiIihaKikYBtttmGDz744CddTP369WPGjBmcf/75NGnShHbt2vHA\nAw/8T1vTKUVwxkWYtpuz7rtvCW+7mvBPN+GXLSnkjyAi9VRBnghPUJ1+IrztmpV8deMV8PH70KIV\n5rjTMT36Yky293rlV7GfP+XLjfLlppjzVfVEuK40arFG2+5AMPx6zOBTYM1q/J9uJBx7DX7RN0lH\nE5E6SkWjljNBCcFBRxKMuA1Su8G81wmvPIdw2nPRMyMiIjVIRaOOMJt3ILhgFOZX5wDgx92Bf+rR\nDbQSEdk4Khp1iDGGoO9BBCPHQrMW+GnP48ONezhRRKQqKhp1kGndBtNjX1iyCN77V9JxRKQOUdGo\no0yvAQD4mVMTzSEidYuKRl21/c7Qdgv8mzPw369OOo2I1BEqGnWUMSa62vh+NX7ua0nHEZE6QkWj\nDjN7DwDAz5ySbBARqTNUNOow035L2G5HeGcufunipOOISB2golHHmb0HgA/xr09LOoqI1AEqGnWc\n6dkXggA/8+Wko4hIHVCQ9TSstTcDvQAPDHPOzcrYtx8wGlgHpIHTnHOhtfYPQN8442jn3OPW2vuB\nbsC3cfMbnHNPF+JnqK1My01g167w1hv4Lz/DdNgq6UgiUovl/UrDWtsf6Oyc6w2cCoypcMg9wCDn\nXB+gJTAwLiRd4jYDgVsyjr/EOTcg/k8FIwum936AntkQkdwVontqf+AJAOfcfKC1tbZVxv5uzrnP\n49cLgTbANGBwvO07oLm1tqQAWesks0dPaNIU/9rLmsRQRHJSiO6p9sDsjPcL421LAZxzSwGstR2A\ng4DLnXPrgPIVik4FnnHOrbPWApxjrT0f+Bo4xzlX5Tzg8bzw1ZZr+3zLNt+ivgew4oVJtPnua5p0\n6ZrnVD+qK+cvKcqXG+WreUmsEf4/C3tYazcHJgFnO+e+zdh+BFHROCjeNA741jk311o7HLgSOKeq\nL6vLizBtTD6/W094YRLfPPV3gs3a5zlZpC6dvyQoX26Ur/qqKmaFKBplRFcW5UqBL8vfxF1VzwKX\nOecmZ2w/GLgMGOicWwLgnHsp43MmAnfmMXfdkuoCm7bBv/EK/rjTMQ0bJZ1IRGqhQoxpTAYGAVhr\nuwJlzrllGftvAm52zj1XvsFauwlwA3C4c25RxvbHrLWd4rcDgLfznL3OMEEJZu/+sGoF/OuNpOOI\nSC2V9ysN59yr1trZ1tpXgRAYaq09CVgCPA/8CuhsrT0tbvJw/GdbwMXjGMTH3Q48aq1dCSwHTs53\n/rrE9BqAf/5xwplTKem2T9JxRKQWKsiYhnNueIVN8zJeN15Ps3sq2fYp0KNGQtVDpuO20HG76JmN\n5UsxLVptsI2ISCY9EV7PmF4DYN1a/BuvJB1FRGohFY16xvTsB8Zo5lsRqRYVjXrGtG4DO+0OH72H\nX7gg6TgiUsuoaNRDWgpWRKpLRaMeMl17Q6NG+JlT8d4nHUdEahEVjXrINGmG2bMXfF0G//kg6Tgi\nUouoaNRTplc88+0MDYiLSPZUNOqrXfaElpvgZ03Hr12bdBoRqSVUNOopU1IS3X67fCm8MyfpOCJS\nS6ho1GP/vYvqtamJ5hCR2kNFoz7bZgdovyV+7mv4lSs2fLyI1HsqGvWYMSYaEP9hDX7OjKTjiEgt\noKJRz5me/QA96Cci2VHRqOdMu/awwy6Qfgu/qMqVc0VEVDQETO8B4D3+9ZeTjiIiRU5FQzDd9oUG\nDTStiIhskIqGYJq3gN17wBefwOf/STqOiBQxFQ0BINh7AIDW2RCRKqloSGS37tCsBf61afhwXdJp\nRKRIqWgIAKZhQ0z3fWHJInjvraTjiEiRUtGQ//pxcSZ1UYlI5RoU4kustTcDvQAPDHPOzcrYtx8w\nGlgHpIHTnHNhZW2stVsB44AS4EvgROfc94X4GeqFHXaGNpvj35yJH7Ia07hJ0olEpMjk/UrDWtsf\n6Oyc6w2cCoypcMg9wCDnXB+gJTCwijZXAWOdc32BD4FT8p2/PommFRkA36/Cz30t6TgiUoQK0T21\nP/AEgHNuPtDaWtsqY38359zn8euFQJsq2gwAJsbHTgIOyHv6ekbrh4tIVQrRPdUemJ3xfmG8bSmA\nc24pgLW2A3AQcDlRd1VlbZpndEd9DXTY0JeXlpbmFD7X9vlW4/lKS/lqx11Y8+5ctmjamJLWbXL8\nuHp2/mqY8uVG+WpeQcY0KjAVN1hrNye6cjjbOfettXaDbdaz7X+UlZVtdMBypaWlObXPt3zlC7v2\ngfff5cvHHyY45Jhqf059PX81Rflyo3zVV1UxK0T3VBnRVUK5UqJBbADibqdngd875yZvoM1ya23T\neNuW8XFSw0zPftC0Of7Jv+LnvZ50HBEpIoUoGpOBQQDW2q5AmXNuWcb+m4CbnXPPZdHmRaD8n77H\nAJltpIaYlpsQnHs5lDQgvOt6/Ltzk44kIkXCFGKCOmvtdUA/IASGAnsBS4DngcVA5gpADzvn7qnY\nxjk3Lx73eBBoAnwCnOyc+6GKr/bqnqo+/+5cwtuuhiAg+O1ITOddNqp9fT9/uVK+3Chf9cXdU5UO\nARSkaCRIRSNHft7rhHeOhkaNCc6/GrNt56zb6vzlRvlyo3zVV1XR0BPhUiWzR0/MqRfA6tWEt1yJ\n1yy4IvWaioZsUNBjX8z/nQsrlhHefAV+wRdJRxKRhKhoSFaCPvtjjj8Tln5H+MfL8d98lXQkEUmA\nioZkLdjvUMygk2DxN1Hh+O7bpCOJSIGpaMhGCQ4+GnP4sbBwAeEfr8AvW5J0JBEpIBUN2WjmF8dh\nDjoSvvwsKhwrlicdSUQKREVDNpoxBjPoZEz/gfD5x4S3XolfvTLpWCJSACoaUi3GGMzxZ2J67Qcf\nv0942yj891raRKSuU9GQajNBgDnpN9BtH3j/bcK7RuN/qOoBfRGp7VQ0JCempITgtAtgt+7w9puE\nf7oBv25d0rFEJE9UNCRnpkFDgjMvhp12hzkz8ffdgg9VOETqIhUNqRGmUWOCoZfB9jvhX38Z/9c7\nqePzmonUSyoaUmNMk6YEv7kCtt4eP30yi8eO1hWHSB2joiE1yjRrQfDbkdBxO1Y8+zjhndfj1+iu\nKpG6QkVDapxp2Yrgd9fSeI8eMHdmNOXI8qVJxxKRGqCiIXlhmjWn3cgxmJ794aP3CK+/WJMcitQB\nKhqSN6ZhQ8yp52EOPgoWfEF43UX4Tz9KOpaI5EBFQ/LKBAHBoJMxx/46mlb9D5fi352TdCwRqSYV\nDSmIYP+fE5xxMaxbSzjmKsIZU5KOJCLVoKIhBWO67UNw3lXQuAn+vpsJnxmvZzlEapkGhfgSa+3N\nQC/AA8Occ7My9jUB7gZ2dc51j7edCpyY8RHdnXMtrLVTgebAinj7Bc652QX4EaSGmB13Jbj4+mhm\n3AnjYPG3cNyvMUFJ0tFEJAt5LxrW2v5AZ+dcb2vtzsB9QO+MQ24A5gK7lm9wzt0L3JvR3mYcf7Jz\n7u1855b8MaVbEwy/gXDMSPzUZ/BLFhGcdgGmUeOko4nIBhSie2p/4AkA59x8oLW1tlXG/kuBCVW0\nvwK4On/xJAmmdRuC343+73xVepZDpHYoRNFoDyzMeL8w3gaAc27Z+hpaa3sAnznnFmRsvspaO81a\ne7e1tmmNp5WCMc2aEwwbgenZT89yiNQSBRnTqMBsxLGnAfdnvL8V+Jdz7iNr7Z3AUODGqj6gtLR0\nowPWZPt8qwv5/OU3suT+21j22Dj4w3DajhxDo+1TBUhXN85fkpQvN8WerzKFKBplZFxZAKXAl1m2\nHQCcW/7GOZfZjTUJ+OUGv7ysLMuv+l+lpaU5tc+3OpVv4GBMg8aE7l6+uug0grOGY3bZq3jyJUD5\ncqN81VdVMStE99RkYBCAtbYrUFZVl1Q5a20psNw5tyZ+b6y1L1prN40PGQBoQLwOCQ74BcEZF8Ha\n+FmO6ZN1S65Ikcl70XDOvQrMtta+CowBhlprT7LWHgVgrR0PPAKkrLVTrbXHx007AF9nfI4H7gFe\nstZOA7YCxuY7vxSW6dYnfpajKf7B26MFnVavSjqWiMRMHf+XnFf3VHJyyee/+Yrwnhvg4/ehfUeC\nMy/GbLlN0eQrBOXLjfJVX9w9Ven4s54Il6Jk2m5BcNFozAFHwILPCa+5QN1VIkVARUOKlmnQkOCX\np0bLyDZsFHVX3ftHdVeJJEhFQ4qe2XNvgituge12xL/2MuE15+M//zjpWCL1Ula33MbTfxzmnLvR\nWtsFuAsIieaR0jzXknemzeYEF43GTxiHn/wE4bW/wxx7GqbvwRizMY/+iEgusr3S+AtQ/k+7scCz\nwDXo7iUpINOgIcHgUwjO+T00aowfdwf+zzfhV69MOppIvZFt0djEOfeYtXZzYA/geufc80DL/EUT\nqZzZoyfB5bfA9jvhX59GePX5+M/UXSVSCNkWDW+tbQYcC0x2zq211jYENC2pJMK0aUdw4bWYg4+G\nr8sIr72QcOqzurtKJM+ynUbkDuBzonGM/eNtD1H17LQieWUaNMAMOgm/466E992Cf+hOeP9tOHEo\npmmzpOOJ1ElZXWk4524HtgbaO+fmxZuvcs5dnLdkIlkyu/eI7q7aYWf8rOmEo87Df/pR0rFE6qSs\nikZ899SZcbdUF2vtP4E7rLV75jeeSHbMZu0ILrgGc8gx8PWXhKN/R/jik/gwTDqaSJ2S691Td+Qj\nlEh1mAYNCI7+P4LfjIAmzfCP3kt4wyX4BV8kHU2kztDdU1LnmN26EYy8DdOtD3w4n/CqYYTPP44P\n1yUdTaTW091TUieZVq0JzryY4Mzh0KQp/u/3E153Mb7s06SjidRquntK6jTTbR+CHbvgH/1TNAXJ\n1b/FHH5sdKuuiGw03T0ldZ5p2YrgtAuiiQ+bt8I/8VfC0Rey5t/vJx1NpNbJdu4pA/wCOCge1/gK\neBqtnCe1iNlzb4LOu+LH34t/5SW++u2JmEMHR/81aJh0PJFaIdsxjRuAYcCbwN+AecAl1toR+Qom\nkg+meQuCk4YRDBtBSeu2+EmPEI46H//Jh0lHE6kVsh3TOATo5pxbXb7BWvsn4HVgZD6CieST6dKN\n9nc+Stlt1+GnPUd47YWYg4/G/PxYTMNGSccTKVrZXmmUAN9X2LZqI9qLFJ2gWQuCE88mOP9qaN0W\n/+zfCa8+D//Re0lHEyla2V5pTAUmWmvvARYDbYDTgCl5yiVSMGbnPQiuvA3/+IP4KU8TXj8cc+Av\nML8Ygmmsu8pFMmVbNH4DnAdcCLQjGgh/CrgtT7lECso0aYo5/gx89z6ED9yGn/wEfs5MguPPxHTp\nmnQ8kaJRZdGw1n4AlM81nbk8WkfgTOAMYMcNfYm19magV/xZw5xzszL2NQHuBnZ1znWPtw0AxgPv\nxIe95Zw711q7FTCOqLvsS+BE51zFbjORajM7diG4Ygx+4sP4F58kvPVKTI++mF+ehtmkddLxRBK3\noSuN03L9Amttf6Czc653PPHhfUDvjENuAOYCu1Zo+rJzblCFbVcBY51z46211wKnAHfmmlEkk2nc\nGDP4ZHyvAYTjxuJnTce//Sbm6BMx/QZiAg3lSf1VZdFwzr1cA9+xP/BE/HnzrbWtrbWtnHNL4/2X\nEo2RDMniswYQXeEATCLqLlPRkLwwW21HMPx6/MvP4yc8iH/oLvyMKQQnno3puF3S8UQSke2YRi7a\nA7Mz3i+Mty0FcM4ts9a2qaTdLtbaicBmwEjn3AtA84zuqK+BDhv68tLS0lyy59w+35QvN1nlG3Ia\n6w45ksV/uolV014gvPp8Wh55PK2GnE7QpGny+RKkfLkp9nyVKUTRqMhs+BA+IHr+wwGdgCnW2h2q\n8TmUlZVtXLoMpaWlObXPN+XLzUbnO/Fcgr32IXzoLpY9Po5lU5+LBsr36FEc+QpM+XJTzPmqKmaF\nKBplRFcW5UqJBrHXyzn3BfBo/PYja+0CYEtgubW2qXNuVfy+OM+41FmmSzeCK2/HP/0ofvIEwtuv\nhq69CX75a8xmbZOOJ5J3hRjRmwwMArDWdgXKnHPLqmpgrR1irb0wft0e2AL4AngROCY+7BjguXyF\nFlkf07gxwdG/Irj8VthhF3hzBuEVQwlfmqQ1O6TOM977DR+VI2vtdUA/oqnVhwJ7AUuccxOsteOB\nrYjunpoN3EM0yP0wsCnQiGhM4xlrbQfgQaAJ8AlwsnPuhyq+2qt7Kjn1IZ8PQ/wrL+L/fj+sXA7b\n7BANlG9TsTc1mXz5pHy5KeZ8cfdUpUMABSkaCVLRSFB9yueXfocf/xf8zClgAszPDsMcMQTTtFlR\n5MsH5ctNMeerqmjohnORGmBabUpw6nnRPFbt2uNfmkR4+VmEr/4DH4ZJxxOpMSoaIjUomsdqDObn\nx8HKFfi/3EJ4/cX4jz9IOppIjVDREKlhpmEjgl8cR3D1HZhufeDfacLRF0ZzWi39Lul4IjlR0RDJ\nE9Nmc4IzLya4YBSUbo3/5wuEvz+T8IUn8WvXJh1PpFpUNETyzOy0O8Hlt2COPwNMgHf3El41DP/O\nnKSjiWw0FQ2RAjAlJQT7HUYw6i7MgENgwReEt4xg3dhr8AsXJB1PJGsqGiIFZFq2IhhyFsHv/wid\nd4G5r0UPBk74K/771Rv+AJGEqWiIJMBs3Yngd6Mxp/8OWm6Cf8YR/v4swtdepo4/OyW1nIqGSEKM\nMQQ9+kZ3WR1mYflS/J9vIvzDJfhPP0o6nkilVDREEmYaNyE48gSCq8bCXr3gw3cJR53PojGj8N8t\nSjqeyE+oaIgUCdOuPSVnX0pw3lXQviMrnn+C8LIzCJ98CL96ZdLxRAAVDZGiY3bZk2DEGFqfexk0\nbYZ/6lHCS88gnPKMnu+QxKloiBQhU1JCi4FHEVxzN+aI42HNGvzDdxGOOAf/5qsaLJfEqGiIFDHT\nuAnB4ccSXHsXZsCh8M0Cwjuvi+az+vDdpONJPaSiIVILmFatCYacSTDydujaGz56j/D64ay741r8\ngs+Tjif1SBJrhItINZn2HSk56xL8h/MJ//4XmDOTcN7rmH4HY35+LKZV66QjSh2nKw2RWsjssDPB\nxdcTnHUJtOuAn/psNFg+6RH86lVJx5M6TEVDpJYyxmC69ia48jbMkDOhUWP8xIejmXSnPYdfp/XK\npeapaIjUcqZBA4IBhxJcezfm8GNh1Ur8uDsIL/11dOWx6JukI0odojENkTrCNGmGOeJ4fP+B+Gcc\n/tUp+IkP4yc9Art3J+h7MHTpiikpSTqq1GIqGiJ1jNl0M8zxZ+KP/j/8rOn4ac/DvNcJ570Ordti\n9j0A0+dATJt2SUeVWqggRcNaezPQC/DAMOfcrIx9TYC7gV2dc90ztv8B6BtnHO2ce9xaez/QDfg2\nPuwG59zThfgZRGob06Qppu9B0Pcg/Kcf4adPxs+cip/0CP4pB126EvQ7CHbroasPyVrei4a1tj/Q\n2TnX21q7M3Af0DvjkBuAucCuGW32A7rEbdoAc4DH492XOOeeyndukbrEbL09ZshZ+EEn/3j18dYb\nhG+9AZtuhulzAGbfAzFtt0g6qhS5QgyE7w88AeCcmw+0tta2yth/KTChQptpwOD49XdAc2ut/ikk\nkiPTuAnBvgdScumNBFfcitnvUPh+Nf5pR3jp6ay79Ur8mzM0x5WsVyG6p9oDszPeL4y3LQVwzi2L\nryb+yzm3DlgRvz0VeMY5t85aC3COtfZ84GvgHOdclbeGlJaW5hQ+1/b5pny5qdf5Skth7z6Eq1ez\navoLLH9uAmvefpPw7TcJWrehxVFDaHnk8ZiS9f81Ua/PXw0o9nyVSWIg3GR7oLX2CKKicVC8aRzw\nrXNurrV2OHAlcE5Vn1FWVlbNmNH/oLm0zzfly43yZdi1G+zajeDz/+CnTyacOYUl941hyUvPEJx0\nLqbjdsnmqwblq76qilkhikZDsKUxAAASwUlEQVQZ0ZVFuVLgyw01stYeDFwGDHTOLQFwzr2UcchE\n4M4azClS75mO22KOOx3/i+Pwj96Ln/EPwlHnYw4ZhDnUYho2TDqiJKwQYxqTgUEA1tquQJlzbllV\nDay1mxANkB/unFuUsf0xa22n+O0A4O28JBap50zzlgSn/JZg2AjYpHW0pseo8/Afv590NEmYKcS8\n/Nba64B+QAgMBfYCljjnJlhrxwNbEd09NRu4B2hB1PWU+Rv6K2B74A/ASmA5cLJz7usqvtqreyo5\nypebYsnnV63EP/4AfuqzYALMgUdgjjieLbfdrijyrU+xnL/1KeZ8cfdUpUMJBSkaCVLRSJDy5abY\n8vn0W4QP3AYLF8DmHWh3wUgWbdZ+ww0TUmznr6JizldV0dDcUyKSFZPajWDEbZiDjoSFX7Hw4tMJ\nH7pL65fXMyoaIpI107gxweBTCIZfT4OtO+GnPkM44lz8O3OSjiYFoqIhIhvNdErRfsxfMYdZWLKI\n8JYRhPffil+xPOlokmcqGiJSLaZhI4IjTyC49CbYuhP+lZcIRwzFz5mZdDTJIxUNEcmJ2boTwSU3\nYo46EVYsI7zjWsK7/4BfujjpaJIHmhpdRHJmGjTAHDoYv1cvwgduw7/xT/y/ZmH6DcQcdCSmdZsN\nf4jUCrrSEJEaYzpsRXDRaMzxZ0KzFvgXn4xWEBw3Fv/1BieCkFpAVxoiUqNMUILZ71D8vgfiZ07B\nP/cYftrz+OkvYHr0xRxyDKbjtknHlGpS0RCRvDANG2L6HoTvsz9+9qv4Z8bjX38Z//rLsEdPgkMH\nYzqlko4pG0lFQ0TyygQlmB598d33jRZ+emb8j8vP7rQ7waGDYafdMSbrCbAlQSoaIlIQxhjYvQfB\nbt3h/Xei4vHuHML3/gXb7RgVj917YAINtRYzFQ0RKShjDKS6UJLqgv/PB1HxmDOTcOw1sOU20TTs\n3ffVuuVFSiVdRBJjtu1MydmXEoy8HdNrP/jyM/yfbyK8/CzC6ZO17GwRUtEQkcSZ0q0JTj2PYNRd\nmAGHwOJv8Q/eTnjF2YSv/gMfrks6osRUNESkaJh27QmGnEUw+h7MfofBom/wf7mFcMS5hLOm48Mw\n6Yj1noqGiBQds2kbguPPILjmbkzfg+DrMvw9NxBe/Vv8nJnU8XWAipqKhogULdOmHcGvziG4+k5M\n7/3gi0+jua2uuQD/1mwVjwSoaIhI0TObdyA45bxowLxHX/jkQ8IxIwmvvxg/f17S8eoVFQ0RqTVM\nh44Ep/+OYMStsGcv+Og9wj9ezrobL8N/8G7S8eoFPachIrWO6bgdJUMvjZ7zePJheHs24R+Gw657\nERxxAma7zklHrLNUNESk1jLbdqZk2Aj8h/MJn3wI3plD+M4c2KMna04+F5pvknTEOqcgRcNaezPQ\nC/DAMOfcrIx9TYC7gV2dc92ramOt3QoYB5QAXwInOue+L8TPICLFy+ywMyUXjMKn3yJ84iGY9zpf\n/fZE2H4nzH6HYbrtg2nQMOmYdULexzSstf2Bzs653sCpwJgKh9wAzM2yzVXAWOdcX+BD4JR8ZheR\n2sWkdiO4aDTBeSNp0r0P/DsdPWE+/DTCiQ/jv/s26Yi1XiEGwvcHngBwzs0HWltrW2XsvxSYkGWb\nAcDE+JhJwAH5iy0itZExBrPLXrQbeSvBqDsxBx4Ba9bgJz0SFY97bsB/+K5u162mQnRPtQdmZ7xf\nGG9bCuCcW2atrbgW5PraNM/ojvoa6LChLy8tLa1m7Jppn2/Klxvly02x59tyz+6wZ3fCMy9k5ZRn\nWT7pUX6YNR0/azoNO+1Ii8MtzfoPJGjSJJF8xX7+KpPEQHh1Js2vrE1Wn1NWVlaNr4uUlpbm1D7f\nlC83ypebWpdvj1743fcmeP8dwilP8cOcmSweM4rF996K2fcATP9DMO3aJ5eviFRVzApRNMqIrhLK\nlRINYlenzXJrbVPn3Cpgy/g4EZGs/GRa9kXf4Kc9Fy1F+/wE/OQnovU+fnYY7LSH1vVYj0KclcnA\nIABrbVegzDm3rJptXgSOiY85BnguL4lFpM4zm7UlOPIEguvvw5x6HmzbOVpR8OYRhCOGEk55Gv+9\nbs6syBRiMMhaex3QDwiBocBewBLn3ARr7XhgK2BXonGMe5xzD1ds45ybZ63tADwINAE+AU52zv1Q\nxVd7dU8lR/lyo3y5qU4+//EH+ClP4WdNh7VroUUrzP4/j27bbd4i8XyFEndPVToEUJCikSAVjQQp\nX26ULze55PNLv8P/4yn8lKdh5Qpo3ATT72DMAUdgNmubeL58q6po6IlwEZEKTKtNMUeegB94dDTm\n8cJE/AtP4v/xNKZXf8zBR2M6bJV0zESoaIiIrIdp0gxz0FH4nx2Of+1l/HOP4V95Cf/KS7BnL4KB\nR2O23ynpmAWloiEisgGmQUNMnwPwvX8Gc18jfO4xmDuTcO5M2LELwcBjoEvX6O6sOk5FQ0QkSyYI\noGtvgr16wftvR8Xj7TcJ338bOm6LGXgMpvu+mJKSpKPmjYqGiMhGip732I2S1G74T/+Nf/5x/Kx/\n4v98E37COMzBR2H2OQDTuHHSUWucnl4REcmB2boTwa8vJLjmLsyAQ2Hpd/iH7ya85DTCZ/+OX7Uy\n6Yg1SkVDRKQGmHbtCYacSXDdnzGHDoa1P+Aff5Bw+KmETz6EX7406Yg1QkVDRKQGmVabEhx1IsF1\n92KOPAGCEvxTj0Yz7I6/D//doqQj5kRFQ0QkD0yz5gSH2ejK45enQtNm+MlPEF7ya8KH7mTtV8X5\nYN+GaCBcRCSPTOMmmAOOwPc/FD/jJfxzj+OnPsuX0ydjevbHHDoI075j0jGzpqIhIlIApmFDTL+B\n+D4H4mdNo+SFJ1k74x/4mVMwXffBHDoYs3WnpGNukIqGiEgBmZISTK/9aH/kcZQ98zjh0+Pxs1/B\nz34FdutOcJgt6qfMVTRERBJgggDTdR+CvXrDO28SPj0e3nqD8K03ILUbwcFHwc57YhoU11/TxZVG\nRKSeMcZAl26UdOmGf//tqHi8O4cw/RY0b4np2hvTfV9I7VYUT5qraIiIFAmzYxdKduyC/+RD/Iwp\n+DdewU+fjJ8+GVpugum2D6Z7X+i8MyZIpoCoaIiIFBmzzQ6YbXbA21Pgg/n4N6bjZ7+Kn/osfuqz\nsMlmmO59MN37QKedCro0rYqGiEiRMkEJpLpgUl3wx54O6bfwb/wT/+YM/EuT8C9Ngs3aYrr1wfTo\nC9t2zvtMuyoaIiK1gCkpgV32xOyyJ/74M2H+PPys6fi5r0ULRL3wJLTdIpplt8e+sFWnvBQQFQ0R\nkVrGNGgAu3XD7NYN/8MP8O4c/OvT8fNejxaKeu6xqHiccVGNf7eKhohILWYaNoQ9emL26Ilf8z28\nPRs/+1XYYsu8fJ+KhohIHWEaNYau+2C67pO37yhI0bDW3gz0AjwwzDk3K2PfAcC1wDrgGefc1dba\nU4ETMz6iu3OuhbV2KtAcWBFvv8A5N7sQP4OIiBSgaFhr+wOdnXO9rbU7A/cBvTMOGQMcDHwBvGyt\nfcw5dy9wb0Z7m3H8yc65t/OdW0RE/lchbu7dH3gCwDk3H2htrW0FYK3tBCxyzn3mnAuBZ+LjM10B\nXF2AnCIisgGF6J5qD2R2IS2Mty2N/1yYse9rYPvyN9baHsBnzrkFGcdcZa1tC8wHfuucW1XVl5eW\nluYUPtf2+aZ8uVG+3Chfboo9X2WSGAiv6sbhivtOA+7PeH8r8C/n3EfW2juBocCNVX1ZWVn1Fzop\nLS3NqX2+KV9ulC83ypebYs5XVTErRNEoI7qiKFcKfLmefVvG28oNAM4tf+Ocm5CxbxLwy5oMKiIi\nVSvEmMZkYBCAtbYrUOacWwbgnPsP0Mpau621tgFweHw81tpSYLlzbk383lhrX7TWbhp/7gBAA+Ii\nIgWU96LhnHsVmG2tfZXoTqmh1tqTrLVHxYecBfwNmA486px7P97egWiMo/xzPHAP8JK1dhqwFTA2\n3/lFRORHxnufdIZ8qtM/nIhIHlU6/lzXnwjP73SPIiL1TOEmYRcRkVpPRUNERLKmoiEiIllT0RAR\nkaypaIiISNZUNEREJGt1/ZbbrGzseh8J5PsD0Jfof6/RzrnHM/b9B/gszgcwxDn3RQGzDQDGA+/E\nm95yzp2bsT/R87e+tVky9v8AvJKxf3/n3DoKwFrbBXgSuNk5d7u1ditgHFBCNNXOic657yu0We/v\naoHy/QVoCPwAnJA5meiGfhcKkO9+oBvwbXzIDc65pyu0SfL8jQfaxbs3A2Y6507POP4kohm9P4o3\nveCcuyZf+aqr3heNaq738W4B8+0HdInztQHmAI9XOOwQ59zyQmWqxMvOuUHr2Zfo+dvA2iwAS5xz\nAwqVp5y1tjlwG/BSxuargLHOufHW2muBU4A7M9ps6Hc13/lGAfc455y1dihwPlBxEeqqfhfynQ/g\nEufcU+tpk+j5c84Nzth/H/DnSpo+6py7MB+Zaoq6p3Jf7yPfpgHlv2zfAc2ttSUFzlAtRXL+MhXT\n2izfA4fyvxN0ToxfTwIOqNBmvb+rBcp3NvBY/Hoh0CZP352NyvJtSNLnDwBrbQrY1Dn3ep6+O6/q\n/ZUGOaz3UQhxV0n58ranEnXxVOw+uctauy3wT6J/aRV6+pRdrLUTiS65RzrnXoi3J37+yq1nbRaA\nJtbah4FtgMecc38sRB7n3FpgrbU/ufBpntEd9TXR/GuZqvpdzXs+59wKgPgfLUOJrowqWt/vQt7z\nxc6x1p5PdP7Occ59k7Ev0fOXYRjRVUhl+ltrnyPqArzQOTenprPlSlca/2tj1vsoGGvtEURF45wK\nu64g6iYYAHQBjilsMj4ARgJHAP8H3GutbbSeY5Oc1qXi2izlLgROBw4ChlhruxcyVBWyOVcFP59x\nwRgH/MM5V7FraGN+F/JhHDDcOfczYC5w5QaOT+L8NQL2dc5NqWT3TOBK59xA4PfAgwUNlyVdaeS2\n3kdBWGsPBi4DBjrnlmTuc849mHHcM8BuwN8LlS0edH80fvuRtXYB0Xn6mCI5f7EBZKzNUs45d1f5\na2vtS0Tn743CxfqJ5dbapvFqlJWdq6p+VwvlL8AHzrmRFXds4Hch7yoUsYlkjAfFiuH89Qcq7ZZy\nzr0HvBe/nmGtbWetLSnUjRnZ0pVGNdf7KBRr7SbADcDhzrlFFfdZa5/P+Ndcfwq8xoi1doi19sL4\ndXtgC6JB76I4f3Gun6zNkrE9Za19OF6rpQHQhx/v/EnCi/x4pXgM8FyF/ev9XS0Ea+0QYI1zbsT6\n9q/vd6FA+R6Lx9Gg8vV2Ej1/sR7AvMp2WGsvstYeF7/uAiwstoIBdX9q9KxYa68D+gEhUV/tXkR3\n1Uyw1vYDro8Pfcw5V+XysnnIdjrRZfb7GZv/QXQ74wRr7TCiroBVRHdWnVvIMQ1rbUvgYWBToBFR\n98TmFMn5izN2A0Y55w6J3w8nustnhrX2euBnRP/bTyzULY5xppuAbYluX/0CGELUhdYE+AQ42Tn3\ng7X2kfj1qoq/q865Sv8CylO+zYHV/DgG8K5z7uzyfEQ9Fz/5XXDOPVPAfLcBw4GVwHKic/Z1EZ2/\no4n+//FP59yjGcc+6Zw7wlrbkaiLLSA6l+cV42C5ioaIiGRN3VMiIpI1FQ0REcmaioaIiGRNRUNE\nRLKmoiEiIllT0RApYtbaAdbaD5POIVJORUNERLKm5zREchDPCTYKaA58CBwP3AgsBvYEdiSaJO9Y\n59xKa+3uRNNbtCF6UO5i59zz8WddDJwBrAWeAi4gesr/z0TTd5xA9NDcaeuZu0gk73SlIVJN8ZQV\n44DjnHOdgClA+VxWRxFNWbEVsAnwa2ttADwC3O6c24loEsW/WWtbWmv3jd/vQTTx5L5xe4CORDMA\n7ExUcH5fiJ9PpDIqGiLVNxCY6pwrn+PoLuAXRCvvPemc+zZeR+QJYB9gO6IJ8x4BcM69QTRdSA+i\ntReeds4ti+fIGsCPi20tdc6Vr7Mxh6iIiCRCs9yKVN+mQD9r7XsZ25YQdT1lDl4vBloTLfX5XYW5\nwRYTzenUloxZbZ1zKwHi9Rgy13tYR1SURBKhoiFSfWXAixWXN43Xqm6bsWkzYBHwFbCZtdZkFI42\n8fZvMtvES/uKFB0VDZHqex643lrbyTn3b2ttT6KZagEGWms3BZYBRxKtM/Ef4HPgl8Aj1tp9iLqr\nXieapdhZay+P2zxBNH6R1PojIpXSmIZINTnnvgR+DUyw1s4HbufHRYheIhqT+JyoC+q++OriWKIl\nSecDY4DBzrkVzrmZROumzAXeBd4E/lbIn0ckG7rlVqSGxd1THzrnRiWdRaSm6UpDRESypqIhIiJZ\nU/eUiIhkTVcaIiKSNRUNERHJmoqGiIhkTUVDRESypqIhIiJZ+39QYDGCXMe16wAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e9e97ccf8>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "%matplotlib inline\n",
    "\n",
    "plt.plot(historyBatch.history['loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fdztrx47EgnK"
   },
   "source": [
    "### Save Batch LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "M3-tep1hEgCh"
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "modelBatch.save('LSTMBatch_Model.h5')  # creates a HDF5 file 'my_model.h5'\n",
    "del modelBatch  # deletes the existing model\n",
    "\n",
    "# returns a compiled model\n",
    "# identical to the previous one\n",
    "modelBatch = load_model('LSTMBatch_Model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 51,
     "output_extras": [
      {
       "item_id": 25
      },
      {
       "item_id": 26
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7523,
     "status": "ok",
     "timestamp": 1521558553637,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "LnVaVLis9K8b",
    "outputId": "d412f2cf-a7ee-4d02-8d2a-e1f22897c0b5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4878/4878 [==============================] - 7s 1ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.1689953310548795, 0.775932759278717]"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modelBatch.evaluate(np.array(X_val),y_val)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "H_ZPjBiD9XxC"
   },
   "outputs": [],
   "source": [
    "del modelBatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "1WNEqVpORO5R"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.download('LSTMBatch_Model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lPsN8yZsets-"
   },
   "source": [
    "### LSTM : 1 Layer Mini batch gradient decent\n",
    "-  100 neurones\n",
    "-  batch size 128\n",
    "-  Mini-batch gradient descent is the recommended variant of gradient descent for most applications, especially in deep learning.\n",
    "-  Mini-batch sizes, commonly called “batch sizes” for brevity, are often tuned to an aspect of the computational architecture on which the implementation is being executed. Such as a power of two that fits the memory requirements of the GPU or CPU hardware like 32, 64, 128, 256, and so on.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 3924,
     "output_extras": [
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {},
      {}
     ]
    },
    "colab_type": "code",
    "id": "x-D2mOzdesc8",
    "outputId": "44f92b57-03a4-4914-95d5-aedc37aabc4e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Update your `LSTM` call to the Keras 2 API: `LSTM(100, input_shape=(100, 100), dropout=0.2, recurrent_dropout=0.2)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_13 (LSTM)               (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 80,501\n",
      "Trainable params: 80,501\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/keras/models.py:942: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "27636/27636 [==============================] - 42s 2ms/step - loss: 0.2447 - binary_accuracy: 0.5763\n",
      "Epoch 2/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.2420 - binary_accuracy: 0.5848\n",
      "Epoch 3/100\n",
      "10496/27636 [==========>...................] - ETA: 25s - loss: 0.2403 - binary_accuracy: 0.600727636/27636 [==============================] - 41s 2ms/step - loss: 0.2437 - binary_accuracy: 0.5816\n",
      "Epoch 4/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.2445 - binary_accuracy: 0.5770\n",
      "Epoch 5/100\n",
      "16512/27636 [================>.............] - ETA: 16s - loss: 0.2439 - binary_accuracy: 0.577827636/27636 [==============================] - 41s 1ms/step - loss: 0.2440 - binary_accuracy: 0.5774\n",
      "Epoch 6/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.2426 - binary_accuracy: 0.5801\n",
      "Epoch 7/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.2287 - binary_accuracy: 0.624327636/27636 [==============================] - 41s 1ms/step - loss: 0.2239 - binary_accuracy: 0.6364\n",
      "Epoch 8/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.2080 - binary_accuracy: 0.6713\n",
      "Epoch 9/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.2026 - binary_accuracy: 0.679027636/27636 [==============================] - 41s 2ms/step - loss: 0.2016 - binary_accuracy: 0.6813\n",
      "Epoch 10/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1962 - binary_accuracy: 0.6962\n",
      "Epoch 11/100\n",
      "18560/27636 [===================>..........] - ETA: 13s - loss: 0.1921 - binary_accuracy: 0.700727636/27636 [==============================] - 41s 1ms/step - loss: 0.1925 - binary_accuracy: 0.7003\n",
      "Epoch 12/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1888 - binary_accuracy: 0.7064\n",
      "Epoch 13/100\n",
      "18560/27636 [===================>..........] - ETA: 13s - loss: 0.1848 - binary_accuracy: 0.717027636/27636 [==============================] - 41s 1ms/step - loss: 0.1850 - binary_accuracy: 0.7166\n",
      "Epoch 14/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1823 - binary_accuracy: 0.7218\n",
      "Epoch 15/100\n",
      "18304/27636 [==================>...........] - ETA: 14s - loss: 0.1781 - binary_accuracy: 0.727527636/27636 [==============================] - 41s 1ms/step - loss: 0.1772 - binary_accuracy: 0.7294\n",
      "Epoch 16/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1743 - binary_accuracy: 0.7364\n",
      "Epoch 17/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1696 - binary_accuracy: 0.743927636/27636 [==============================] - 41s 1ms/step - loss: 0.1709 - binary_accuracy: 0.7419\n",
      "Epoch 18/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1693 - binary_accuracy: 0.7462\n",
      "Epoch 19/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1652 - binary_accuracy: 0.753727636/27636 [==============================] - 41s 1ms/step - loss: 0.1651 - binary_accuracy: 0.7545\n",
      "Epoch 20/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1634 - binary_accuracy: 0.7567\n",
      "Epoch 21/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1593 - binary_accuracy: 0.768627636/27636 [==============================] - 41s 1ms/step - loss: 0.1609 - binary_accuracy: 0.7632\n",
      "Epoch 22/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1580 - binary_accuracy: 0.7651\n",
      "Epoch 23/100\n",
      "18176/27636 [==================>...........] - ETA: 14s - loss: 0.1570 - binary_accuracy: 0.770127636/27636 [==============================] - 41s 1ms/step - loss: 0.1561 - binary_accuracy: 0.7722\n",
      "Epoch 24/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1543 - binary_accuracy: 0.7740\n",
      "Epoch 25/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1518 - binary_accuracy: 0.780427636/27636 [==============================] - 41s 1ms/step - loss: 0.1520 - binary_accuracy: 0.7801\n",
      "Epoch 26/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1499 - binary_accuracy: 0.7788\n",
      "Epoch 27/100\n",
      "18560/27636 [===================>..........] - ETA: 13s - loss: 0.1484 - binary_accuracy: 0.783827636/27636 [==============================] - 41s 1ms/step - loss: 0.1488 - binary_accuracy: 0.7839\n",
      "Epoch 28/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1452 - binary_accuracy: 0.7903\n",
      "Epoch 29/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1440 - binary_accuracy: 0.792227636/27636 [==============================] - 41s 1ms/step - loss: 0.1439 - binary_accuracy: 0.7922\n",
      "Epoch 30/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1436 - binary_accuracy: 0.7940\n",
      "Epoch 31/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1412 - binary_accuracy: 0.796327636/27636 [==============================] - 41s 1ms/step - loss: 0.1409 - binary_accuracy: 0.7960\n",
      "Epoch 32/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1387 - binary_accuracy: 0.8023\n",
      "Epoch 33/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1385 - binary_accuracy: 0.802527636/27636 [==============================] - 41s 1ms/step - loss: 0.1375 - binary_accuracy: 0.8043\n",
      "Epoch 34/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1369 - binary_accuracy: 0.8044\n",
      "Epoch 35/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1359 - binary_accuracy: 0.805127636/27636 [==============================] - 41s 1ms/step - loss: 0.1366 - binary_accuracy: 0.8050\n",
      "Epoch 36/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1351 - binary_accuracy: 0.8078\n",
      "Epoch 37/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1306 - binary_accuracy: 0.814627636/27636 [==============================] - 41s 1ms/step - loss: 0.1317 - binary_accuracy: 0.8117\n",
      "Epoch 38/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1325 - binary_accuracy: 0.8119\n",
      "Epoch 39/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1297 - binary_accuracy: 0.814727636/27636 [==============================] - 41s 1ms/step - loss: 0.1309 - binary_accuracy: 0.8144\n",
      "Epoch 40/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1297 - binary_accuracy: 0.8159\n",
      "Epoch 41/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1281 - binary_accuracy: 0.818327636/27636 [==============================] - 41s 1ms/step - loss: 0.1297 - binary_accuracy: 0.8157\n",
      "Epoch 42/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1281 - binary_accuracy: 0.8186\n",
      "Epoch 43/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1257 - binary_accuracy: 0.823027636/27636 [==============================] - 41s 1ms/step - loss: 0.1261 - binary_accuracy: 0.8211\n",
      "Epoch 44/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1250 - binary_accuracy: 0.8253\n",
      "Epoch 45/100\n",
      "18048/27636 [==================>...........] - ETA: 14s - loss: 0.1236 - binary_accuracy: 0.826927636/27636 [==============================] - 41s 1ms/step - loss: 0.1241 - binary_accuracy: 0.8256\n",
      "Epoch 46/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1233 - binary_accuracy: 0.8256\n",
      "Epoch 47/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1209 - binary_accuracy: 0.829927636/27636 [==============================] - 41s 1ms/step - loss: 0.1217 - binary_accuracy: 0.8282\n",
      "Epoch 48/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1218 - binary_accuracy: 0.8295\n",
      "Epoch 49/100\n",
      "18560/27636 [===================>..........] - ETA: 13s - loss: 0.1212 - binary_accuracy: 0.829727636/27636 [==============================] - 41s 1ms/step - loss: 0.1208 - binary_accuracy: 0.8293\n",
      "Epoch 50/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1191 - binary_accuracy: 0.8324\n",
      "Epoch 51/100\n",
      "18816/27636 [===================>..........] - ETA: 13s - loss: 0.1194 - binary_accuracy: 0.832527636/27636 [==============================] - 41s 1ms/step - loss: 0.1196 - binary_accuracy: 0.8316\n",
      "Epoch 52/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1192 - binary_accuracy: 0.8318\n",
      "Epoch 53/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1165 - binary_accuracy: 0.837527636/27636 [==============================] - 41s 1ms/step - loss: 0.1176 - binary_accuracy: 0.8350\n",
      "Epoch 54/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1162 - binary_accuracy: 0.8377\n",
      "Epoch 55/100\n",
      "18176/27636 [==================>...........] - ETA: 14s - loss: 0.1173 - binary_accuracy: 0.836727636/27636 [==============================] - 41s 1ms/step - loss: 0.1175 - binary_accuracy: 0.8367\n",
      "Epoch 56/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1143 - binary_accuracy: 0.8405\n",
      "Epoch 57/100\n",
      "18048/27636 [==================>...........] - ETA: 14s - loss: 0.1178 - binary_accuracy: 0.834527636/27636 [==============================] - 41s 1ms/step - loss: 0.1167 - binary_accuracy: 0.8368\n",
      "Epoch 58/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1124 - binary_accuracy: 0.8449\n",
      "Epoch 59/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1107 - binary_accuracy: 0.845927636/27636 [==============================] - 41s 1ms/step - loss: 0.1108 - binary_accuracy: 0.8452\n",
      "Epoch 60/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1115 - binary_accuracy: 0.8449\n",
      "Epoch 61/100\n",
      "17920/27636 [==================>...........] - ETA: 14s - loss: 0.1115 - binary_accuracy: 0.846527636/27636 [==============================] - 41s 1ms/step - loss: 0.1117 - binary_accuracy: 0.8456\n",
      "Epoch 62/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1109 - binary_accuracy: 0.8474\n",
      "Epoch 63/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1070 - binary_accuracy: 0.854927636/27636 [==============================] - 41s 1ms/step - loss: 0.1094 - binary_accuracy: 0.8512\n",
      "Epoch 64/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1105 - binary_accuracy: 0.8472\n",
      "Epoch 65/100\n",
      "17920/27636 [==================>...........] - ETA: 14s - loss: 0.1082 - binary_accuracy: 0.850527636/27636 [==============================] - 41s 1ms/step - loss: 0.1090 - binary_accuracy: 0.8494\n",
      "Epoch 66/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1082 - binary_accuracy: 0.8521\n",
      "Epoch 67/100\n",
      "17920/27636 [==================>...........] - ETA: 14s - loss: 0.1066 - binary_accuracy: 0.855127636/27636 [==============================] - 41s 1ms/step - loss: 0.1068 - binary_accuracy: 0.8544\n",
      "Epoch 68/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1081 - binary_accuracy: 0.8539\n",
      "Epoch 69/100\n",
      "17920/27636 [==================>...........] - ETA: 14s - loss: 0.1071 - binary_accuracy: 0.852427636/27636 [==============================] - 41s 1ms/step - loss: 0.1069 - binary_accuracy: 0.8527\n",
      "Epoch 70/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1050 - binary_accuracy: 0.8561\n",
      "Epoch 71/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1041 - binary_accuracy: 0.857327636/27636 [==============================] - 41s 1ms/step - loss: 0.1062 - binary_accuracy: 0.8537\n",
      "Epoch 72/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1048 - binary_accuracy: 0.8560\n",
      "Epoch 73/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1057 - binary_accuracy: 0.854527636/27636 [==============================] - 41s 1ms/step - loss: 0.1049 - binary_accuracy: 0.8559\n",
      "Epoch 74/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1039 - binary_accuracy: 0.8582\n",
      "Epoch 75/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1013 - binary_accuracy: 0.860327636/27636 [==============================] - 41s 1ms/step - loss: 0.1023 - binary_accuracy: 0.8597\n",
      "Epoch 76/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1025 - binary_accuracy: 0.8595\n",
      "Epoch 77/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.1013 - binary_accuracy: 0.860127636/27636 [==============================] - 41s 1ms/step - loss: 0.1026 - binary_accuracy: 0.8577\n",
      "Epoch 78/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1030 - binary_accuracy: 0.8596\n",
      "Epoch 79/100\n",
      "18176/27636 [==================>...........] - ETA: 14s - loss: 0.1022 - binary_accuracy: 0.862527636/27636 [==============================] - 41s 1ms/step - loss: 0.1017 - binary_accuracy: 0.8623\n",
      "Epoch 80/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1020 - binary_accuracy: 0.8592\n",
      "Epoch 81/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1010 - binary_accuracy: 0.862327636/27636 [==============================] - 41s 1ms/step - loss: 0.1010 - binary_accuracy: 0.8612\n",
      "Epoch 82/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1005 - binary_accuracy: 0.8627\n",
      "Epoch 83/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.1020 - binary_accuracy: 0.860127636/27636 [==============================] - 41s 1ms/step - loss: 0.1016 - binary_accuracy: 0.8616\n",
      "Epoch 84/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.1002 - binary_accuracy: 0.8639\n",
      "Epoch 85/100\n",
      "18304/27636 [==================>...........] - ETA: 13s - loss: 0.0987 - binary_accuracy: 0.864927636/27636 [==============================] - 41s 1ms/step - loss: 0.0985 - binary_accuracy: 0.8651\n",
      "Epoch 86/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.0992 - binary_accuracy: 0.8653\n",
      "Epoch 87/100\n",
      "18560/27636 [===================>..........] - ETA: 13s - loss: 0.0978 - binary_accuracy: 0.865827636/27636 [==============================] - 41s 1ms/step - loss: 0.0986 - binary_accuracy: 0.8646\n",
      "Epoch 88/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.0980 - binary_accuracy: 0.8668\n",
      "Epoch 89/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.0972 - binary_accuracy: 0.869127636/27636 [==============================] - 41s 1ms/step - loss: 0.0968 - binary_accuracy: 0.8695\n",
      "Epoch 90/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.0963 - binary_accuracy: 0.8684\n",
      "Epoch 91/100\n",
      "18432/27636 [===================>..........] - ETA: 13s - loss: 0.0971 - binary_accuracy: 0.867727636/27636 [==============================] - 41s 1ms/step - loss: 0.0975 - binary_accuracy: 0.8670\n",
      "Epoch 92/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.0964 - binary_accuracy: 0.8691\n",
      "Epoch 93/100\n",
      "18688/27636 [===================>..........] - ETA: 13s - loss: 0.0918 - binary_accuracy: 0.876427636/27636 [==============================] - 41s 1ms/step - loss: 0.0947 - binary_accuracy: 0.8721\n",
      "Epoch 94/100\n",
      "27636/27636 [==============================] - 41s 1ms/step - loss: 0.0951 - binary_accuracy: 0.8702\n",
      "Epoch 95/100\n",
      "13568/27636 [=============>................] - ETA: 20s - loss: 0.0947 - binary_accuracy: 0.8712"
     ]
    }
   ],
   "source": [
    "lstm_out = 100\n",
    "batch_size = 128\n",
    "\n",
    "model = Sequential()\n",
    "model.add(LSTM(lstm_out, input_shape=(X_a.shape[1], X_a.shape[2]), dropout_U = 0.2, dropout_W = 0.2))\n",
    "model.add(Dense(1,activation='sigmoid'))\n",
    "model.compile(loss='mean_squared_error',\n",
    "              optimizer='adam',\n",
    "              metrics=['binary_accuracy'])\n",
    "print(model.summary())\n",
    "history128 = model.fit(X_train, y_train, batch_size = batch_size, nb_epoch = 100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 298,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 890,
     "status": "ok",
     "timestamp": 1521491968820,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "4-VvZYyp_F9G",
    "outputId": "8066bcda-2b05-4e6c-fa60-a0430b492365"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEZCAYAAAB8culNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmAFMX5//F39bIsNyzLuaIGDxoF\nL/AAL0BQQVGMYkk0QQ0aTSBBc6jxSNRo8PxiiBrF82c8y4h4Ax4IKoiIiBjJJB7xYBVWQO576vdH\nD2TYXdxh2ZnZnfm8/mGmu2vmeRiYZ7qqu8p47xEREUkWZDsAERGpe1QcRESkEhUHERGpRMVBREQq\nUXEQEZFKVBxERKQSFQeRnRCGYd8wDH0Yhm1SPN6HYTh0O/teD8Pw9tqNUKRmVBxERKQSFQcREamk\nQbYDEEmnMAw98BPg58BBwD+B04HfAWcBq4GLYrHYk4njOwJ/AfoATYEZwOhYLLYgsb8HMB7YB/gQ\neKDC+3UExgFHAc2AN4FRsVjs4xrEfjbwW2BPoBy4C7ghFov5MAzbAncC/YBGwALgklgsNjUMwwAY\nk8ivNVAGjI3FYnfsaAySv3TmIPngV8CPgc5AB2A68AbQDniW6Mt8iwlAIdGXfymwGHg2DMMg8aX7\nFPAu0AY4BxhZ4b2eAdYCIdAR+DLxmjskDMOBRMXgt0ALogL3+8SfANcntu8BtAL+H/BwGIYNgGHA\ncOBIogJ3NvDnMAz329E4JH+pOEg+eCIWi30Wi8UWATOBZbFY7PFYLLYBeA7oEIZhszAMDwB6Ab+L\nxWJLY7HYCuByYC/gYOAQ4AfAdbFYbG3ibOK+LW+SOKs4JNF+eSwWW0n05d4tDMODdzDmC4CnYrHY\n5FgstikWi00H/kH0xQ9RQdgArEnsvx3oFIvFNiX2bQZWxWIxH4vF3gKKY7HY/B2MQfKYioPkgy+T\nHq8BFlZ4DlHXzB7AhuQuoFgs9gXRl/CeQKfE/q+S2v8z6XGXxJ+fh2G4LgzDdcAiIE5UVHbEHsBH\nFbZ9nIgD4AbgQGBhGIYPh2H4I6Agse8x4DPgizAMnw/D8FdEBUMkZRpzkHwQr+b5FkWAqWK7AXxi\nf8UfVMnP1yZeu2ksFttcgzgrxlIVDxCLxd4Lw3APYABwIlHX2MgwDPvEYrFlwFFhGB4CDCYab7ki\nDMNDY7HY5zsZl+QJnTmI/M8nQGEYhuGWDWEYdiEag/gP0RlHgzAMS5PaJPfj/4fo/9T+Se1NGIY/\nqGEsFccIuifegzAMWwHEYrGXYrHYKOAw4AjggDAMi8IwbB6LxWbHYrE/JuJZBZxWgzgkT6k4iPzP\nu8B84IYwDFuGYVhM1H0zD3gPmAUsAS4Pw7BxGIbdiQalAYjFYh8BrwNjwzAsDcOwEXAVMDPxeEfc\nD5wahuGAMAwbhGE4ADg1sR3gbeC6MAybJgbKDwPWA58TnUVMSFw5BdAVKAb+vYMxSB5TcRBJiMVi\nHjiZqLv1Y6LLQzcAAxMDu+uAk4iuAlpC9EU9psLL/Dix71/AN8DRwPGJtjsSy1NEg9njgGXAWOC8\nWCz2dOKQ04FDga+B74BfA6fGYrElwKVEYx3zwzBcQ3S11J9jsdjzOxKD5DejleBERKQinTmIiEgl\nKg4iIlKJioOIiFSi4iAiIpXk0k1wGlkXEamZSjd/5lJxoKysrEbtSktLa9y2PsvHvPMxZ8jPvJVz\n6m2qom4lERGpRMVBREQqUXEQEZFKVBxERKQSFQcREalExUFERCpRcRARkUryvjj492ay6qWnvv+Y\nxV/jP5iNX74sQ1GJiGRXTt0EVxPxN19m2fx3McN+RtB/8NbtfvUq/Nuv42e9Dp8lrZHSpj0m7I4Z\nOBTTYZdqX3/atGn06dOn2uNuv/12TjvtNDp27FjtsSIi6Zb3xSE48wK48TPiT9yDL2mDObAX/sP3\niD84DpYvBRPAvgdi9gjxn38Cn8bwb72Kf3sapv9gzIlnYJo0rfK1v/nmG1577bWUisOoUaNqOzUR\nkRrL++Jg2rSnzdVjWXTJ+cTvuQXT8wj8zKlQ0ABz0o8wfQZiWhZvPd57D+/NJP7k/fgpE/GzphFc\nORbTqnWl177tttv417/+xTHHHMOAAQP45ptvuOWWW7jpppsoLy9n3bp1nHPOOfTu3ZuLLrqI0aNH\nM23aNFavXs2XX35JWVkZI0eO5LDDDsvkX4mISGaKg7V2LNCLaHK80c652Un7+hEttbgZiAHnOefi\niX2NgQ+BPznnHtyZGOJPPoCf81aV+74tKICiRrByeVQYggJo1hw/41X8jFerfkHvYZfdYeHn+H/N\nw/TqV+mQYcOG8fTTT9O5c2e++OILxo0bx7Jlyzj44IMZOHAgZWVlXH311fTu3XubdosXL+aGG27g\nnXfe4dlnn1VxEJGMS/uAtLW2D7C3c643MIJoTdxk44GhzrkjgObAwKR9VwJL0x0jAA2LoHkLaNwU\nWrWGBoXff7wx0D4xYdW3i6p9+a5duwLQvHlzYrEYo0aN4oYbbmDFihWVjt1vv/0AaNu2LatXr96x\nPEREakEmzhz6AxMBnHMLrLXF1toWzrkt34o9kx6XAyUA1tquwL7AC7URRHD6uXD6uVXuq+nsjf6b\nr4i/NxO+XVztsYWFUbF55ZVXWLFiBePGjWPlypVccMEFlY4tKCj433tojW8RyYJMFIcOwJyk5+WJ\nbSsAthQGa21H4DjgqsRxtwKjgLNTfaPtTT2brrbxktYsBBquWk67KtovXLiQwsJCmjdvTsuWLbe+\nR5cuXejUqRNPPPEE8Xic0tJSioqKaNu27TbHrlq1iqKiop3KqzrpfO26Kh9zhvzMWznXXDYGpCst\nKmGtbQc8B/zCObfEWjscmOmc+8xam/ILZ2U9h5bFrC/7ssr2TZs2Zf78+RQXFxMEAWVlZRx44IFc\nccUVzJo1i0GDBlFSUsKYMWNYv3495eXlrFy5cuux5eXlrF+/Pm1z0mu++/yRj3kr59TbVCUTxaGM\n6ExhayzA11ueWGtbAC8BVzjnpiQ2nwjsYa0dDHQC1ltrv3LOvZKBeHdMSTv4/GN8fDMmKNhmV6tW\nrXjiiSe22dahQwfuu+++rc+PPfZYAM4+OzpB6ty589Z9nTt35rbbbktX5CIi25WJ4jAFuAa421rb\nAyhzzq1M2n8rMNY5N2nLBufcGVseW2uvBv5bJwsDYEra4T+NwXdLoXXbbIcjIlIr0l4cnHMzrLVz\nrLUzgDgw0lp7DrAcmAwMB/a21p6XaPKoc258uuOqNW3aR39+u0jFQURyRkbGHJxzl1XYNC/pcVE1\nba+u9YBqU5t2APhvF2O6ZDkWEZFakvcT7+0sU5I4c1hS/eWsIiL1hYrDziqJzhxYUv2NcCIi9YWK\nw84qicYZfAo3womI1BcqDjvJFDaElq3VrSQiOUXFoTa0aQdLy/GbN2c7EhGRWqHiUAtMSXuIx+G7\nJdkORUSkVqg41IbE5aypTMAnIlIfqDjUhsQVS15XLIlIjlBxqAVGZw4ikmNUHGqDboQTkRyj4lAb\nWrcFY/AprAgnIlIfqDjUAlNYqHsdRCSnqDjUljbtYNm3utdBRHKCikMtMSXtonsdln2b7VBERHaa\nikNt0aC0iOQQFYfasmVdh/JvshyIiMjOU3GoJaZ9YpHuxfm1oLmI5KaMrARnrR0L9AI8MNo5Nztp\nXz9gDLAZiAHnOefi1tqbgKMSMY5xzk3IRKw11n4XAPwiFQcRqf/SfuZgre0D7O2c6w2MAMZVOGQ8\nMNQ5dwTQHBiYKBjdE20GArelO86d1qIVNGoM3yzMdiQiIjstE91K/YGJAM65BUCxtbZF0v6ezrmv\nEo/LgRJgOnB6Ytt3QFNrbUEGYq0xY0x09rD4a3xcl7OKSP2WiW6lDsCcpOfliW0rAJxzKwCstR2B\n44CrnHObgdWJ40cALya2fa/S0tIaB7kzbbdY8oM9WfP5x7QvLKBB+51/vUyojbzrm3zMGfIzb+Vc\ncxkZc6jAVNxgrW0HPAf8wjm3JGn7EKLicFwqL1xWVrP+/tLS0hq3TRZv0RqARR/MxXTb6ZdLu9rK\nuz7Jx5whP/NWzqm3qUomupXKiM4UtsYCfL3lSaKL6SXgSufclKTtxwNXAIOcc8szEOfOS5wt+EUa\ndxCR+i0TxWEKMBTAWtsDKHPOrUzafysw1jk3acsGa21L4GZgsHNuaQZirBWmQ3TFErpiSUTqubR3\nKznnZlhr51hrZwBxYKS19hxgOTAZGA7sba09L9Hk0cSfbQBnrd3yUsOdc1+kO96d0k5nDiKSGzIy\n5uCcu6zCpnlJj4u202x8msJJG9O4CbQs1pmDiNR7ukO6trUvhSWL8Rs3ZjsSEZEaU3GoZab9LuA9\nlH9d/cEiInWUikNt23J/g7qWRKQeU3GoZUaXs4pIDlBxqG3tdTmriNR/Kg61rU0HMIHOHESkXlNx\nqGWmsDBa+EdnDiJSj6k4pEP7UljxHX7N6uqPFRGpg1Qc0sBsGXfQqnAiUk+pOKTDliuWtPCPiNRT\nKg5pYHbbM3rw0dzsBiIiUkMqDunQuQu0aY9/byZ+3dpsRyMissNUHNLABAGmVz9Yvw4/9+1shyMi\nssNUHNLE9O4LgJ/5WnYDERGpARWHNDHtSmHPrvCvD/BLv812OCIiO0TFIY3M4ceA9/hZ07IdiojI\nDlFxSCPT80hoUIif+Rre+2yHIyKSsoysBGetHQv0Ajww2jk3O2lfP2AMsBmIAec55+Lf16a+ME2b\nYQ44FD/nLfjiE9h9r2yHJCKSkrSfOVhr+wB7O+d6AyOAcRUOGQ8Mdc4dATQHBqbQpt4wvY8BwE+f\nkuVIRERSl4lupf7ARADn3AKg2FrbIml/T+fcV4nH5UBJCm3qj/16RPc8vP0aftWKbEcjIpKSTBSH\nDkRf+luUJ7YB4JxbAWCt7QgcB7xYXZv6xAQFmP6DYcMG/LRJ2Q5HRCQlGRlzqMBU3GCtbQc8B/zC\nObfEWlttm6qUlpbWOKidaVud+NCfUPbc45jpk+h4zshoWu86Ip1511X5mDPkZ97KueYyURzK2PZX\nfynw9ZYnie6il4ArnHNTUmmz3Tcqq9ksqKWlpTVum7IjBhB/+RkWPusIevdL73ulKCN51zH5mDPk\nZ97KOfU2VclEt9IUYCiAtbYHUOacW5m0/1ZgrHNu0g60qXdM/5OiFeJenqjLWkWkzkv7mYNzboa1\ndo61dgYQB0Zaa88BlgOTgeHA3tba8xJNHnXOja/YJt1xppspaYfp0Tu6rPXfH0K4X7ZDEhHZroyM\nOTjnLquwaV7S46IU29R75tgh+DlvEZ/wEMGlN2IC3YMoInWTvp0yyOzZFXPwkfBpDD99crbDERHZ\nLhWHDDNnnAeNm+AnPIT/bmm2wxERqZKKQ4aZVq0xpw6Htavx7r5shyMiUiUVhywwRw+Ezl3ws9/A\nz5+T7XBERCpRccgCEwQEw0dCEBB/5G9aSlRE6hwVhywxnTpjjj8VlizGP/NItsMREdmGikMWmcFn\nQLtS/KvP4T+NZTscEZGtVByyyDQsIhg+Crwn/tDt+E0bsx2SiAig4pB1JuyOOfp4WPg5fvLT2Q5H\nRARQcagTzGlnQ8vW+OefwC/Or4nCRKRuUnGoA0yTZtHNcZs2En/kbk3MJyJZp+JQR5iDj4BuB8FH\nc/HvvpntcEQkz6k41BHGGIIzL4QGhfgn7sWvWZ3tkEQkj6k41CGmXUfMiRaWL8M/9WC2wxGRPKbi\nUMeY40+FXXbHT59M/LXnsx2OiOQpFYc6xhQWEoy6Epq3xD9+L/79WdkOSUTykIpDHWTatCf45R+g\nsJD4Pbfg//ufbIckInkmIyvBWWvHAr0AD4x2zs1O2tcIuBvo5pw7OLGtGfAQUEy0Utw1zrm8Wh3H\ndN6b4PzfEr9zDPG7biS47i5Mg4x8XCIi6T9zsNb2AfZ2zvUGRgDjKhxyM/B+hW3nADHnXD9gKPCX\ndMdZF5kDD8P0OyGanO+d6dkOR0TySCa6lfoDEwGccwuAYmtti6T9lwMV5434FihJPC5OPM9L5rhT\nIAjwkyfg4/FshyMieSITxaEDUJ70vDyxDQDn3MqKDZxzjwO7WWs/BqYDv013kHWVKWmHOeQoKPsC\ntDCQiGRINjqxTXUHWGt/DHzhnBtorT0AuA84uLp2paWlNQ5qZ9qm24afXMiiWdNoMPU52g8aUquv\nXZfzTpd8zBnyM2/lXHOZKA5lJJ0pAKXA19W0OQKYDOCcm2etLbXWFjjnNn/vG5XVbNK60tLSGrfN\niKKm0L0nGz6cw8Lpr2L22qdWXrbO550G+Zgz5Gfeyjn1NlXJRLfSFKJBZay1PYCyqrqSKvgYOCzR\nZndgVXWFIdcFA08DID7pqSxHIiL5IO1nDs65GdbaOdbaGUAcGGmtPQdY7px72lr7JLArEFprXwfG\nE13aer+1dloixgvTHWed16Ub7NkV5r1DfNokgj4Dsx2RiOSwjIw5OOcuq7BpXtK+07fTzKYvovrH\nGEMw4tfE//wb/GN34zt2wnTpnu2wRCRH6Q7pesS07UBwYVRn43+7Af/toixHJCK5SsWhnjHhfphh\nP4NVK4jfcT1+44ZshyQiOSilbiVr7T7Aic65W6y13YG7iMYPRjvn5qYzQKks6DuI+Jef4adPwk94\nKFpFTkSkFqV65vAA8Fni8R3AS8D1iceSBcaOgA6d8K88i/+o4uwjIiI7J9Xi0NI595S1th1wAHBj\nYiK85ukLTb6PKSoiOO/XUFBA/IG/4FdXd3WwiEjqUi0O3lrbBBgGTHHObbLWFhLNmCpZYnbfC3PS\nj+C7Jfi/34mP5/WtICJSi1K9lPVO4CuicYb+iW2PUHnCPMkwM/A0/Idz8HPewt+yjOCnF2PatM92\nWCJSz6V05uCcux3YDejgnNtyj8K1zrlL0xaZpMQUFEQrx/U4HP7zEfFrfkV8xqvZDktE6rmUikPi\naqULE91J3a21bwJ3WmsPTG94kgrTtDnBhZdifnoxGIN/4C94zeAqIjthZ69WujMdQcmOM8YQ9O5H\n8Ns/gwmIP3k/ftOmbIclIvWUrlbKMWa3PTBHHQtff4l/I69WVhWRWqSrlXKQGXImNGqMf/ZR/OpV\n2Q5HROqhVIvDlquV/kDUnQS6WqnOMi2KMSdaWLUS/8IT2Q5HROohXa2Uo0z/k6FNe/xrL+Dnzc52\nOCJSz6R6tZIBTgbutda+aK19AOia1shkp5jCQoKzfg7GEL/9T8TdffhNG7MdlojUE6l2K90MjAbe\nAx4jWo/h99baP6YrMNl5pnsPgstvgfa74F9+hvhNv8d/tzTbYYlIPZDqHdKDgJ7OuXVbNlhr7wHe\nAa5JR2BSO8yunQmu/D/8I3fh355K/LY/EvxuDKZps2yHJiJ1WKpnDgXA+grb1u5Ae8ki06gx5qcX\nYfqdCAs/J/7Xa/Hr11XfUETyVqpnDq8Dz1prxwPLgBLgPGBqKo2ttWOBXoAnWgNidtK+RkRrRndz\nzh2ctP0s4BJgE/AH59wLKcYqVTDGwLDzYc0q/KxpxO8cg/+zZlwXkaql+sv/V8CbwG+B8cBFRAXj\nouoaWmv7AHs753oDI4BxFQ65GXi/QpsS4I/AkcBgYEiKccr3MEGAOWc07H8IfDSXpbePwXuf7bBE\npA763jMHa+1/iH7tA5ikXZ2AC4ELgC7VvEd/YCKAc26BtbbYWtvCObcisf9yojORs5LaDABecc6t\nBFYCP0shF0mBadCA4IJLiN/0e9a88hympB1mgGqviGyrum6l2lh/sgOQPAtceWLbCgDn3MrEmUKy\nHwBNrLXPAsXA1c65aqcaLS0trXGQO9O2Ptp07V9YdNFw4k8+QOvuB9GoR69sh5Qx+fZZb5GPeSvn\nmvve4uCcm1Yr77ItU/0hGKKziR8CuwNTrbW7O+e+tw+krKysRgGVlpbWuG191ubKW1h86fmUj7mU\n4JIbMLvsnu2Q0i5fP+t8zFs5p96mKpm42qiM6ExhayzA19W0WQTMcM5tcs59QtS11DZN8eWtoq77\nYX4yCtasJj7md8Rnv5HtkESkjshEcZgCDAWw1vYAyhJjCdW1OcZaGyS6nJoB36Y3zPwUHH4MwQWX\nAAY//mbij9+jO6lFJP3FwTk3A5hjrZ1BdKXSSGvtOdbaHwJYa58EHgdCa+3r1toznXMLgX8AbxOt\nHfFL51w83bHmK3PwkQRX3AIdd8W/+hzxW67Af7ck22GJSBaZHLqU0WvMYcdUzNuvW4t/6Hb87Deg\nZTHBzy7BdOmWxQhrnz7r/KGcU29DFWPBusNZtjKNGmPO/y3mjBGwcjnx/7uS+HQtGCSSj1QcZBvG\nGIIBQwh+cx00aYb/+x3Ep76Y7bBEJMNUHKRKpkt3gt9eDy1a4R+9i/hrz2c7JBHJIBUH2S5TultU\nIFoW4x8bT3zKxGyHJCIZouIg38t03DUqEK1a45+8n/iT9+PjunBMJNepOEi1TIdOBJfdBB064adM\nxN97K36j7oUQyWUqDpISU9KO4LIbYa998LPfIH79r4nPmobfvDnboYlIGqg4SMpM0+YEF1+LOfJY\nKPsSf++txK/6Of69GdkOTURqmYqD7BDTsIjg7F8SXH8Xpu8gWLaE+Phb8F99lu3QRKQWqThIjZi2\nHQjO+jnBLy6HzZuI338bftOmbIclIrVExUF2itmvJ+aIAfDlZ/gXn8x2OCJSS1QcZKcZOwKK2+Bf\ndPgvPs12OCJSC1QcZKeZJk0Jho+CzZuJ/20M/qP3q28kInWaioPUCtO9B+akYbCknPjYP7D5zj/j\nF36utSFE6qnq1pAWSVlw8pn4Aw8j/tg9MPdt4nPfBmOgVQlmz66Ys0dhGjXJdpgikgIVB6lVZrc9\nCS4ZE60JMf9d/JLFUP4N/t038WtWEfzyKkyDwmyHKSLVUHGQWmeMwRx6NBx6NAA+MRbBvHfwD46D\nn16MCdSjKVKXZaQ4WGvHAr0AD4x2zs1O2tcIuBvo5pw7uEK7xsCHwJ+ccw9mIlapfaaggOD83xEf\nexV+1jRoWQxDz8WYSotPiUgdkfafb9baPsDezrnewAiidaST3Qxs7/KWK4GlaQxPMsQUFRGMuvJ/\nk/dNeIgcWqJWJOdk4ty+PzARwDm3ACi21rZI2n858HTFRtbarsC+wAsZiFEywDRrQXDxtdB+F/yk\np/CP36Ppv0XqqEx0K3UA5iQ9L09sWwHgnFtprS2pot2twCjg7FTfKLFQdo3sTNv6LON5l5ay+db7\nKb/iF2x87XmaFAS0uvASgkaNMhiCPut8oZxrLhsD0tV2NFtrhwMznXOfWWtTfuGysrIaBVRaWlrj\ntvVZNvP2F10Dt13N6pefZfWs6ZhBp2GOHohpWJTW99VnnT+Uc+ptqpKJbqUyojOFrbEAX1fT5kRg\niLX2beA84Cpr7YA0xSdZYJq1IPjtdZjBZ8CG9fgn7iN+2XnE772V+NQX8V98it+4IdthiuStTJw5\nTAGuAe621vYAypxzK7+vgXPujC2PrbVXA/91zr2S1igl40yjJpghZ+H7nxQNUr8xObqaadY0PEQ3\n0LVuCx12IRhwMqZ7z2yHLJI30l4cnHMzrLVzrLUzgDgw0lp7DrDcOfe0tfZJYFcgtNa+Dox3zj2a\n7rik7jDNWmBOHY4/5cewqAz/yQL47D/4RQth0UL451zi/5yLObQP5owRmBatsh2ySM4zOXQ5odeY\nw46pL3n7rz4j/tAd8Nm/oUkzgl/8HhPuV6PXqi8517Z8zFs5p96GKsaCdZuq1HmmU2eCy27EDPsZ\nrF9H/J5b8atWZDsskZym4iD1ggkKCPoPxgw5E5YvxT96d7ZDEslpKg5Sr5jjfwh7dsXPfoP4O9MB\n8OvW4t+fhS//JsvRieQOTbwn9YoJCgh+ehHxa0bjH/kb8flz8HNnwvp10LAh5vQRmD4DNW+TyE7S\nmYPUO6ZdKeb0n8Ka1fi3p0KLVpgBJ0ODhlHBuON6/Ipl2Q5TpF7TmYPUS6bPQGjcBFPcBvbeF2MM\n/rgfEn/gNpj3DvHLP8D0PQFz3Cm69FWkBlQcpF4yxmAO67PttuISgouuwU+bhH/R4SdPwE99Hvbu\nBg0KIQhY0f0gfK9j0j5Nh0h9p+IgOcUEAabfCfgjB+DffAU/6R/wz7lb9y+f+zZMfoZg+ChM2D2L\nkYrUbSoOkpNMYcOoSPQdBBs3wObNsGE9Td6czKpnHid+y+WYvoMwZ5ynZUtFqqABaclpxhhMwyJM\n4yaYlsUUn/9rgt/fBLvsjn/9JeLjrsWvXZPtMEXqHBUHyTumcxeC398CBxwKC+YRv+ky/HdLsh2W\nSJ2i4iB5yRQVEfz895i+g+Cr/xL/08XEJz+NX6ezCBFQcZA8ZgoKMGdeGN0zsX49/h8PEL90BPEJ\nD+G/+Srb4YlklQakJa8ZYzDHnYI/YgB+6gv4V5/Dv/QP/Ev/gN33whx6FKbnkZiSttkOVSSjVBxE\nANO0GWbwGfhjT8HPnYl/Zzr88z385x/jn3wAOneJCkWfQZjChtkOVyTtVBxEkpiiIkyvvtCrL37l\ncvx7M/Fz3oJ/zcd/9m/8m68QnPcbTKcfZDtUkbRScRDZDtO8ZTRNR5+BUaF49tHo8tfrf4MZciam\nXWl0GezmjZhuPdX1JDklI8XBWjsW6AV4YLRzbnbSvkbA3UA359zBSdtvAo5KxDjGOTchE7GKVMU0\nb4k56+f47j2JPzgO/9T/I3kNRW8C2P9ggr6DoFsPzQor9V7ai4O1tg+wt3Out7V2H+B+oHfSITcD\n7wPdktr0A7on2pQAcwEVB8k6c8ChBFf/FT/rdQgCaNwUNmzAz3g1mvBv3jvQ7SCCcy/CtCzOdrgi\nNZaJS1n7AxMBnHMLgGJrbYuk/ZcDT1doMx04PfH4O6CptbYg3YGKpMK0LCY47ocEA4YQHDGAoN8J\nFFxxK8EVt0K3g+Cfc4lf8yv8/Dn4ZUvw898lPnkC8Tem4D9egF+zOtspiFQrE91KHYA5Sc/LE9tW\nADjnVibODrZyzm0GtvwPGgG8mNj2vRILZdfIzrStz/Ix77TlXFqK73UUq559nO8e+CvxcddUOmRL\nV1Sjgw+neOTlNGjXIT2xVBlTTthPAAAQWklEQVSePut8UFs5Z2NAOuXOWGvtEKLicFwqx5eVldUo\noNLS0hq3rc/yMe+M5HxoX4IOuxJ/+uFodbpOnTG77BadMSz8Av/JAta9O4Ovf3465vSfYo46Lu1j\nFPqs80NNct5eMclEcSgjOlPYGgvwdXWNrLXHA1cAA51zy9MUm0hamN32pGD0H7fdlvjTe4+f8Sr+\nifvwf78D/850gmHnb7081i8tj27CK2iAsSMwgSYykMzLRHGYAlwD3G2t7QGUOedWfl8Da21LooHq\nAc65pRmIUSRjjDGYIwbg9z2I+CN/iway/3QRps8gKCzEv/YCbNoYHVzcBnP8D7e2jb/6HHy8AHPu\naC1YJGllvPfVH7WTrLU3AEcDcWAkcBCw3Dn3tLX2SWBXoquV5gDjgWbA1cC/k15muHPui+95G69u\npR2Tj3nXxZz9/HeJP34vLE7E1boNZtBQ/PMOVi0n+N0YzJ5diU+egP/HgwCY084mGHhayu9RF/NO\nN+Wcehuq6O7PSHHIEBWHHZSPedfVnP2mjfhpk8EQjUEUNsQvmEd87B+gdVtMn4H4CQ9Bq5Jo8aJ4\nnODPd2Oataj+xam7eaeTck69DVUUB3VmitQBpkEhQf/BBMcM3jp3k9nnAMyJFpYsjgpDi1YEv7kO\nM9jC2tX4F1yWo5ZcpuIgUoeZwcNg34OiwnDxtZgOu2D6nABt2uOnvohfXO21HSI1ouIgUoeZggKC\n0X8kuOG+rVczmcJCzA9/Aps34Sc+nN0AJWdp4j2ROs4EQTRVR/K2g4/Ev/wMfvYbxBs3wZx+LqZR\nk637/bo1sGY1rF0bFZH27TIdttRzKg4i9ZAJAoIRFxO/60b89Mn4j97HnPJjWPg5fv678NV/tzm+\nrEUr/IG9MIceBV26a2JAqZaKg0g9ZTp0Irji//DPPYafNAF/763RjgaF0HX/aOK/xk1g0yb4cA5+\n+iT89EnQpTvBT36B6dAJIJrvadok6Lof5vD+KhwCqDiI1GumsBBz6nD8gYfh572D6dwF9jkAU9Ro\nm+M6tm9P2etTopvo5r1D/JpfYY45Cf/5xxCbHx309lT8rGkEPxmJaZu5OZ+kblJxEMkBZo8Qs0e4\n/f0FBZh9DqBgnwPw780k/tjd+CmJyZD3PYig3wnE35gCH8wmfvUozNGDMD0Phz1CTd+Rp1QcRPKM\n6dGbYJ8D8G+9jNlzX0znvQEIDjgUP/sN/OP34F95Bv/KM9CqNeao4zHHn4opiqbr8Js24WdNi1bA\nO6zf1u2SW1QcRPKQadwEM2DIttuMwRx6NP6g3rDgffx7M/BzZ0VjGm+9jBn6U8Djn/47lH8DgH/2\nMczA0zB9Bm69eU9yg4qDiGzDFBbC/odg9j8EP2wt/sUn8S9PxI+/KTqgoAGm34nQuAn+1efxT9yL\nf/U5gl9cjtm1c3aDl1qj4iAi22UaNY4GvI8cgH/6YSgsxJz0o60D1n7AkKh4vPIM8RsuITh3NPQ8\nAj54l/hzj0H515hjT8Ecd4pmka1nVBxEpFqmXSnmgksqb2/eAnPGCHyXbsTvG0v87pug/S6waCEY\nA42a4J95BP/my5gThkLDIli3FjCYnodjmrfMfDKSEhUHEdlp5qBeBL+/mfgd18HiMszBR2IGnwGt\n2+JfeAL/ynP4v9+5TRvv7sMc0R9z7BBMu/xbzrOuU3EQkVphdtmN4I/jYNVKTEnb/20fei7+6OPx\n8+dAYUNo1BiWL8O/+hz+9Zfw0yZjjj0ZM+QsdT3VISoOIlJrTFEjqHADHiS6pfpve3bgjxkcXRE1\n8WH8lIn4D94lOOdX0K4jLF8KK76DtWvx69dGXVHl3+C/+QoWf4PZsyvm5B9h2rTPVGp5R8VBRLLC\nFBRgDjkKv/8h+AkP4V97nvgNlcc1KmnUGD/zNfw70zF9BsIeISwth6XfQtv20RQgKS6CJNuXkeJg\nrR0L9AI8MNo5NztpXyPgbqCbc+7gVNqISO4wRY0wP/oZvkdv4pMmRFdEtWwNLVpBk6bRmUhRY0yb\ndtFgd6PG0c16Ex/Gv/Y8vPb8Nq/nn34Yc8hRrBs8FN+iNaZRE/z6dfi3XsG/8iysXYMZPAzTdxCm\noCBLWdd9aS8O1to+wN7Oud7W2n2A+4HeSYfcDLxPtIZ0qm1EJMeYcD8Kwv1SO/awPvieh+Nnvwnr\n12Jat4VWrfGxD/Gvv4if+RrlM1+LrpjquCt8txTWrIomJWzQAP/4ePz0SQTDzsfsc0CaM6ufMjFp\nSn9gIoBzbgFQbK1NPue7HHh6B9uISJ4zDQoJevcj6HsCZv9DMLvtSXDsEII//Y3g4mtoftpw6NId\nliyGIMCcNIzgxvsIrr8bc9Rx8PWXxP/vKuKP3o3fsD7b6dQ5mehW6gDMSXpenti2AsA5t9JaW7Ij\nbbYnsVB2jexM2/osH/POx5whz/LuFE1H3grwmzdHxSF5KvLL/syGj4ez5NY/sGnqCzT49F+0PGcU\nGz6JsW72m2z84jMa7hVS1O0girofRMN9DiBo1Dg7ueyg2vqcszEgXZPJ4lNqU1ZWVoOXjv4ya9q2\nPsvHvPMxZ8jPvKvNuUkL/KU3Yv7xABunvsi311wcbQ8CaNOB9R/OZf3896JtBQ1gjy6YrgdgjjoO\nU1zx92zdUJPPeXvFJBPFoYzoV//WWIDqVkWvSRsRkR1iGhZhzrwQv98h+H++B3t2xXQ7CNOkGX7N\nKvh4QTSOEZsfPf7PR/hJT0WD2YOGVrrD23/2n2hBpXYdMUcfj2naPEuZ7bxMFIcpwDXA3dbaHkCZ\nc25lGtqIiNSI2a8nZr+e225r0mzrBIQAfvUq/Jw38S+4aP3u6ZNhnwMxe++D6dCJ+PTJMO+dre39\n809El9UOPG2bmwLrC+O9T/ubWGtvAI4G4sBI4CBguXPuaWvtk8CuRFcrzQHGO+cerdjGOTevmrfx\n6lbaMfmYdz7mDPmZd7py9hs3Rut2vzwxGuxOttc+BCdafNkX+Fefi+69aNyU4JxfYnocXuuxVLQT\n3UqVuu4zUhwyRMVhB+Vj3vmYM+Rn3pnI2S8px3/8EXz1X0y4H3Q7aOvAt9+8Gf/my3h3L2zYgOl3\nAuaIAfiyL6Hsi+gy29JdMaW7QWERLFmMX7o4uhu8UePEvR3to9X4kgbT/Vef4T//FHNYH0yDbTt/\narM46A5pEZEaMiVtMSV94LA+lfcVFGD6DMTvvS/xu2/CT30RP/XFSsd9389zD9H4xZHHYTrsQnzq\nC7Ag6kTxM14luPDStM1sq+IgIpJGpnQ3gstvxU96ClYth9Ldo7MFPL7si+gsYtMmKGkLrdthGjf5\n33xSn8Twc97CT/h//ysiXfePpj7/YDbx639DMPKKtCyypOIgIpJmpqgIM+TMytu3c0f41j6evifg\nf3R+tGb3t4swvfphdu2Mj8fxLzr8M49Giyxde2etD3qrOIiI1GGmSbNoWdbkbUGAGTwM3+kHxGe8\nBkW1P9W5ioOISD1lDuxFwYG90vLamZhbSURE6hkVBxERqUTFQUREKlFxEBGRSlQcRESkEhUHERGp\nRMVBREQqUXEQEZFKcmpW1mwHICJST+X0rKw1WX5URESqoG4lERGpRMVBREQqUXEQEZFKVBxERKQS\nFQcREalExUFERCrJpUtZa8RaOxboRXSfxGjn3Owsh5QW1tqbgKOIPvMxwGzg70AB8DXwE+fc+uxF\nmD7W2sbAh8CfgFfJ8byttWcBlwCbgD8AH5D7OTcDHgKKgSLgGuAb4G9E/7c/cM79PHsR1i5rbXfg\nGWCsc+52a+2uVPEZJ/4tXATEgfHOuftSfY+8PnOw1vYB9nbO9QZGAOOyHFJaWGv7Ad0TeQ4EbgOu\nBe5wzh0FfAz8NIshptuVwNLE45zO21pbAvwROBIYDAwhx3NOOAeIOef6AUOBvxD9Ox/tnDsCaGmt\nHZTF+GqNtbYp8FeiHzpbVPqME8f9ARgA9AUutta2TvV98ro4AP2BiQDOuQVAsbW2RXZDSovpwOmJ\nx98BTYn+sTyb2PYc0T+gnGOt7QrsC7yQ2NSX3M57APCKc26lc+5r59zPyP2cAb4FShKPi4l+DHRO\n6gnIpbzXAycAZUnb+lL5Mz4MmO2cW+6cWwu8BRyR6pvke7dSB2BO0vPyxLYV2QknPZxzm4HViacj\ngBeB45O6FhYDHbMRWwbcCowCzk48b5rjef8AaGKtfZboS/Jqcj9nnHOPW2vPsdZ+TJT3ScAdSYfk\nTN7OuU3AJmtt8uaqPuMORN9pVNieknw/c6gop6fgsNYOISoOoyrsysm8rbXDgZnOuc+2c0gu5m2I\nfkGfStTV8gDb5pmLOWOt/THwhXNuL+AY4OEKh+Rk3tuxvVx36O8g34tDGVF13aKUaDAn51hrjweu\nAAY555YDqxIDtQC7sO0paq44ERhirX0bOA+4itzPexEwwzm3yTn3CbASWJnjOUPUXTIZwDk3D2gM\ntEnan6t5b1HVv+uK32879HeQ78VhCtHgFdbaHkCZc25ldkOqfdbalsDNwGDn3JaB2VeA0xKPTwMm\nZSO2dHLOneGcO8Q51wu4l+hqpVzPewpwjLU2SAxONyP3c4ZoEPYwAGvt7kRFcYG19sjE/lPJzby3\nqOozngUcYq1tlbia6wjgjVRfMJem7K4Ra+0NwNFEl3qNTPzqyCnW2p8R9T3/O2nz2URfmI2Az4Fz\nnXMbMx9dZlhrrwb+S/Tr8iFyOG9r7QVE3YcA1xFdtpzrOTcD7gfaE42lXkV0KevdRD+CZznnfp29\nCGuPtbYn0VjaD4CNwELgLOBBKnzG1tqhwO+ILuf9q3PukVTfJ++Lg4iIVJbv3UoiIlIFFQcREalE\nxUFERCpRcRARkUpUHEREpBIVB5E6wFrbNzH1g0idoOIgIiKV6D4HkRQk5qW6jmhG24+BM4FbgGXA\ngUAXokkchznn1lhr9ydaS6AEWAdc6pybnHitS4ELiNZbeB74DdCH6KbEB4AfAw2B85xzUzOVo0gy\nnTmIVMNauwfRQio/cs7tAUwF7krs/iHRFCy7Ai2B8621AfA4cLtzrivRvE6PWWubJ6ZzOA84AOhO\ntO7C0MRrdQLmO+f2ISosV2YiP5GqqDiIVG8g8Lpz7sPE87uAk4lW3XrGObfEORcnWhvkcKAz0YRn\njwM4594lmtLgEKJ5+F9IrLewgWge/gmJ113hnNsyJ/9comIhkhX5vp6DSCpaAUdba/+VtG05UZdR\n8iDyMqK1BNoC3znnfIV97YhmCt06M6Zzbg1AYm7+5HVENhMVH5GsUHEQqV4Z0epqQ5M3WmsfZNtp\noVsTrUC2CGhtrTVJBaIksf3b5DaJmVNF6hwVB5HqTQZutNbu4Zz71Fp7KNEsmAADrbWtiKaIPgV4\ngmj216+AM4DHrbWHE3UzvQOsBZy19qpEm4lE4wu5vNaA1EMacxCphnPua+B84Glr7QLgdqIiANEi\n7xOIisEy4P7E2cIwYFTi+HHA6c651c65t4nW1ngf+Ah4D3gsk/mIpEKXsorUUKJb6WPn3HXZjkWk\ntunMQUREKlFxEBGRStStJCIilejMQUREKlFxEBGRSlQcRESkEhUHERGpRMVBREQq+f99w09jAVlX\nmwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fabfb19d9b0>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history128.history['loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 36,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8202,
     "status": "ok",
     "timestamp": 1521492023412,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "rSkrb30r_T-C",
    "outputId": "6fbfcb7d-d5dc-4ee6-f70b-a427dd393b6e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4878/4878 [==============================] - 8s 2ms/step\n"
     ]
    }
   ],
   "source": [
    "score128 = model.evaluate(np.array(X_val),y_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "N8jXuDg8_ruM"
   },
   "source": [
    "### 79% on validation best performence until now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 36,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 548,
     "status": "ok",
     "timestamp": 1521492032112,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "hZOAqJJZ_eX6",
    "outputId": "11cbd8aa-7f14-4670-ec2c-c3d2f445b627"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.14789183973019707, 0.7939729397782735]"
      ]
     },
     "execution_count": 98,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score128"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Xxu4D5rH_9rW"
   },
   "source": [
    "### Let's try on test data now\n",
    "-  with all test exemples except one to see if the model can predict one exemple at a time\n",
    "-  we keep the last exemple of test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 36,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 570,
     "status": "ok",
     "timestamp": 1521492395870,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "nf8buC8jAuF0",
    "outputId": "fcfee3df-d87b-4620-e0aa-e83e9b30de21"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5737"
      ]
     },
     "execution_count": 104,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_test[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 36,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9212,
     "status": "ok",
     "timestamp": 1521492413540,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "c4wUFptX_13i",
    "outputId": "3cb3c497-1fee-40a3-9c91-3a537d2325a0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5737/5737 [==============================] - 9s 2ms/step\n"
     ]
    }
   ],
   "source": [
    "score128_test = model.evaluate(np.array(X_test[:-1]),y_test[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 36,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 600,
     "status": "ok",
     "timestamp": 1521492417240,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "EXpMtQOsAnLW",
    "outputId": "de8b4dc5-540a-42ad-b814-add2a26a84fa"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.14900220963043556, 0.7937946662226267]"
      ]
     },
     "execution_count": 106,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score128_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 36,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1472,
     "status": "ok",
     "timestamp": 1521492485392,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "UQYLn0q_BAM8",
    "outputId": "366570fb-ab50-461b-9035-42f1dba2784f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "1/1 [==============================] - 0s 44ms/step\n"
     ]
    }
   ],
   "source": [
    "score128_test_one = model.evaluate(np.array(X_test[-1:]),y_test[-1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 36,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 500,
     "status": "ok",
     "timestamp": 1521492493418,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "4HTUpg9SBPLI",
    "outputId": "981b5b12-8c5e-4046-9c57-e7e354ab9bdf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.1409723162651062, 1.0]"
      ]
     },
     "execution_count": 110,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score128_test_one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "UgqWNS8QEacz"
   },
   "outputs": [],
   "source": [
    "predict_128 = model.predict_classes(np.array(X_test[:-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 136,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 650,
     "status": "ok",
     "timestamp": 1521493379449,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "KnmQyAQ_EnGw",
    "outputId": "6468b121-9ff7-4003-8567-124949560d80"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0],\n",
       "       [1],\n",
       "       [0],\n",
       "       ...,\n",
       "       [1],\n",
       "       [1],\n",
       "       [0]], dtype=int32)"
      ]
     },
     "execution_count": 122,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "TziykSk3EpwQ"
   },
   "outputs": [],
   "source": [
    "predict_128_one = model.predict_classes(np.array(X_test[-1:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {}
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1006,
     "status": "ok",
     "timestamp": 1521493447827,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "dS7fK6mKEttG",
    "outputId": "0479421a-2048-43c3-d026-73c94e3b9d5c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicted : [[0]] true label [0]\n"
     ]
    }
   ],
   "source": [
    "print(\"predicted :\",predict_128_one,\"true label\",y_test[-1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "-0ctGrjYBp3S"
   },
   "outputs": [],
   "source": [
    "model.save('LSTM_Model.h5')  # creates a HDF5 file 'my_model.h5'\n",
    "del model  # deletes the existing model\n",
    "\n",
    "# returns a compiled model\n",
    "# identical to the previous one\n",
    "model = load_model('LSTM_Model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "WqfaPvoSCnum"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.download('LSTM_Model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Q9TSG9R5Hs4Y"
   },
   "source": [
    "### Stacked LSTM layer\n",
    "-  Now we try 3 stacked LSTM layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1023,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      },
      {
       "item_id": 519
      },
      {
       "item_id": 996
      },
      {
       "item_id": 1256
      },
      {
       "item_id": 1652
      },
      {
       "item_id": 2086
      },
      {
       "item_id": 2518
      },
      {
       "item_id": 2949
      },
      {
       "item_id": 3305
      },
      {
       "item_id": 3688
      },
      {
       "item_id": 3972
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2190189,
     "status": "ok",
     "timestamp": 1521560891524,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "yr8uOWsJHaRX",
    "outputId": "38c101ee-1d9d-4712-ae79-720db750f10f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_2 (LSTM)                (None, 100, 100)          80400     \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 100, 32)           17024     \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 32)                8320      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 105,777\n",
      "Trainable params: 105,777\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/keras/models.py:942: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "27636/27636 [==============================] - 110s 4ms/step - loss: 0.2441 - binary_accuracy: 0.5754\n",
      "Epoch 2/20\n",
      "27636/27636 [==============================] - 110s 4ms/step - loss: 0.2446 - binary_accuracy: 0.5757\n",
      "Epoch 3/20\n",
      "11008/27636 [==========>...................] - ETA: 1:06 - loss: 0.2442 - binary_accuracy: 0.577327636/27636 [==============================] - 110s 4ms/step - loss: 0.2445 - binary_accuracy: 0.5755\n",
      "Epoch 4/20\n",
      "27636/27636 [==============================] - 110s 4ms/step - loss: 0.2444 - binary_accuracy: 0.5755\n",
      "Epoch 5/20\n",
      "16512/27636 [================>.............] - ETA: 44s - loss: 0.2443 - binary_accuracy: 0.576127636/27636 [==============================] - 110s 4ms/step - loss: 0.2444 - binary_accuracy: 0.5755\n",
      "Epoch 6/20\n",
      "27636/27636 [==============================] - 110s 4ms/step - loss: 0.2444 - binary_accuracy: 0.5755\n",
      "Epoch 7/20\n",
      "18176/27636 [==================>...........] - ETA: 37s - loss: 0.2442 - binary_accuracy: 0.576527636/27636 [==============================] - 110s 4ms/step - loss: 0.2444 - binary_accuracy: 0.5755\n",
      "Epoch 8/20\n",
      "27636/27636 [==============================] - 110s 4ms/step - loss: 0.2325 - binary_accuracy: 0.6048\n",
      "Epoch 9/20\n",
      "18688/27636 [===================>..........] - ETA: 35s - loss: 0.2037 - binary_accuracy: 0.678927636/27636 [==============================] - 110s 4ms/step - loss: 0.2019 - binary_accuracy: 0.6828\n",
      "Epoch 10/20\n",
      "27636/27636 [==============================] - 109s 4ms/step - loss: 0.1892 - binary_accuracy: 0.7061\n",
      "Epoch 11/20\n",
      "18688/27636 [===================>..........] - ETA: 35s - loss: 0.1815 - binary_accuracy: 0.720827636/27636 [==============================] - 109s 4ms/step - loss: 0.1812 - binary_accuracy: 0.7228\n",
      "Epoch 12/20\n",
      "27636/27636 [==============================] - 110s 4ms/step - loss: 0.1746 - binary_accuracy: 0.7347\n",
      "Epoch 13/20\n",
      "18688/27636 [===================>..........] - ETA: 35s - loss: 0.1695 - binary_accuracy: 0.745727636/27636 [==============================] - 109s 4ms/step - loss: 0.1689 - binary_accuracy: 0.7469\n",
      "Epoch 14/20\n",
      "27636/27636 [==============================] - 109s 4ms/step - loss: 0.1659 - binary_accuracy: 0.7504\n",
      "Epoch 15/20\n",
      "18688/27636 [===================>..........] - ETA: 35s - loss: 0.1603 - binary_accuracy: 0.761827636/27636 [==============================] - 109s 4ms/step - loss: 0.1613 - binary_accuracy: 0.7600\n",
      "Epoch 16/20\n",
      "27636/27636 [==============================] - 109s 4ms/step - loss: 0.1572 - binary_accuracy: 0.7686\n",
      "Epoch 17/20\n",
      "18688/27636 [===================>..........] - ETA: 35s - loss: 0.1519 - binary_accuracy: 0.776827636/27636 [==============================] - 108s 4ms/step - loss: 0.1529 - binary_accuracy: 0.7758\n",
      "Epoch 18/20\n",
      "27636/27636 [==============================] - 109s 4ms/step - loss: 0.1485 - binary_accuracy: 0.7842\n",
      "Epoch 19/20\n",
      "18688/27636 [===================>..........] - ETA: 35s - loss: 0.1441 - binary_accuracy: 0.791827636/27636 [==============================] - 109s 4ms/step - loss: 0.1448 - binary_accuracy: 0.7919\n",
      "Epoch 20/20\n",
      "27636/27636 [==============================] - 108s 4ms/step - loss: 0.1398 - binary_accuracy: 0.8001\n"
     ]
    }
   ],
   "source": [
    "lstm_out = 100\n",
    "batch_size = 128\n",
    "\n",
    "model3L = Sequential()\n",
    "model3L.add(LSTM(lstm_out,return_sequences=True, input_shape=(X_a.shape[1], X_a.shape[2])))#dropout_U = 0.2, dropout_W = 0.2)\n",
    "model3L.add(LSTM(32, return_sequences=True))  # returns a sequence of vectors of dimension 32\n",
    "model3L.add(LSTM(32))  # return a single vector of dimension 32\n",
    "model3L.add(Dense(1,activation='sigmoid'))\n",
    "model3L.compile(loss='mean_squared_error',\n",
    "              optimizer='adam',\n",
    "              metrics=['binary_accuracy'])\n",
    "print(model3L.summary())\n",
    "history128_3Layer = model3L.fit(X_train, y_train, batch_size = batch_size, nb_epoch = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 63
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 17962,
     "status": "ok",
     "timestamp": 1521560913290,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "4uaqkYLZIUyQ",
    "outputId": "5ae34af3-4fa6-4bb0-8234-4f552b00c963"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4878/4878 [==============================] - 17s 4ms/step\n"
     ]
    }
   ],
   "source": [
    "score3L = model3L.evaluate(np.array(X_val),y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 582,
     "status": "ok",
     "timestamp": 1521560926689,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "OtHkarpkGSL3",
    "outputId": "893b1e1a-2d04-4a5a-cfd1-0bda41abb4c1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.15905393698558595, 0.7621976219517816]"
      ]
     },
     "execution_count": 30,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score3L"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 298,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 715,
     "status": "ok",
     "timestamp": 1521560969034,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "K7kvkiYWGZTE",
    "outputId": "aae0356b-4515-47d9-b6ed-f23c1d93f498"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEZCAYAAAB8culNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xu8VXP+x/HXd51uul9k6rgktxVF\nilIMhX5kGAb5YhiakctPjYxLGPdhksuINMYY9zHk64fkMiOMe5lSCWGTQXRClO5Jre/vj7VPtn06\nndM5Z+91zt7v5+PR4+y91vru/W61O5+9vmut79d47xEREckUJB1ARETqHxUHERGpQMVBREQqUHEQ\nEZEKVBxERKQCFQcREalAxUGkFsIwHBiGoQ/DcPNqbu/DMBxSyboXwzAcX7cJRWpGxUFERCpQcRAR\nkQoaJR1AJJfCMPTAr4D/BXoBc4BjgPOBE4AVwNmpVOrh9PadgZuBAUALYAowMpVKvZde3xu4HdgZ\neAe4O+v9OgPjgH2BlsCrwIhUKjW3BtlPBs4DtgcWArcBY1KplA/DsCNwK7A/0Ax4DxiVSqVeCMMw\nAK5J//3aA2XA2FQq9edNzSDFS0cOUgzOAk4EugKdgJeBV4AtgEnEv8zLPQo0Jv7lXwp8BUwKwzBI\n/9J9BHgD2BwYCgzPeq/HgVVACHQGPku/5iYJw3AwcTE4D2hNXOAuSv8E+GN6+XZAW+Be4P4wDBsB\nxwEnAT8lLnAnA6PDMNx1U3NI8VJxkGLwUCqV+jiVSn0JTAUWp1KpCalUag3wBNApDMOWYRj2BPoB\n56dSqUWpVGop8HtgB2BPoA+wLXB1KpValT6auLP8TdJHFX3S7ZekUqllxL/cu4dhuOcmZj4deCSV\nSj2TSqXWplKpl4H/I/7FD3FBWAOsTK8fD2yVSqXWptetA5anUimfSqVeA9qlUqm3NzGDFDEVBykG\nn2U8XgnMz3oOcdfMdsCazC6gVCo1j/iX8PbAVun1n2e0n5PxeKf0z0/DMFwdhuFq4EsgIi4qm2I7\n4N2sZXPTOQDGALsD88MwvD8Mw+OBkvS6B4GPgXlhGD4ZhuFZxAVDpNp0zkGKQVTF83JNAbOB5Qbw\n6fXZX6gyn69Kv3aLVCq1rgY5s7NsiAdIpVIzwzDcDhgEHErcNTY8DMMBqVRqMbBvGIZ9gMOIz7dc\nHIZh31Qq9Wktc0mR0JGDyA8+AhqHYRiWLwjDcCficxAfEh9xNArDsDSjTWY//ofE/6d2y2hvwjDc\ntoZZss8R9Ei/B2EYtgVIpVL/TKVSI4C9gH2AnmEYNg3DsFUqlZqeSqUuT+dZDhxdgxxSpFQcRH7w\nBvA2MCYMwzZhGLYj7r6ZDcwE/gN8A/w+DMPNwjDsQXxSGoBUKvUu8CIwNgzD0jAMmwGXAlPTjzfF\nXcBRYRgOCsOwURiGg4Cj0ssBXgeuDsOwRfpE+V7Ad8CnxEcRj6avnALoBrQDPtjEDFLEVBxE0lKp\nlAcOJ+5unUt8eegaYHD6xO5q4OfEVwF9Q/yL+pqslzkxve594AtgP+DgdNtNyfII8cnsccBiYCww\nLJVKPZbe5BigL7AA+BY4BzgqlUp9A1xAfK7j7TAMVxJfLTU6lUo9uSkZpLgZzQQnIiLZdOQgIiIV\nqDiIiEgFKg4iIlKBioOIiFRQSDfB6cy6iEjNVLj5s5CKA2VlZTVqV1paWuO2+aB8taN8taN8tdMQ\n8m2IupVERKQCFQcREalAxUFERCpQcRARkQpUHEREpAIVBxERqUDFQUREKiio+xzqio8iWLMaVq2C\n1SthVfrP6pX49E9WrYJVK2D1Kvj+ewiCH/8xWT83tK7CY5PRzkAQP1/evj3RkiU/XmcCzPrHJn0L\nS/o+FmMyHlPJssxtgfWj8/qM2wn9D+syl2VuC6z6tD1+8aL4NbPfp/x59nuWPzcGfrIlpv3mNfvH\nEpGcKPriEL30L75841XWLf32h1/6q1dm/AKsnZeXf89+LRtXud2tC1dzZNsmdG5c8WBucSVt6sst\n4V/X9gUaNyG46HrM1l3rIo6I1IGiLw58/AFrPpgDmzWHZptBh47QrDls1hyzWfP0481gsxbrH5v0\nesrXN24MUfTjP34dX3y5kBcnPMTAM0/PWr8OfASRT/+MGJHx3Jcv9x6iiHZt2rB48aL062a2y9iu\nJt/8swugqdmRR5vWrVmyZEnGe2e/r8/KWP7eHlauwD/7ONHfbiC45EZMk8qmThaRfCr64hAMPYvO\nF13DggUL6vy1bx7/V97/8CMOPOUMBg0axBdffMENN9zAddddx8KFC1m9ejVDhw6lf//+nH322Ywc\nOZKXXnqJFStW8Nlnn1FWVsbw4cM5ctBhLKnHt9+3Ki1lWS3yRWvX4l94Cv/w3ZgTzqjDZCJSU0VT\nHKKH78bPeG2D6xaUlLBu3bpNfk2zxz4Ex/y60vXHHXccjz32GF27dmXevHmMGzeOxYsXs+eeezJ4\n8GDKysq44oor6N+//4/affXVV4wZM4Zp06YxadIkjjzyyE3O1pCYIUPxH7yDf/FpfI/emJ59k44k\nUvR0tVKedOvWDYBWrVqRSqUYMWIEY8aMYenSpRW23XXXXQHo2LEjK1asyGvOJJgmTQmGnQuNGhPd\nMw7/7aKkI4kUvaI5cgiO+TVU8i0/H6MmNm4cn5R+7rnnWLp0KePGjWPZsmWcfvrpFbYtKSlZ/7hY\n5vg2W20bH0FM+BvR3TcTjLw8vhpLRBKRl+JgrR0L9CM+PTnSOTc9Y93+wDXAOiAFDHPORel1mwHv\nAFc55+7JR9a6ZIyp0F21dOlSOnfuTBAEvPzyy6xduzahdPWPOeAw/Dsz4Z0Z+OcmYQ76RdKRRIpW\nzr+aWWsHADs65/oDpwDjsja5HRjinNsHaAUMzlh3CdBg+xi6dOnChx9++KOuof3224+pU6dyzjnn\n0KxZMzp27Mi9996bYMr6wxhD8OuR0KoN/rH78PP+m3QkkaKVjyOHA4GJAM6596y17ay1rZ1z5Z3t\ne2Q8Xgh0ALDWdgN2AZ7KQ8acaNu2LQ899NCPlnXq1Ik777xz/fP/+Z//AeDkk08GoGvXH67179q1\nKzfddFMektYfpnVbgl+fTTTuyvTlrWMxTXV5q0i+5aM4dAJmZDxfmF62FKC8MFhrOwMHAZemt/sT\nMAI4ubpvVNmMRrlumw9Fla+0lMWfpFg+aQKbPT2B9sMvqoOXLKL9lwPKVzv1Pd+GJHFCusJcpdba\nLYAngDOdc99Ya08CpjrnPrbWVvuFNU1oMnKRzw8eAjOmsuLpR1jVNcTs3q/Gr1WM+68uKV/tNIR8\nG5KPy0HKiI8U1mcB1t9xZq1tDfwTuMQ5Nzm9+FDgCGvt68Aw4FJr7aA8ZJV6wjRuQnDqefHlrffe\ngv/2m6QjiRSVfBSHycAQAGttb6DMObcsY/2fgLHOuX+VL3DOHeuc6+Oc6wfcQXy10nN5yCr1iNmy\nC8b+BpYvI7rrpnhARBHJi5x3KznnplhrZ1hrpwARMNxaOxRYAjwDnATsaK0dlm7ygHPu9lznkobB\nDPxZfHnrW9Pxzz6OObiw7xYXqS/ycs7BOXdh1qLZGY83eimKc+6KOg8kDYYxhuDk3xJdeRb+sb/j\nu+2G6bJ90rFECp5uQZV6r/zyVtatJbrjBvx3q5OOJFLwVBykQTA9emMGHQ5fzMe7O6tuICK1ouIg\nDYY56mTYqiv+5WfwM6cmHUekoKk4SINhGjcmOPVcaNyE6L7x+MW6vFUkV1QcpEExpdvEl7euWEZ0\n11hd3iqSIyoO0uCYAYdAz77w/lv4yY8lHUekIKk4SINTfnkrbdrhJ96P/+TDpCOJFBwVB2mQTKs2\nBL85G9atI7rjRnwNpnkVkcqpOEiDZXbphem/P3w5H3T0IFKnVBykQTO77wWAnzMz4SQihUXFQRq2\nbj0hCPBzZiWdRKSgqDhIg2aat4DtQvj4Q/yK5UnHESkYKg7S4JnuvcBH8N6bSUcRKRgqDtLgme69\nAdS1JFKHVByk4euyPbRshZ8zC+990mlECoKKgzR4JijB7Lw7LP4aFnyWdByRgqDiIIVBXUsidUrF\nQQqC6b47QDylqIjUmoqDFATTtgNs2QU+nINf813ScUQaPBUHKRime2/4fg18MCfpKCINnoqDFAzT\nvReg8w4idUHFQQrHjrtAkyYaZ0mkDqg4SMEwjZvATrvCgs/wixYmHUekQVNxkIKiriWRuqHiIAWl\nfCgNVBxEakXFQQpLpy2hfUf8e7PxkWaHE6kpFQcpKMaYuGtp5XL4WLPDidSUioMUHI3SKlJ7jfLx\nJtbasUA/wAMjnXPTM9btD1wDrANSwDDnXGStvQ7YN53xGufco/nIKgVg593i2eHenQWHH590GpEG\nKedHDtbaAcCOzrn+wCnAuKxNbgeGOOf2AVoBg9MFo0e6zWDgplznlMJhmreMZ4f77weaHU6khvLR\nrXQgMBHAOfce0M5a2zpj/R7Ouc/TjxcCHYCXgWPSy74FWlhrS/KQVQqEZocTqZ18FIdOxL/0yy1M\nLwPAObcUwFrbGTgIeNo5t845tyK9ySnly/KQVQqEzjuI1E5ezjlkMdkLrLVbAE8AZzrnvslYfgRx\ncTioOi9cWlpa41C1aZsPyrdp/E9+QlmrNpj338J7X+/yZVO+2lG+upeP4lBGxpECUAosKH+S7mL6\nJ3Cxc25yxvKDgYuBwc65JdV6o7KyGgUsLS2tcdt8UL6a8d12I5r+Cms/+5iFjZolHadS9XX/lVO+\n2mkI+TYkH91Kk4EhANba3kCZc25Zxvo/AWOdc/8qX2CtbQNcDxzmnFuUh4xSiNJDaayeMTXhICIN\nT86PHJxzU6y1M6y1U4AIGG6tHQosAZ4BTgJ2tNYOSzd5IP1zc8BZa8tf6iTn3Lxc55XCYXbphQdW\nz3wd9to/6TgiDUpezjk45y7MWjQ743HTSprdnqM4UiRMu3h2uO/emYlZ8x2mSWUfNRHJpjukpaCZ\n7r3iaUM/fDfpKCINioqDFLQfhvDWBEAim0LFQQrbjt0xTZvqfgeRTaTiIAXNNG5C0x57QNk8/KKv\nk44j0mCoOEjBa9a7H0A8EJ+IVIuKgxS8ZnvsHT9Q15JItak4SMFrtFWXeHa4d9/U7HAi1aTiIAVP\ns8OJbDoVBykKGqVVZNOoOEhxyJwdTkSqpOIgRcE0bwldd9LscCLVpOIgRcN07x3PDvf+7Ko3Fily\nKg5SNH4YSkNdSyJVUXGQ4rHtDtCiFX7OTLz3SacRqddUHKRomKAEs3NPWPQ1fPF50nFE6jUVByku\nGqVVpFpUHKSomF103kGkOlQcpKiY9ptD6TbwwTv479ckHUek3lJxkKJjuveCNWvgwzlJRxGpt1Qc\npOiYHhpKQ6QqKg5SfHbsDk2a4N/RSWmRyqg4SNExjZvATj00O5zIRqg4SFFaf7e0BuIT2SAVBylK\n5UN4a3Y4kQ1TcZDi1GkraL+5ZocTqYSKgxSleHa43vHscJ/MTTqOSL2j4iBFS6O0ilROxUGKV7ee\nYAKNsySyASoOUrRMi5awXXp2OI3SKvIjjfLxJtbasUA/wAMjnXPTM9btD1wDrANSwDDnXLSxNiJ1\nJTjoF0R/GUN0zziCUddggpKkI4nUCzk/crDWDgB2dM71B04BxmVtcjswxDm3D9AKGFyNNiJ1wvTe\nG9NnX/joffyzk5KOI1Jv5KNb6UBgIoBz7j2gnbW2dcb6PZxz5cf0C4EO1WgjUmfM8adDqzb4iffj\nF6h7SQTy063UCZiR8XxhetlSAOfcUgBrbWfgIOBS4m6mSttUprS0tMYha9M2H5Svdjaer5SVv72Y\nb0aPotEDf2GL6+7AlOS3e6lh77/kKV/dy8s5hywme4G1dgvgCeBM59w31toq22xIWVlZjQKVlpbW\nuG0+KF/tVCtf126YPvuyZvorzL/vLwQHH5WfcBTI/kuQ8tVOZYUrH91KZcTf+tdnARaUP0l3F/0T\nuMQ5N7k6bURywfzydGjdFj/xH/gFnyUdRyRR+SgOk4EhANba3kCZc25Zxvo/AWOdc//ahDYidc60\nbE1w4pmw9nuiu2/WsBpS1HLereScm2KtnWGtnQJEwHBr7VBgCfAMcBKwo7V2WLrJA86527Pb5Dqn\nCIDp1Q/TdwB+2kv4yRMxg49OOpJIIvJyzsE5d2HWotkZj5tWs41IXpjjT8W/Pxv/+AP4nn0xnbdO\nOpJI3ukOaZEspmVrgl9ldC+tU/eSFB8VB5ENMLv3w+w1AD7+AD95YtJxRPKuWt1K1tqdgUOdczdY\na3sAtxGfCxjpnNOQllKQzPGn4d9/Cz/pH/iefTCl2yQdSSRvqnvkcDfwcfrxn4kvPf1j+rFIQTIt\nWqWvXlqr7iUpOtUtDm2cc4+kb1brCVzrnHuGeCwkkYJldt8L028gfPIhfvJjSccRyZvqFgdvrW0O\nHAdMds6ttdY2ppIrjUQKiTnuVGjTDj/pAfz8eUnHEcmL6haHW4HPgcuIu5MA/gHoq5QUPNOiFcGv\nhqe7l25S95IUhWoVB+fceGAboJNzrvwehT845y7IWTKResT07Ivptz98Ohf/r0eSjiOSc9UqDumr\nlc5Idyf1sNa+Ctxqrd09t/FE6o+4e6k9/okJ+PmfJh1HJKdqe7XSrbkIJVIfmRYt4+6ldemrl9au\nTTqSSM7oaiWRTWB69sH0T3cvPfNo0nFEckZXK4lsInPsqdA23b30+SdJxxHJCV2tJLKJTIuWBCeN\nUPeSFDRdrSRSA2bXPTF7HwjzPtLVS1KQqju2kgEOBw5Kn3f4EngKeCeH2UTqNXPsKfh338Q/OQHf\naUvMnj9NOpJInalut9L1wEhgJvAg8XwMF1lrL89VMJH6zjRvSXD6+dC4CdHtNxC9+mzSkUTqTHUn\n+zkE2MM5t7p8gbX2b8A04MpcBBNpCMwOuxCcezXRTVfg772FaPVKgkFHJB1LpNaqe+RQAnyXtWzV\nJrQXKVhm2x0Jzr8mvkHuoTuJnpiA9z7pWCK1Ut0jhxeBSdba24HFQAdgGPBCjnKJNChmy20IRl1D\ndOOl+EkPwKoVcMxvMMYkHU2kRqr7zf8s4FXgPOB24GzignF2bmKJNDxmi84EF1wLnbfGP/s4/u9/\nxkcapE8apo0eOVhrPwTKj48zvwJtBZwBnA7slJtoIg2PadeB4PzRRDddjn9lMqxeBb/5HaZRdQ/S\nReqHqj6xw/KSQqSAmFZtCM79I9EtV+Gnv4JfvYrgjAswTTSggDQcGy0OzrmX8hVEpJCY5i0Izr6S\n6NbR8PYbROP+QDDiYkyz5klHE6kWXW0kkiOmaVOCEZdA7/6QepvoxsvwK5YlHUukWlQcRHLING5M\ncNooTP8D4OMPiK7/Pf7bRUnHEqmSioNIjpmSEszQszD7HwrzPyW6/iL8N18lHUtko1QcRPLABAHm\n+NMwP7Pw1QKiay/Ef/F50rFEKqXiIJInxhiCI0/EHH0yLP6a6LqL8PP+m3QskQ1ScRDJs2Dw0ZgT\n/heWLyW64WK+e3d21Y1E8iwvd+ZYa8cC/YhvqBvpnJuesa4Z8Fegu3Nuz/SylsB9QDvi2eauTE9L\nKlIQgoGHEG3WHH/XWBZeciacOJyg38CkY4msl/MjB2vtAGBH51x/4BRgXNYm1wNvZi0bCqScc/sD\nQ4Cbc51TJN+CvQYQnHkxlDTC33kj0QO34dd+n3QsESA/3UoHAhMBnHPvAe2sta0z1v+eitONfk08\nuB/ERw9f5zqkSBJMzz785Kb7YMsu+Beejs9DLFqYdCyRvHQrdQJmZDxfmF62FMA5t8xa2yGzgXNu\ngrV2qLV2LnFxOLQ6b1RaWlrjkLVpmw/KVzv1Pd+W4+5n8S2jWfniP+GP59L+gtE0271v0rHWq+/7\nT/nqXhKjgVU5hrG19kRgnnNusLW2J3AnsGdV7crKymoUqLS0tMZt80H5aqch5Pti0WL8L8/AbNmF\naMIdLLxkBOaIX2IOGYIJkr1upCHsP+WrucoKVz4+dWXERwrrswALqmizD/AMgHNuNlBqrS3JTTyR\n+sEYQzDwZwSjroG27fET7ye6dTR+5fKko0kRykdxmEx8UhlrbW+gzDlX1QAzc4G90m26AMudcxoY\nX4qC2S4kuHQs7NwTZk8juvoc3Q8heZfz4uCcmwLMsNZOIb5SaXj6fMKRANbah4EJQGitfdFa+0vi\nS1u3tda+BDxAPHeESNEwrdoQnH1FfEf1wi+Ixowieu35pGNJEcnLOQfn3IVZi2ZnrDumkmY2d4lE\n6j8TlGCOPBG/XUh01434e24m+ui9eBiOxk2SjicFTndIi9RzpmcfgkvGwtZd8a9Mjsdl+vrLpGNJ\ngVNxEGkATMdOBBdeh9nnQPh0bnwe4p0ZVTcUqSEVB5EGwjRpijn5LMyvhsN3q4jG/YFo0oP4KEo6\nmhQgFQeRBsQYQ7DfwQQXXAvtO+KfeJBo3JX4JYuTjiYFRsVBpAEy2+4YX+7aYw+YM4vo8hFE019N\nOpYUEBUHkQbKtGhF8NtLMcefBt9/h7/9OqLbr8cvX5p0NCkAKg4iDZgJAoIDDiO4bBxs3w0//RWi\ny0fgZ09LOpo0cCoOIgXA/KSUYNQ1mCFDYeVyovFXE91zM37liqSjSQOl4iBSIExQQnDwUfE9Edts\nj3/teaIrf4t/N3u6FJGqqTiIFBizZReCi67H/Px4WLKYaOxl8URC361OOpo0ICoOIgXINGpEcPjx\nBBddD6XbxBMJXXkWfu67SUeTBkLFQaSAmS47EFxyI+bgo+DrL4muu4jo/+7Gf78m6WhSz6k4iBQ4\n07gJwZCh8TwRHTvhn3mM6Krf4T+dm3Q0qcdUHESKhNlhF4LLbsbsfygs+Ixo9HlEkx7Ar12bdDSp\nh1QcRIqIadqM4JenE5xzVTzb3BMTiK7+HdELT+GXfpt0PKlHVBxEipDZuSfB5bdg9hkEZfPwD/yV\n6PyhrBt7OdGU5/GrViYdURKWl8l+RKT+Mc1bYIaehf/FCfg3XsX/52V4dxb+3Vn4v98Ku+1J0Hc/\nog4/TzqqJEDFQaTImbYdMIOOgEFH4L8qw097BT/tZZg5lWjmVMruHQ+798X03Q923h1TUpJ0ZMkD\nFQcRWc9sUYo57Fj8oRY+/wQ/7WWCmVNYN/UF/NQXoFUbzB77xIVi+26YQD3ThUrFQUQqMMbA1l0x\nW3el8/ALKHv1Bfy0l/BvvIZ/8Wn8i09D+46YPvti+g3EbLVt0pGljqk4iMhGmSDA7LAzZoed8cee\nCu+/hf/PS/g3X8c/8yj+mUehd3+CX/wK03mrpONKHVFxEJFqMyUl0L0Xpnuv+C7rt98gmjwxPj/x\n5n8w+wzC/Px4TLsOSUeVWlJxEJEaMY2bQO+9CXr1hzf/Q/ToffhXJuP/8yLmwJ9jBh+Nad4y6ZhS\nQyoOIlIrxhjo1Y9gtz74Kc/jJz2I/+cj+JeewRx6DGb/Q+NCIg2KLjUQkTphSkoI9j2I4OrbMEed\nDHj8w3cTXXIG0WvP46N1SUeUTaDiICJ1yjRtSnDI0QSjb8ccfCQsXYK/52aiK0fiZ0/He590RKkG\nFQcRyQnTohXBkF8T/PG2eJiOBZ8Tjb+K6LqL8HPfSzqeVEHFQURyyrTvSDD0LIIrxsHue8Hcd4mu\nvYB1f/4jvmxe0vGkEjohLSJ5YUq3oWT4xfi57xI9cm98hdPs6Zi9D8AcajEdOyUdUTLkpThYa8cC\n/QAPjHTOTc9Y1wz4K9DdObdnxvITgFHAWuAy59xT+cgqIrlldtiFYNQYeGt6fPnra8/hp/4bs9dA\nzM+OwXTaMumIQh66lay1A4AdnXP9gVOAcVmbXA+8mdWmA3A58FPgMOCIXOcUkfwxxmB69iW4/GbM\nsHPhJ1vip/6b6LLhRH+7AT9f3U1Jy8eRw4HARADn3HvW2nbW2tbOuaXp9b8HOgAnZLQZBDznnFsG\nLANOy0NOEckzE5Rg9hqA77MvzHqd6MmH8NNejkeF7b03waEWs812SccsSvkoDp2AGRnPF6aXLQVw\nzi1LHylk2hZobq2dBLQDrnDOPV/VG5WWltY4ZG3a5oPy1Y7y1U5e8m01BH/Y0aye9gpLJ9zBmplT\niGZOoVnffWl9/DCa7tQ92Xy1UN/zbUgSJ6RNNbfpABwJdAFesNZ2cc5t9ALpsrKyGgUqLS2tcdt8\nUL7aUb7ayXu+rXfAn3cNwZxZRE89xOppr7B62ivQvRfBYcdidtgl2XybqCHk25B8FIcy4iOF9VmA\nBVW0+RKY4pxbC3xkrV0GdAS+yk1EEalPjDHQozdB916QepvoyYdgziyiObMg3JXgsGMh3DXeTnIi\nH8VhMnAl8FdrbW+gLH0uoao291hrryXuVmoJfJ3bmCJS3xhjoNtulHTbLb4EtrxIpN6GHXYmOPRY\nfOfOSccsSCYft7Jba8cA+wERMBzoBSxxzj1mrX0Y2BroTnxu4nbn3APW2tOJr24CuNo5N6mKt/Hq\nVkqG8tWO8m0a//EHRE85mD0NgCY77cLag4+GXfesl0cS9W3/ZUt3K1XYcXkpDnmi4pAQ5asd5asZ\nP++/cZGYOSVe0GUHgsOPr3dFor7uv3KVFQfdIS0iDZLZZjtK/vdCNl+zkq/uugU/4zWiW66Ki8TP\nj4fd6leRaGhUHESkQWuy7Q4EZ1yAn/8p/smH4iIxvrxIHAe79VGRqAEVBxEpCGbLLpjTR+Hnz8M/\n9RD+jVeJxl8N22wfF4mefVUkNoFGZRWRgmK23IbgtPMJrrgF02df+Oy/RH/+I9HVv8O/+brmk6gm\nFQcRKUimNLtIfEz059EqEtWk4iAiBW19kbhyPKbvfj8UiavOxs9SkaiMioOIFAXTeWuCU89LF4kB\n8PknRLeOJvrD2fjZ01Qksqg4iEhRiYvEuQRX/hmz1wCY/wnR+KuJxozCvzc76Xj1hoqDiBQl03kr\ngmHnElxxC/TeG/6bIrrxUtb96RL8R+8nHS9xupRVRIqaKd2Gkv+9EP/pXKKJ98M7M4nGjIKefQl+\ncQJmq65JR0yEioOICGC67EDJyCvwH8wheuzvMHsa0expmD77Yg7/ZdFNX6riICKSwezUnWDUNTBn\nJtFj9+Onv4Kf8Rpm7wMxhx1voBZLAAAMFklEQVSH6dAx6Yh5oeIgIpIlnk9iD4LuvWHmVKLH/4F/\n9Vn86y9g9huM+dkxmDbtko6ZUyoOIiKVMMbAHnsT9NoL//pL+CcexP/7Sfyrz2IOPAxz8FGYFq2S\njpkTKg4iIlUwQQlm7wPwffeNjyCedPh/PoJ/8V+Yg36B2f9QTIuWScesU7qUVUSkmkyjxgQDf0Yw\n+q+YIb+GkgD/+D+IRv2a6B+34b+Yn3TEOqMjBxGRTWSaNMUcfCR+v4PxLz+D//cT+Befxr/0T9it\nD8Ggwxv8HNcqDiIiNWQ2ax4XiUGH42dOwT/7+PpLYNm6K2bQ4fgjjk06Zo2oOIiI1JIpKYlHfu2z\nL/6j9/HPPo6fORV/982UTbwfv99gzIBDMK1aJx212lQcRETqkNm+G2b7bvivv4yvbHrtOfzj/8A/\n/TCm30DMoMMxpdskHbNKOiEtIpIDZvOfENhTKL33acyxw6BNO/wrk4kuH8G6my7Hz5lVr0eC1ZGD\niEgOBc1bEAw6HH/AofDmNKLnHoc5s4jmzILSbeIjif4HYBrVr1/H9SuNiEiBMkEJ9O5PSe/++E8+\nxD87CT/jVfx94/HPTSI47lTMzj2TjrmeupVERPLMbLtjPKfE6L9h9j0IFnwWDxf+lzH4r79MOh6g\n4iAikhjTfnOCk0YQXPwn2L4bzJxCdNlwokkP4td8l2g2FQcRkYSZLjsQXHAt5pTfQfOW+CceJLps\nOH7GlMROWqs4iIjUA8YYgn77E1x9K2bw0fDtIqLbxhDdeCl+/ry851FxEBGpR0yz5gRHnxxPX9pj\nD3j/LaI/nEU04W/4lcvzliMvVytZa8cC/QAPjHTOTc9Y1wz4K9DdObdnVrvNgHeAq5xz9+Qjq4hI\nfWA6bUnJyMvxb00neugO/PNP4P/zEuaokzD7HBhf/ZRDOT9ysNYOAHZ0zvUHTgHGZW1yPfBmJc0v\nARblMJ6ISL1mdutDcMV4zFEnw/dr8PeNJxp9Pv6j93P6vvnoVjoQmAjgnHsPaGetzRxg5PfAY9mN\nrLXdgF2Ap/KQUUSk3jKNGxMccjTBVX/B7DUAPp1LNGYU0Z1j8d/m5vtzPopDJ2BhxvOF6WUAOOeW\nVdLuT8A5OcwlItKgmHYdCIadS3DBGNhmO/zrLxBddiZ+0cKqG2+iJO6QrnKAc2vtScBU59zH1tpq\nv3BpaWmNQ9WmbT4oX+0oX+0oX+3Ueb7SUvw++7Ni8kRWz5hKu627UNKmbZ2+RT6KQxkZRwpAKbCg\nijaHAttZaw8DtgK+s9Z+7px7bqNvVFZWo4ClpaU1bpsPylc7ylc7ylc7Oc3Xsz/07M+XK1bCipU1\neonKClc+isNk4Ergr9ba3kDZRrqSAHDOrZ8dw1p7BfBJVYVBRETqTs7POTjnpgAzrLVTiK9UGm6t\nHWqtPRLAWvswMAEIrbUvWmt/metMIiKycXk55+CcuzBr0eyMdcdU0faKXGQSEZHK6Q5pERGpQMVB\nREQqUHEQEZEKVBxERKQCFQcREanAJDWRRA4UzF9ERCTPKoxckcTwGblS5bAcIiJSPepWEhGRClQc\nRESkAhUHERGpQMVBREQqUHEQEZEKVBxERKSCQrqUtUrW2rFAP+J7IkY656ZnrBsEjAbWAU87565K\nKON1wL7E/zbXOOcezVj3CfBZOiPACc65+XnMNhB4GJiTXvS2c+63GesT3YfW2lOAX2Us2tM51zJj\n/ffAaxnrD3TOrSPHrLU9gMeBsc658dbarYG/AyXEE1/9yjn3XVabSj+recp3N9AY+B440Tn3Rcb2\nA9nI5yAP+e4B9gC+SW9yvXPuqaw2Se6/h4GO6dXtgdedc6dlbD8UuAr4KL3oWefcH3OVr6aKpjhY\nawcAOzrn+ltrdwbuAvpnbDIOOBiYD7xkrX3EOfdunjPuD/RIZ+wAzAIezdrsEOfc8nzmyvKSc25I\nJesS3YfOuTuBO2H9v3f2HLNLnHMD85UnnaMFcAvwfMbiPwB/ds49bK0dDfwG+EtGm6o+q7nOdzVw\nu3POWWuHE8/lPiqr6cY+B7nOB3CRc+7JStokuv8ypyGw1t4F3LGBpg85587LRaa6UkzdSgcCEwGc\nc+8B7ay1rQGstdsBi5xznznnIuDp9Pb59jJQ/sH6FmhhrS1JIMcmq0f7sNxlxN/OkvYd8DPi6XLL\nDQQmpR8/AQzKalPpZzVP+c4EHkk/Xgh0yNF7V8eG8lUl6f0HgLU2BNo656bl6L1zqmiOHIjnsZ6R\n8XxhetnS9M+FGeu+ArbPX7RYuotjRfrpKcRdM9ndHrdZa7cFXiX+9pTvYUN2sdZOIj5cvtI592x6\neb3YhwDW2j7AZ5ldIWnNrLUPAF2AR5xzN+Y6i3NuLbDW2h8dxLTI6Eb6Cuic1Wxjn9Wc53POrQBI\nfzEZTnykk62yz0HO86WNsNaeQ7z/Rjjnvs5Yl+j+yzCS+KhiQwZYa/9F3HV3nnNuVl1nq61iOnLI\ntrHhNhIdisNaewRxcRiRteoy4kP8gUAP4Oj8JuND4vnAjwBOBu601japZNsk9+Ew4J4NLD8POA04\nCDjBWrtnPkNVojr7Ke/7Ml0Y/g782zmX3aWzKZ+DXPg7cKFz7gDgTeCKKrZPYv81AX7qnHthA6tf\nB65wzg0GLgHuy2u4aiqmI4cy4m8P5UqJTwZuaN2WbNphbJ2x1h4MXAwMds4tyVznnLsvY7ungV2B\n/8tXtvTJ74fSTz+y1n5BvK8+ph7tQ+LiWeEEqXPutvLH1trnifffG/mLtd5ya+1mzrlVbHg/beyz\nmi93Ax86567MXlHF5yDnsorVJDLO16TVh/03ANhgd5Jz7n3g/fTjqdbajtbaknxcHLEpiunIYTIw\nBMBa2xsoc84tA3DOfQK0ttZua61tBByW3j6vrLVtgOuBw5xzi7LXWWufyfiGNgB4J8/5TrDWnpd+\n3An4CfHJ5/q0D0uB5c65NVnLQ2vtA9Zak863Dz9cbZNvz/HDUd/RwL+y1lf6Wc0Ha+0JwBrn3OWV\nra/sc5CnfI+kz3FB/EUg+/9BovsvrQ8we0MrrLWjrLXHpx/3ABbWt8IAhTVkd5WstWOA/YCIuC+1\nF/EVLI9Za/cDrk1v+ohz7oYE8p1GfIj8QcbifxNfKviYtXYk8WH8KuIrmX6bz3MO1tpWwANAW6AJ\ncdfCFtSvfbgHcLVz7pD08wuJr6yZaq29FjiA+N9/Uj4uH0zn+ROwLfFlofOBE4i7vZoBnwK/ds59\nb62dkH68Kvuz6pzb4C+aHOXbAljND3307zrnzizPR9zj8KPPgXPu6TzmuwW4EFgJLCfeZ1/Vo/13\nFPH/jVedcw9lbPu4c+4Ia+1WxF1jAfG+/F19PGldVMVBRESqp5i6lUREpJpUHEREpAIVBxERqUDF\nQUREKlBxEBGRClQcROoBa+1Aa+3cpHOIlFNxEBGRCnSfg0g1pMe7uhpoAcwFfgncACwGdgd2Ih7s\n7Tjn3Epr7W7Ewzp0IL6h7ALn3DPp17oAOB1YCzwJnEt8x/sdxMNWnEh8c9mwSsbmEck5HTmIVCE9\nVMPfgeOdc9sBLwDl4zQdSTxUw9ZAG+BUa20ATADGO+e6EQ8E+KC1tpW19qfp5z2JB0/8abo9wFbE\nd8PvTFxYLsnH309kQ1QcRKo2GHjROVc+hs9twOHEM7k97pz7Jj2HxURgb6Ar8cBvEwCcc28QD5PR\nh3js/6ecc8vS4z8N5IcJnZY658rneZhFXCxEElFMo7KK1FRbYD9r7fsZy5YQdxllnkReDLQjniLy\n26xxrxYTj1m0ORmjsDrnVgKk5wPInG9gHXHxEUmEioNI1cqA57KnxUzPZbx5xqL2wCLgS6C9tdZk\nFIgO6eVfZ7ZJTwcrUu+oOIhU7RngWmvtds65/1pr+xKPrAow2FrbFlgG/IJ4noNPgM+BY4EJ1tq9\nibuZphGPqOustZem20wkPr+Q1NwXIhukcw4iVXDOLQBOBR6z1r4HjOeHyW6eJz5n8Dlx19Fd6aOF\n44insnwPGAcc45xb4Zx7nXjOjjeBd4GZwIP5/PuIVIcuZRWpoXS30lzn3NVJZxGpazpyEBGRClQc\nRESkAnUriYhIBTpyEBGRClQcRESkAhUHERGpQMVBREQqUHEQEZEK/h9dZsp7h55DXQAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e9a468c50>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history128_3Layer.history['loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WaL8caglGnPS"
   },
   "source": [
    "#### Save LSTM stacked 3L model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "noaDaGf4GZa4"
   },
   "outputs": [],
   "source": [
    "model3L.save('LSTM3L_Model.h5')  # creates a HDF5 file 'my_model.h5'\n",
    "del model3L # deletes the existing model\n",
    "\n",
    "# returns a compiled model\n",
    "# identical to the previous one\n",
    "model3L = load_model('LSTM3L_Model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "j6X4CT7MG496"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.download('LSTM3L_Model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4eN9Er5JIJIy"
   },
   "source": [
    "### CNN + LSTM\n",
    "-  From Dimensional Sentiment Analysis Using a Regional CNN-LSTM Model paper https://www.aclweb.org/anthology/P/P16/P16-2037.pdf\n",
    "-  the  overall  framework  of  the  proposed  regional  CNN-LSTM  model.  First,  the  word  vectors  of  vocabulary words  are  trained from  a  large  corpus  using  the word2vec toolkit. For each given text, the regional CNN model uses a sentence as a region to divide the given text into R regions,  i.e. r1,...,ri rj, rk,..., rR.In  each region, useful  affective features  can  be  extracted once  the  word  vectors  sequentially pass  through a  convolutional  layer  and  maxpooling  layer.  Such local  (regional) features  are  then  sequentially integrated across  regions  using  LSTM to build a text vector for VA prediction.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 989,
     "output_extras": [
      {
       "item_id": 43
      },
      {
       "item_id": 246
      },
      {
       "item_id": 421
      },
      {
       "item_id": 594
      },
      {
       "item_id": 768
      },
      {
       "item_id": 937
      },
      {
       "item_id": 1107
      },
      {
       "item_id": 1273
      },
      {
       "item_id": 1444
      },
      {
       "item_id": 1613
      },
      {
       "item_id": 1693
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 456687,
     "status": "ok",
     "timestamp": 1521562345763,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "wTTwdPcum_oA",
    "outputId": "ff35584d-67b7-4681-f8fb-9f6af197a620"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_3 (Conv1D)            (None, 100, 100)          30100     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 50, 100)           0         \n",
      "_________________________________________________________________\n",
      "lstm_6 (LSTM)                (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 110,601\n",
      "Trainable params: 110,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Epoch 1/20\n",
      "12928/27636 [=============>................] - ETA: 13s - loss: 0.6715 - acc: 0.591727636/27636 [==============================] - 24s 859us/step - loss: 0.6407 - acc: 0.6275\n",
      "Epoch 2/20\n",
      "27636/27636 [==============================] - 23s 829us/step - loss: 0.5648 - acc: 0.7014\n",
      "Epoch 3/20\n",
      "22144/27636 [=======================>......] - ETA: 4s - loss: 0.5349 - acc: 0.729927636/27636 [==============================] - 23s 822us/step - loss: 0.5337 - acc: 0.7289\n",
      "Epoch 4/20\n",
      "27636/27636 [==============================] - 23s 814us/step - loss: 0.5134 - acc: 0.7439\n",
      "Epoch 5/20\n",
      "24832/27636 [=========================>....] - ETA: 2s - loss: 0.4925 - acc: 0.761427636/27636 [==============================] - 23s 815us/step - loss: 0.4921 - acc: 0.7616\n",
      "Epoch 6/20\n",
      "27636/27636 [==============================] - 23s 815us/step - loss: 0.4725 - acc: 0.7707\n",
      "Epoch 7/20\n",
      "25344/27636 [==========================>...] - ETA: 1s - loss: 0.4600 - acc: 0.778627636/27636 [==============================] - 23s 814us/step - loss: 0.4594 - acc: 0.7792\n",
      "Epoch 8/20\n",
      "27636/27636 [==============================] - 23s 821us/step - loss: 0.4533 - acc: 0.7832\n",
      "Epoch 9/20\n",
      "25600/27636 [==========================>...] - ETA: 1s - loss: 0.4285 - acc: 0.798827636/27636 [==============================] - 23s 822us/step - loss: 0.4285 - acc: 0.7989\n",
      "Epoch 10/20\n",
      "27636/27636 [==============================] - 23s 825us/step - loss: 0.4093 - acc: 0.8106\n",
      "Epoch 11/20\n",
      "25344/27636 [==========================>...] - ETA: 1s - loss: 0.3952 - acc: 0.819427636/27636 [==============================] - 23s 822us/step - loss: 0.3955 - acc: 0.8197\n",
      "Epoch 12/20\n",
      "27636/27636 [==============================] - 23s 822us/step - loss: 0.3748 - acc: 0.8321\n",
      "Epoch 13/20\n",
      "25472/27636 [==========================>...] - ETA: 1s - loss: 0.3595 - acc: 0.838927636/27636 [==============================] - 23s 823us/step - loss: 0.3617 - acc: 0.8366\n",
      "Epoch 14/20\n",
      "27636/27636 [==============================] - 23s 825us/step - loss: 0.3563 - acc: 0.8423\n",
      "Epoch 15/20\n",
      "25472/27636 [==========================>...] - ETA: 1s - loss: 0.3298 - acc: 0.855927636/27636 [==============================] - 23s 823us/step - loss: 0.3295 - acc: 0.8561\n",
      "Epoch 16/20\n",
      "27636/27636 [==============================] - 23s 826us/step - loss: 0.3075 - acc: 0.8680\n",
      "Epoch 17/20\n",
      "25216/27636 [==========================>...] - ETA: 1s - loss: 0.3048 - acc: 0.871027636/27636 [==============================] - 23s 824us/step - loss: 0.3038 - acc: 0.8717\n",
      "Epoch 18/20\n",
      "27636/27636 [==============================] - 23s 824us/step - loss: 0.2863 - acc: 0.8782\n",
      "Epoch 19/20\n",
      "25472/27636 [==========================>...] - ETA: 1s - loss: 0.2510 - acc: 0.896027636/27636 [==============================] - 23s 822us/step - loss: 0.2565 - acc: 0.8938\n",
      "Epoch 20/20\n",
      "27636/27636 [==============================] - 23s 819us/step - loss: 0.2450 - acc: 0.8988\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Conv1D,MaxPooling1D\n",
    "modelCNNLSTM = Sequential()\n",
    "\n",
    "modelCNNLSTM.add(Conv1D(filters=100, kernel_size=3, input_shape=(X_a.shape[1], X_a.shape[2]), padding='same', activation='relu'))\n",
    "modelCNNLSTM.add(MaxPooling1D(pool_size=2))\n",
    "modelCNNLSTM.add(LSTM(100))\n",
    "modelCNNLSTM.add(Dense(1, activation='sigmoid'))\n",
    "modelCNNLSTM.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "print(modelCNNLSTM.summary())\n",
    "HistoryCNNLSTM = modelCNNLSTM.fit(X_train, y_train, epochs=20, batch_size=128)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 16
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4550,
     "status": "ok",
     "timestamp": 1521562353885,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "HjhMVK0BHrO4",
    "outputId": "f7ccff3d-d885-4262-aa12-73db4bd7415a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4878/4878 [==============================] - 4s 834us/step\n"
     ]
    }
   ],
   "source": [
    "scoreCNNLSTM = modelCNNLSTM.evaluate(np.array(X_val),y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 542,
     "status": "ok",
     "timestamp": 1521562355449,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "TpKSQATpJ7g4",
    "outputId": "10f29d0e-2f51-43cb-86fd-0493d544ec8d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.6485775013398127, 0.7712177122259981]"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scoreCNNLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 298,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 565,
     "status": "ok",
     "timestamp": 1521562384235,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "_iaxD0kNJ87r",
    "outputId": "255ed438-ca80-4a99-cde0-7f5b41d81237"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEZCAYAAAB8culNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xm8VfP+x/HXd50maSaOE1FkK3OG\nW4bmS+Yb+Vz35pJrVmToInPGCJHhmq7hDoaPKSEkklJRoV/IlrF0RBqUkGp9f3+sVbZzTp3TOWfv\ndYbP8/HwsPda67v3+yzb+Zz1/a79/TrvPcYYY0ymIOkAxhhjqh4rDsYYY4qx4mCMMaYYKw7GGGOK\nseJgjDGmGCsOxhhjirHiYEwFpFKpbqlUyqdSqc3LeLxPpVJ917PvjVQqdWflJjSmfKw4GGOMKcaK\ngzHGmGLqJB3AmGxKpVIe+BtwJrAn8CFwLPAPoB+wAjg3nU4/GR+/FXA70BXYFJgMDEqn07Pj/R2B\n+4D2wAfAQ0XebytgJHAg0AiYBAxMp9OfliP7icBgYHtgIXAPMCydTvtUKtUSuBvoDjQAZgMXptPp\n8alUKgBuiH++FkAhMCKdTt+1sRlM7WVXDqY2OAc4HmgD5ANvAhOBLYDRRL/M13oGqEv0y78A+A4Y\nnUqlgviX7tPAdGBzoD8woMh7PQf8DKSArYB58WtulFQq1ZuoGAwGmhAVuCHxvwGui7e3BZoBjwD/\nTaVSdYDjgBOAA4gK3InA9alUateNzWFqLysOpjZ4Ip1Of5FOp78FpgBL0un04+l0+lfgeSA/lUo1\nSqVSuwOdgH+k0+nF6XR6GXAJsAOwN7APsB1wbTqd/jm+mvjX2jeJryr2idv/kE6nlxP9ct85lUrt\nvZGZTweeTqfTr6TT6dXpdPpN4CmiX/wQFYRfgZ/i/XcCW6fT6dXxvjXAj+l02qfT6beA5ul0etZG\nZjC1mBUHUxvMy3j8EzC/yHOIumbaAr9mdgGl0+m5RL+Etwe2jvd/ndH+w4zHO8b//iqVSv2SSqV+\nAb4FQqKisjHaAh8V2fZpnANgGLAHMD+VSv03lUr9BciL9z0GfAHMTaVSL6RSqXOICoYxZWZjDqY2\nCEt5vlZ9wJWw3QE+3l/0D6rM5z/Hr71pOp1eU46cRbOUxAOk0+l3U6lUW6AXcBhR19iAVCrVNZ1O\nLwEOTKVS+wCHE423XJpKpfZNp9NfVTCXqSXsysGY33wG1E2lUqm1G1Kp1I5EYxBziK446qRSqYKM\nNpn9+HOI/p/aLaO9S6VS25UzS9Exgl3i9yCVSjUDSKfTL6XT6YHAH4D9gd1TqVT9VCrVOJ1OT0un\n01fGeX4EjilHDlNLWXEw5jfTgVnAsFQq1TSVSjUn6r6ZCbwLvA0sAi5JpVKbpFKpXYgGpQFIp9Mf\nAW8AI1KpVEEqlWoAXA5MiR9vjAeBo1OpVK9UKlUnlUr1Ao6OtwNMBa5NpVKbxgPlfwBWAl8RXUU8\nE985BbAT0Bz4ZCMzmFrMioMxsXQ67YEjibpbPyW6PfRXoHc8sPsLcATRXUCLiH5R31DkZY6P930M\nLAC6AAfHbTcmy9NEg9kjgSXACOCUdDr9bHzIscC+wDfAUuB84Oh0Or0IuIhorGNWKpX6iehuqevT\n6fQLG5PB1G7OVoIzxhhTlF05GGOMKcaKgzHGmGKsOBhjjCnGioMxxphiatKX4Gxk3RhjyqfYlz9r\nUnGgsLCwXO0KCgrK3TYXLF/FWL6KsXwVUx3ylSQnxUFERhBNaOaBQao6LWPfNkRzwdQD3lXVM0Sk\nG/Akv81bM0tVz85FVmOMMTkoDiLSFWinqp1FpD3RF4c6ZxxyC3CLqj4rIneJSOt4+wRVLXE5RWOM\nMdmViwHpnsAoAFWdDTQXkSYAIhIQLYoyOt4/QFXn5iCTMcaYDchFt1I+MCPj+cJ42zKgJbAcGCEi\nHYGJqjokPq6DiIwmWslqqKq+moOsxhhjSGZA2hV53IpoWcYvgRdF5DDgfWAooETz2o8XkR1U9dcN\nvfD6BlbKoiJtc8HyVYzlqxjLVzFVPV9JclEcComuFNYqIJosDOB74CtV/QxARF4DdlbVF4En4mM+\nE5EFREXkiw2+kd2tlAjLVzGWr2IsX8Wsr3DlYsxhLNAXIO46KlTV5QCquhr4XETaxcfuBaRFpJ+I\nDI7b5ANb8vvVu4wxxmRR1ouDqk4GZojIZOLVqkSkv4j0iQ85F3go3v8D0Zq+o4GuIjKRaMH2M0vr\nUiovP2MyK8bZTMbGGJMpJ2MOqnpxkU0zM/Z9SjQ/fqblRPPmZ1044SUWp2cRXH8/brOWlf76EyZM\noGvXrqUed+edd3LMMcew1VZblXqsMcZkW62fW8n9oSuEIX7CS5X+2gsWLOD1118v07EDBw60wmCM\nqTJq1PQZ5eH2ORD3zCOEE8fijzgOV7depb32bbfdxscff0yPHj3o1asXCxYs4Oabb+amm25i4cKF\n/PLLL/Tv35/OnTtz7rnnMmjQICZMmMCKFSuYN28ehYWFDBgwgD59+pT+ZsYYU4lqTXEIn3wIP+Ot\nEve51avhpxWEF54E9Tcp82u6vfYnOPak9e4/7rjjePbZZ2nTpg1z585l5MiRLFmyhL333pvevXtT\nWFjIVVddRefOnX/X7rvvvmPYsGG88847jB492oqDMSbnak1x2JBgk01Z89MK+PnnjSoOG2OnnXYC\noHHjxqTTaV544QWCIGDZsmXFjt11110BaNmyJStWrMhKHmOM2ZBaUxyCY0+C9fyVX1BQwLxLB8D7\nbxOc9g9c21Slv3/dunUBGDduHMuWLWPkyJEsX76c008/vdixeXl56x7bGt/GmCTU+gHptYIehwPg\nx79Yaa/pnGPNmjW/27Zs2TK22morgiDgzTffZPXq1ZX2fsYYU1msOKy1026QvzV++iT8siWV8pLb\nbrstc+bM+V3XUJcuXZgyZQrnn38+DRo0oGXLljzyyCOV8n7GGFNZXA3qtvAVnT4jHP8i/tF7cX86\nnuAwqeR45Vcdvn5v+crP8lWM5auYePqMYivB2ZVDBte5OzTYBP/GS3jr7jHG1GJWHDK4Bg1x+/WE\npYtg5ttJxzHGmMRYcSjCdTsUgPD1yhuYNsaY6saKQxFuq62hwx7wyQf4r79MOo4xxiTCikMJgu6H\nAZV7W6sxxlQnVhxKstvesNkW+Klv4Ff8mHQaY4zJOSsOJXBBHq77ofDrSvzk15KOY4wxOWfFYT3c\n/r2gbj38+BfxYZh0HGOMyamczK0kIiOAToAHBqnqtIx92wCPAfWAd1X1jNLa5IJr1AS3bxf8W+Pg\nw3dh171z+fbGGJOorF85iEhXoJ2qdgZOJloqNNMtwC2qui+wRkRal6FNTrge0cC03dZqjKltctGt\n1BMYBaCqs4HmItIEQEQC4ECiNaNR1QGqOndDbXLJtd4edmgPH8zAf1d1v/5ujDGVLRfFIR9YmPF8\nYbwNoCXRetEjRGSSiNxQhjY55dbd1lr5y4gaY0xVlcR6Dq7I41bA7cCXwIsiclgpbdYrnkCqXNbX\n1h92DIVPPYSf8hr5Z1xAsEnDcr9HRVTkZ8sFy1cxlq9iLF/ly0VxKOT3f/UXAN/Ej78HvlLVzwBE\n5DVg51LarP+NKjgr6/r4A/6If/5xCkc9TtC1d7neoyKqw6yOlq/8LF/FWL6KWV/hykW30ligL4CI\ndAQKVXU5gKquBj4XkXbxsXsB6Q21SYLr0hvy8qLbWmvOFOfGGLNeWS8OqjoZmCEik4nuOhogIv1F\npE98yLnAQ/H+H4DnS2qT7Zwb4pq1wHXcD+Z/BZ98mGQUY4zJiZyMOajqxUU2zczY9ylwQBnaJMr1\nOAw/bSLh6y+Ql9ol6TjGGJNV9g3pstq+PWzTBt6fil+8sPTjjTGmGrPiUEbOOVyPwyEM8RNeSTqO\nMcZklRWHjeD27QKbNsZPfAW/alXScYwxJmusOGwEV68+7oBesPwH/PRJSccxxpisseKwkVzXQ8A5\nWwjIGFOjWXHYSK5lPuy2D3zxCf6LT5KOY4wxWWHFoRyCHraMqDGmZrPiUB477Q75rfDTJuKXLU06\njTHGVDorDuXgggDX7TBYvRo/cWzScYwxptJZcSgnt18PqL8JfsLL+DVrko5jjDGVyopDOblNGuL2\n6w5LvoeZbycdxxhjKpUVhwpYuxBQOHYUfvXqhNMYY0zlseJQAW6rbWCPP8BnHxPePxy/2r41bYyp\nGaw4VFBw8vmQ2hXenUL4z2H4Vb8mHckYYyrMikMFuQabEJx9BXTYE/5vGuGd1+F/XZl0LGOMqRAr\nDpXA1a9PMPBS2HVv+Og9wpFX41f+knQsY4wpt5ws9iMiI4BOgAcGqeq0jH1fAvOAtfeD9gPaAU8C\na5ddm6WqZ+cia3m5uvUIzhpCeN9weG8q4W1XEZxzBW6ThklHM8aYjZb14iAiXYF2qtpZRNoDDwKd\nixx2iKr+mNGmHTBBVftmO19lcnXqEpx2If7BEdGqcSOuIDj3KlzDRklHM8aYjZKLbqWewCgAVZ0N\nNBeRJjl430S4OnVwp5yP69QdvviE8NYr8CuWJx3LGGM2Si66lfKBGRnPF8bblmVsu0dEtgMmAUPi\nbR1EZDTQAhiqqq/mIGulcEEenHQO1K2LnziW8OZLCc6/Bte4adLRjDGmTHIy5lCEK/L8CuBlYDHR\nFcYxwBRgKKBAW2C8iOygqhu8T7SgoKDcoSrSdn38hdeytElTfnzxSYLbrmSL6+4mr8Xm5XqtbOSr\nTJavYixfxVi+ypeL4lBIdKWwVgHwzdonqvrvtY9FZAywq6o+BTwRb/5MRBYArYAvNvhGhYXlClhQ\nUFDutqXxRx2PW/krq8c9R+HgkwkuuBbXfLMqk68yWL6KsXwVY/kqZn2FKxdjDmOBvgAi0hEoVNXl\n8fOmIvKKiNSLj+0KfCAi/URkcHxMPrAlMD8HWSudcw4nf8cdcgx8O59w+BD8ou+SjmWMMRuU9eKg\nqpOBGSIyGRgJDBCR/iLSR1V/AMYAU0XkLaLxiKeA0UBXEZkIPAecWVqXUlXmnMP1OQF3xHGwcAHh\nTUPw331TekNjjElITsYcVPXiIptmZuy7Hbi9yP7lwBHZzpVLzjnckX8lrFMX/+x/CIdfQnDBNbj8\nrZOOZowxxdg3pHMsOPRY3LF/h6WLCIdfgi+cm3QkY4wpxopDAoKD/oT76+mwbCnhzZfiv97gOLsx\nxuScFYeEBN0Pw/1tAPy4jHDYRYTjRuNDW1HOGFM1WHFIUNDlYILT/gF16uKfeIDwhgvxcz9POpYx\nxlhxSJrb+wCCa+7GdeoGX84hvO58wqcexq+0ab+NMcmx4lAFuMZNCU4+n+DcodCiJf6VZwivGoj/\n8L2koxljaikrDlWI23lPgqvuxB18NCxeSHjblYQP3MKapYuTjmaMqWWsOFQxrn59gr79CS69FbZr\nh397AgvOOJbwrdfw3icdzxhTS1hxqKJc67YEQ27CHXcqftWv+IdvJ7zlMvy3VXeOFmNMzWHFoQpz\nQR5BzyPI/+eTsNs+kJ5FeNXZhC8qfvWqpOMZY2owKw7VQJ0t8gkGXkZwxkWwaSP8qP8SXnMe/rOP\nk45mjKmhrDhUE8453F77E1x9F65rbyicS3jjRYT/uwf/04qk4xljahgrDtWMa9iI4PizCC4cBvlb\n498YQ3jlAPz/TUs6mjGmBrHiUE25dh0ILr8Nd+RfYfkywjuuIXxwBH7Fj0lHM8bUAFYcqjFXty7B\nEccRXHYrbLsDfsr46Cri/beTjmaMqeasONQAbuvtCIYMxx19AqxYTnjXdYQP3IL/cVnS0Ywx1VRO\nFvsRkRFAJ8ADg1R1Wsa+L4F5wNopSfup6vwNtTHFubw83CF98bvvS/jwSPzbE/CzZxL0OwPXcb+k\n4xljqpmsXzmISFegnap2Bk4mWiq0qENUtVv8z/wytjElcAWtCS66Ede3P/y0gvCfwwjvG45f/kPS\n0Ywx1UguupV6AqMAVHU20FxEmmShjYm5vDyCg48muOJ22H4n/LSJhFcOxE+flHQ0Y0w1kYtupXxg\nRsbzhfG2zA7xe0RkO2ASMKSMbUwp3FZbE1x4A37c89EX5+69CaZPIvjrGbgmzZKOZ4ypwnIy5lCE\nK/L8CuBlYDHR1cIxZWhTooKCgnKHqkjbXKhQvv5nseqPh7H49mv4dcZkmPMhzc64kE26HIRzZTq1\n2c2XA5avYixfxVT1fCXJRXEoJPqrf60C4Ju1T1T132sfi8gYYNfS2qz3jQrLNyldQUFBudvmQqXk\nc3Xx51yFG/8i4TOPsOimS2Hs8wTHn4lr2jz5fFlk+SrG8lVMdchXklyMOYwF+gKISEegUFWXx8+b\nisgrIlIvPrYr8MGG2pjyc0FA0PMIgitHwo47w/tTCa8YQDhlvE0Hboz5nawXB1WdDMwQkclEdx0N\nEJH+ItJHVX8AxgBTReQtorGFp0pqk+2ctYnbooDggutwfz0d1qzGPziC8NbL8d98nXQ0Y0wV4WrQ\nX4zeupU2nl+4gPCx+2DWdMirgzv4aNxhx+Lq1a8S+SqD5asYy1cx1SEfJYzr2jekaznXMp/g7MsJ\nzhwCTZrhx2h02+us6UlHM8YkyIqDiaYD79g5mg78oD7R+tUjr2bNP4fhF3+fdDxjTAKsOJh1XINN\nCI49ieDy22CH9vDuZMIrziIcOwq/Zk3pL2CMqTGsOJhi3NbbEfzjBtyJZ0PduvgnHyS89jz8p7OT\njmaMyRErDqZELggIDvgjwTX/xB3wR/j6y2jluX/fabO9GlMLWHEwG+QaNSE48WyCi4ZBq23xE8cS\nXn4m4Vvj8GGYdDxjTJZYcTBl4nboQHDZCNyxJ8GqVfiHRxIOvwQ//6ukoxljssCKgykzV6cOwUF9\nCK6+Czp2hk8/IrzmXJbcdwt+4YKk4xljKlESE++Zas61aEnemUPws6YTPnovPz73GDz3GOy8J0G3\nQ2DXfXB5eUnHNMZUgBUHU25u170Jrt6NZp99xOLnHoMP3yP88D1othnuwIOif5pvlnRMY0w5WHEw\nFeLq1mPTHofyw0574L/+Ej/hZfzU8fjnH8O/+ATsvi9B10Og/e64wHoxjakurDiYSuO23g7X7wz8\nMSfi33kTP+EleG8q4XtToWU+rmtv3H49cY2bJh3VGFMKKw6m0rkGm+C6HIw/8CD4cg5+wkv4aRPx\nTz2MH/VfXMf9cd0OgR3aV9piQ8aYymXFwWSNcw7a7IhrsyP+2JOj7qYJL+PfmYB/ZwIUtI6uJjp1\nwzVslHRcY0wGKw4mJ9ymjXA9j8D3OBw++TC6mnh3Cv6x+/CP3w9bFOBat4Vt2uK2aQOt29o618Yk\nyIqDySnnHKR2waV2wS9bin9rHP7D92Du5/hpE2HaRNatMNK0RVQktmmLa90GtmkLm29pA9vG5ECZ\nioOItAcOU9WbRWQX4B4gBAap6ntlaD8C6AT4uM20Eo65Aeisqt1EpBvwJPBhvHuWqp5dlqym+nBN\nmuEO6QuH9I2WKV30XVQk5n2On/cFzP0cZk3Hz5r+W8FosAls0wa3Tdu4cLSJuqfq1E3yRzGmxinr\nlcNDwPD48V3AS8D0+PF+G2ooIl2BdqraOS4yDwKdixzTAegCrMrYPEFV+5Yxn6nmnHOw+ZbRlUHH\n3z4efvkyyCgWft7n8OnH+DkfRfsBmjQjOOUCXPvdkwlvTA1U1uvzpqr6tIhsAewO3KiqrwCNy9C2\nJzAKQFVnA81FpEmRY24BLi1jFlOLuMZNcB32IDi4D8GpF5B39V0EdzxBMGQ47vizcAceBCt+JBxx\nJeGLapMBGlNJynrl4EWkIXAcMFZVV4tIXaAsCw3nAzMyni+Mty0DEJH+wATgyyLtOojIaKAFMFRV\nXy3tjeK1UMulIm1zwfIV0aYN0B2AlR9/wKJhF7Fm1H+pX/gVm50/lKDx7//+sPNXMZavYqp6vpKU\ntTjcDXxNNM7QM972P+DZcrznuhvbRaQFcBLQC2iVccwcYCigQFtgvIjsoKq/buiFy7uId3VYANzy\nbUCTFvght8ADN/PLOxOZP/AvBGdejGu9fdXIVwrLVzGWr2LWV7jK1K2kqncCrYF8VZ0Zb75aVS8q\nQ/NCoiuFdVmAb+LHPYCWwESiQtNRREao6nxVfUJVvap+Bizg98XDmN9xjZsQDLoSd/if4ftvCW+4\nkHBSqRebxpj1KFNxiAeSz4i7k3YRkUnA3SKyRxmajwX6xq/TEShU1eUAqvqUqnZQ1U5AH+BdVT1P\nRPqJyOC4TT6wJTB/o386U6u4II/gqH4EZ18O9erjH7mD8OGRhCt/STqaMdVOWQekHwK+iB+vvVvp\nOqLupg1S1cnADBGZDIwEBohIfxHps4Fmo4GuIjIReA44s7QuJWPWcrvtQ3DZrdB6e/xb4/hu8N9t\nvQljNlJZxxyK3q3UM76KuLksjVX14iKbZpZwzJdAt/jxcuCIMmYzphjXMp/g4hvxj93Hqolj4drz\nCP5+Hm73fZOOZky1UNYrh4rcrWRMIlzdegQnDKT5uVfAqlWEd15L+Ox/8OGapKMZU+UlcbeSMTnV\n6I9H8kPjFoT3DMOPeRL/xSfRl+Zs7iZj1isXdysZkzjXum00DrH7vjB7JuE15+E/+zjpWMZUWWWd\nW8kBRwIHxeMO3wIvAh9kMZsxlco1bERw1iX4V57BP/tfwuFDcMeejOtxmK0rYUwRZR1zGA4MAt4F\nHiMaUB4iIldmK5gx2eCCgOCQvgTnDYWGjfCP34e/bzh+2dKkoxlTpZR1zOEQYC9VXXfDuIjcD7xD\n9E1mY6oV1353gstvI7z3Rvz0SfgP38Md1Q/X7RBcXl7S8YxJXFmvHPKAlUW2/bwR7Y2pclzzzQj+\ncQPuuNMA8I/fR3jt+fhPP0o4mTHJK+uVwxvAaBG5D1gCbAacAozPUi5jcsLl5eF6Ho7fZ3/80//G\nT36N8MaLcZ274/r2xzVpnnREYxJR1uJwDnAeMJhoLqRvgReAO7KUy5icck2a404ahD/wIMJH78FP\nGY9//+24q+lQ62oytc4Gi4OIzIF1i3Bl3s6xNXAGcDqwY3aiGZN7bof2BJfdip/wCn7Uf/CP34+f\n9CrBX07H7bhz0vGMyZnSrhxOyUkKY6oQF+Thuh+K32s//DP/xr81LrrttVPc1dTUuppMzbfB4qCq\nE3IVxJiqxjVphut/TtzVdC9+6nj8zLdxR/4F1/1w62oyNZrdbWRMKdz2OxFcejOu3xngAvwT/yK8\n5lz8J/YdUFNzWXEwpgxckEfQ7VCCa++J1q0unEs4/BLCB27BL12cdDxjKl1Z71YyxhCtOOdOGBh1\nNf3vHvzbE/DvTcF16oHrcTiuVeukIxpTKaw4GFMOrs2OBJcMx096FT/mKfybL+PffBna707Q8wjY\ndS9cYGMSpvrKSXEQkRFAJ6LbYgep6rQSjrkB6Kyq3craxpgkuSAP16U3/oA/wsxphK89H834Onsm\ntMzHdT8Mt39PXMNGSUc1ZqNlfcxBRLoC7VS1M3Ay0VKhRY/pAHTZmDbGVBUuyMPt2Ym8wdcRXHl7\nNCaxdDFe/0V44d+j7qdvvk46pjEbJRcD0j2BUQCqOhtoLiJNihxzC3DpRrYxpspxW7chOGEgwU0P\n4o4+ETZthH9jDOEVZ7Hmtivxs6bjwzDpmMaUKhfdSvnAjIznC+NtywBEpD8wAfiyrG3Wp6CgoNwh\nK9I2FyxfxeQ+XwHsuBO+/5n8PHUCPz73OCs/fI/ww/eoU7ANjY74M5v2Opwg7nKy81cxlq/yJTEg\nvW4aDhFpAZwE9AJalaXNhhQWFpYrUEFBQbnb5oLlq5jE87VpD+cOJZj7Gf71F1j99pssvfdmlj58\nF27/nmx53N9Z6KruvSGJn79SWL6KWV/hykW3UiHRX/3rsgDfxI97EE3kN5FoPeqO8UD0htoYUy25\n1tsT9B8UdTn96XjYpCH+9RdYcMaxhM/+B7/q16QjGrNOLorDWKAvgIh0BApVdTmAqj6lqh1UtRPQ\nB3hXVc/bUBtjqjvXuCnBYUJww/24UweTt/kW+DFPEl49CD/H1pIwVUPWi4OqTgZmiMhkoruOBohI\nfxHpszFtsp3TmFxzdeoQ7NuF/LufwPU8Ar4tJLzp4mjK8F9+SjqeqeWc9770o6oHb2MOybB8FbM2\nn/90NuEjd8CCr6FFS4K/nYXbZa+k41Wb81dVVYd8lDCua3MrGVNFuB3aE1xxO+7wP8MPiwlvH0r4\n4Aj8jxu8Sc+YrLDiYEwV4urWJTiqH8Flt8K2O+CnjCe8YgB++iRq0FW+qQasOBhTBbmt2xAMGY7r\nexL88jPhvTcR3n09fumipKOZWsKKgzFVlMvLIzi4D8FVI2HHXeD9twmvGEg4caxdRZiss+JgTBXn\ntigguOBa3N/OAjz+33cS3no5/jv76o/JHisOxlQDLggIuvQmuOpO2G0f+Pj/CIeeTTh2FD5ck3Q8\nUwNZcTCmGnEtNicYeBnu1MFQrwH+yQcJrxtM+PYE/KpVScczNUjVndDFGFMi5xxu3y749nvgn7g/\nWo3ugVvwje7H7d8L1+Vg3BZbJR3TVHNWHIypplzjJrhTLsAf+Rf8m6/g3xqHf+UZ/CvPQIc9Cbr2\nht33xeXZinRm41lxMKaac1sU4PqehD+qH37GZPyEl+Gj9wg/eg+atcAdeBDugINwLTZPOqqpRqw4\nGFNDuLr1cJ26Qadu+Plf4Se8jJ86Hv/84/gXFHbfJ7qa6LAnLrDhRrNhVhyMqYFcq21xfz0df8yJ\n+HfejK4m3n+b8P23YfMtcV16R+tbN2mWdFRTRVlxMKYGc/UbRGtaH3gQ/ss50dXEOxPwzzyCf+5/\nuI6dcfv3gtbb4xrbSrzmN1YcjKkl3HbtcNu1wx97En7qG/g3XsJPm4ifNjE6oHFTKGiNK2gNBdvE\n/25NtNaWqW2sOBhTy7iGjXA9Dsd3PwzmfISfNR3/zTwonAuffIBPzwJg7QQd85u1INyyVbGi4RrZ\nlUZNZsXBmFrKOQc77ozbced12/zKlbBgHr4wKha+cC7uu8ISiwZNmsFW2+DapnCHHotrsEnufwiT\nNTkpDvG60J2IPleDVHVaxr7SaqIQAAATDElEQVRTgZOBNcBMolXfugJPAh/Gh81S1bNzkdWY2szV\nrw/b7oDbdod12woKCpj/xeew4OvfFQ2+mbeuaPgPZkTf3G7RMsH0pjJlvTiISFegnap2FpH2wINA\n53hfQ+A44EBVXSUir6/dB0xQ1b7ZzmeMKZ2r36BY0QDwK3/B64P4N18mvH4wwYBLcW12TCilqUy5\nuNm5JzAKQFVnA81FpEn8/CdV7RkXhoZAU2BBDjIZYyqBq98Ad/yZuD+fAst+IBx+CeE7byYdy1SC\nXHQr5QMzMp4vjLetW/tQRC4GBgG3qernItIa6CAio4EWwFBVfbW0N4rXQi2XirTNBctXMZavYkrN\nd8IZ/NxhVxYNuwR//81suuIHmvz1tGhcoyrkS1hVz1eSJAaki31aVHWYiNwOjBGRScAcYCigQFtg\nvIjsoKq/buiFy7uId3VYANzylZ/lq5gy5ytog7voRvyd17Ds0ftZPudjXP9zcPXqV418CakO+UqS\ni26lQqIrhXVZgG8ARKSFiHQBUNWfgZeA/VV1vqo+oapeVT8j6mpqlYOsxpgKcK1aE1xyM+zQAT9t\nIuHwS/BLFycdy5RDLorDWKAvgIh0BApVdXm8ry7wsIg0ip/vC6RFpJ+IDI7b5ANbAvNzkNUYU0Gu\ncVOC86/Bde4BX84hvH4wfu5nSccyGynrxUFVJwMzRGQyMBIYICL9RaSPqn4LXE3UbTQF+B4YHf/T\nVUQmAs8BZ5bWpWSMqTpc3bq4kwbhjjkRli4ivPFi/LtTko5lNkJOxhxU9eIim2Zm7HsYeLjI/uXA\nEdlNZYzJJuccrvcx+C1bET5wC+E/b8AdfQKu9zE5G6g25Wfz9hpjssrt2Yngohuh+eb4Z/6Nf+g2\nW9K0GrDiYIzJOte6bTRQ3WZH/JTxhLdehl/+Q9KxzAZYcTDG5IRr1oJg8HW4fQ6ET2cTXncBfv5X\nSccy62HFwRiTM65efdypg3FH/AUWfUc47EL8rOlJxzIlsOJgjMkp5xzBkX/BnfYPWLOG8I5rCJ98\nCL/KbkisSqw4GGMSEexzIMGFN8DmW+LHPkt47fn4r+z7EFWFFQdjTGLcdu0IrhyJ63YoFM4lvGEw\n4QuP49esSTparWfFwRiTKFe/AUG/MwjOHQqNm+GfezQai/jm66Sj1WpWHIwxVYLbeU+Cq+7AdeoW\nTbtxzbmE40bjwzDpaLWSFQdjTJXhNm1EcPL5BGdeDPUb4J94gPDWy/GLvks6Wq1jxcEYU+W4jvsR\nDL0D9vgDpGcRXnU24Vvj8N6X3thUCisOxpgqyTVpTnDWJbj+g8A5/MMjCe+6Dv/DkqSj1QpWHIwx\nVZZzjmD/ngRX3gE77QYz3yG8aiB+xltJR6vxrDgYY6o8t1lLgvOuxh13GqxcSXjPjYQP3IJf8WPS\n0WqsJJYJNcaYjeaCANfzcPzOexA+eBv+7Qn49Af8PHAIvkU+NGpsU4FXIisOxphqxeVvTXDRjfiX\nn8Y//xjfXzs42lGvHjRvCS02x7XYHFq0hOab4zZr+dv2+g2SDV+N5KQ4iMgIoBPggUGqOi1j36nA\nycAaokWABqiq31AbY0zt5vLycIcJfrd9aPj+FH6c+wUsXgiLv4dv55N5T9Pv7m/atDHEhcO12Bya\nt8Tlt4Ld9sHl5eX4p6jasl4cRKQr0E5VO4tIe+BBoHO8ryFwHHCgqq4SkdeBziJSd31tjDFmLbdN\nG5r/YX9+Lixct82vXAlLvofFC/FLvodFC2HJ9/jF0Ta+LYR5X6wrGh4gf2uCY06E3fe1rqlYLq4c\negKjAFR1tog0F5EmqrpMVX+K968tFE2BBUD/9bXJQV5jTDXm6teH/FaQ34qSfs177+GnH6OrjMUL\n8TPfwU8aR3jXddCuA0Hfk3BtUznPXdXkojjkAzMyni+Mt637RS8iFwODgNtU9XMRKbVNSQoKCsod\nsiJtc8HyVYzlq5iamS8uAIf8iVVzT2Xpw3fwy9tvEt7wDzY5oBdNTxxA3YJtEsyXrCQGpIsVc1Ud\nJiK3A2NEZFJZ2pSkMOPScmMUFBSUu20uWL6KsXwVUyvy1akPpwwm6NKb8KmH+XnSOH6eMh7X9RDc\n4X/GNW6abL4sWl/hysX3HAqJ/upflwX4BkBEWohIFwBV/Rl4Cdh/Q22MMSZb3I67EAwZTnD6hdCi\nJf71FwgvOY3wRY3GMmqRXBSHsUBfABHpCBSq6vJ4X13gYRFpFD/fF0iX0sYYY7LGOYfb+wCCq++K\nvnRXpy5+1H8JLzudcNKr+LB2rDWR9eKgqpOBGSIyGRgJDBCR/iLSR1W/Ba4GxovIFOB7YHRJbbKd\n0xhjMrk6dQl6Hk5w3b24Q4+Fn37EP3IH4dBB+FnTa/wkgK4G/YDexhySYfkqxvJVTK7y+SWL8M/9\nDz/5dfAhpHYl6Nsft127KpGvvOIxh2Ljuja3kjHGlIFrvhlB/3MIrrgNdt07mkr8ugsI778Z/1PN\nm+PJioMxxmwEt/V25J1zBcEF18K2O+DfeZPw5kvxy2rWVOJWHIwxphzcTrsRXDIc16U3zPuC8MYh\nNWrFOisOxhhTTi7Iwx1/Ju6QvvBdIeGwi/CFc5OOVSmsOBhjTAU45wiOPgHX9yRYuohw+BD8F3OS\njlVhVhyMMaYSBAf3wZ0wEFasILzlMvzH/5d0pAqx4mCMMZUkOPAggjMuhDWrCG8fin9vatKRys2K\ngzHGVCLXcT+Cs6+AvDzCfw5jxavPJx2pXKw4GGNMJXMd9iA4/xrYpCGLbxtKOO65pCNtNCsOxhiT\nBa5tiuDCYeRt1hL/xL8IR/23Wk25YcXBGGOyxLVqzRY3PQAt8/EvKv7Re/FhmHSsMrHiYIwxWVQn\nvxXBRTfC1tvh3xiD/9et+NWrk45VKisOxhiTZa5pc4J/XA/b7xRNt3H39VV+fQgrDsYYkwOuYSOC\n866GXTrCrOmEt11ZpSfss+JgjDE54uo3IBhwKW6fA+HTj6r0hH1WHIwxJodcnbq4U87HdY0n7Bt2\nUbTC3E8rko72O3Vy8SYiMgLoBHhgkKpOy9jXHbgBWEO0ROgpQBfgSeDD+LBZqnp2LrIaY0y2uSAP\n+p0JjZpEdzE9cgf+0Xtxu++L69QNdt4TV6duohmzXhxEpCvQTlU7i0h74EGgc8Yh9wHdVfVrEXkS\n6A38BExQ1b7ZzmeMMUlwzuH+dDx+/174d97ETx2Pnz4JP30SNGqM2/vAqFC0TeFcsYXasi4X3Uo9\ngVEAqjobaC4iTTL276WqX8ePFwKb5SCTMcZUCa5lPsFhQnD13QSX3YrrdSQEefg3xhAOu5Dw0tMJ\nRz+K/za3S41mfQ1pEbkPeFFVn4ufTwROVtVPihy3FTAR+AOwK3A38CnQAhiqqq+W8lbV56uHxhiz\nAX7Nan55fxo/jR/Dz5PH41f+AkC91C407H4IDbscRF7T5pX5lsUuTXIy5lBaCBHZAngeOEtVF4nI\nHGAooEBbYLyI7KCqv27ohcu7iHd1WADc8pWf5asYy1cx5c631bbw1zNxR/eH96fip77Brx/N5Nf0\nByy9/1bosCeuc3fcbvvi6tevUL6S5KI4FAL5mVmAb9Y+ibuYXgIuVdWxAKo6H3giPuQzEVkAtAK+\nyEFeY4ypMlyDTXCdukOn7vgfluCnvYmfOgFmTcfPmo5v1Jjgsttwm7Ws1PfNRXEYS3QVcK+IdAQK\nVXV5xv5bgBGq+vLaDSLSD9hKVW8WkXxgS2B+DrIaY0yV5Zo2x/U6Cnodhf9mHn7qBPz8L6Fe+a8c\n1vteuZglUESGEd2eGgIDgD2BH4BXgCXAlIzDHwUei//dDKhHNOYwppS38datlAzLVzGWr2IsX8XE\n3UrJjDmo6sVFNs3MeLy+kndEluIYY4wphX1D2hhjTDFWHIwxxhRjxcEYY0wxVhyMMcYUY8XBGGNM\nMVYcjDHGFGPFwRhjTDE5+RJcjtSYH8QYY3KsSky8ly25n/DcGGNqKOtWMsYYU4wVB2OMMcVYcTDG\nGFOMFQdjjDHFWHEwxhhTjBUHY4wxxdSkW1lLJSIjgE5E34kYpKrTMvb1Aq4H1gBjVPWahDLeBBxI\n9N/mBlV9JmPfl8C8OCNAv3hJ1Vxl6wY8CXwYb5qlqmdn7E/0HIrIycDfMjbtraqNMvavAt7K2N9T\nVdeQZSKyC/Ac0YqHd4rINsB/gDyiJXP/pqori7RZ72c1R/keAuoCq4DjVXVBxvHd2MDnIAf5Hgb2\nAhbFhwxX1ReLtEny/D0JrF2zswUwVVVPyzi+P3AN8Fm86VVVvS5b+cqr1hQHEekKtFPVziLSHngQ\n6JxxyEjgYKLlSCeIyNOq+lGOM3YHdokzbga8BzxT5LBDVPXHXOYqYoKq9l3PvkTPoar+C/gXrPvv\nLUUO+UFVu+UqT5xjU+AO4LWMzVcDd6nqkyJyPfB34J8ZbUr7rGY737XAfaqqIjIAOB+4sEjTDX0O\nsp0PYIiqvrCeNomeP1U9NmP/g8ADJTR9QlUHZyNTZalN3Uo9gVEAqjobaC4iTQBEpC2wWFXnqWoI\njImPz7U3gbUfrKXApiKSl0COjVaFzuFaVxD9dZa0lcChQOY6kd2A0fHj54FeRdqs97Oao3xnAU/H\njxcCm2XpvcuipHylSfr8ASAiKaCZqr6TpffOqlpz5QDkAzMyni+Mty2L/70wY993wPa5ixaJuzhW\nxE9PJuqaKdrtcY+IbAdMIvrrKdfThnQQkdFEl8tDVfXVeHuVOIcAIrIPMC+zKyTWQEQeBbYFnlbV\nW7OdRVVXA6tFfncRs2lGN9J3wFZFmm3os5r1fKq6AiD+w2QA0ZVOUev7HGQ9X2ygiJxPdP4Gqur3\nGfsSPX8ZBhFdVZSkq4i8TNR1N1hV36vsbBVVm64citrQdBuJTsUhIkcRFYeBRXZdQXSJ3w3YBTgm\nt8mYAwwFjgJOBP4lIvXWc2yS5/AU4OEStg8GTgMOAvqJyN65DLUeZTlPOT+XcWH4D/C6qhbt0tmY\nz0E2/Ae4WFV7AO8DV5VyfBLnrx5wgKqOL2H3VOAqVe0NXAb8O6fhyqg2XTkUEv31sFYB0WBgSfta\nsXGXsZVGRA4GLgV6q+oPmftU9d8Zx40BdgWeylW2ePD7ifjpZyKygOhcfUEVOodExbPYAKmq3rP2\nsYi8RnT+pucu1jo/isgmqvozJZ+nDX1Wc+UhYI6qDi26o5TPQdYVKVajyRiviVWF89cVKLE7SVU/\nBj6OH08RkZYikpeLmyM2Rm26chgL9AUQkY5AoaouB1DVL4EmIrKdiNQBDo+PzykRaQoMBw5X1cVF\n94nIKxl/oXUFPshxvn4iMjh+nA9sSTT4XJXOYQHwo6r+WmR7SkQeFREX59uf3+62ybVx/HbVdwzw\ncpH96/2s5oKI9AN+VdUr17d/fZ+DHOV7Oh7jgugPgaL/HyR6/mL7ADNL2iEiF4rIX+LHuwALq1ph\ngJo1ZXepRGQY0AUIifpS9yS6g+VZEekC3Bgf+rSq3pxAvtOILpE/ydj8OtGtgs+KyCCiy/ifie5k\nOjuXYw4i0hh4FGgG1CPqWtiCqnUO9wKuVdVD4ucXE91ZM0VEbgR6EP33H52L2wfjPLcA2xHdFjof\n6EfU7dUA+Ao4SVVXicjj8eOfi35WVbXEXzRZyrcF8Au/9dF/pKpnrc1H1OPwu8+Bqo7JYb47gIuB\nn4Afic7Zd1Xo/B1N9P/GJFV9IuPY51T1KBHZmqhrLCA6l+dVxUHrWlUcjDHGlE1t6lYyxhhTRlYc\njDHGFGPFwRhjTDFWHIwxxhRjxcEYY0wxVhyMqQJEpJuIfJp0DmPWsuJgjDGmGPuegzFlEM93dS2w\nKfAp8FfgZmAJsAewI9Fkb8ep6k8ishvRtA6bEX2h7CJVfSV+rYuA04HVwAvABUTfeH+AaNqK44m+\nXHbKeubmMSbr7MrBmFLEUzX8B/iLqrYFxgNr52nqQzRVwzZAU+BUEQmAx4E7VXUnookAHxORxiJy\nQPx8d6LJEw+I2wNsTfRt+PZEheWyXPx8xpTEioMxpesNvKGqa+fwuQc4kmglt+dUdVG8hsUoYD+g\nDdHEb48DqOp0omky9iGa+/9FVV0ez//Ujd8WdFqmqmvXeXiPqFgYk4jaNCurMeXVDOgiIh9nbPuB\nqMsocxB5CdCcaInIpUXmvVpCNGfR5mTMwqqqPwHE6wFkrjewhqj4GJMIKw7GlK4QGFd0Wcx4LePN\nMza1ABYD3wItRMRlFIjN4u3fZ7aJl4M1psqx4mBM6V4BbhSRtqr6uYjsSzSzKkBvEWkGLAf+RLTO\nwZfA18CfgcdFZD+ibqZ3iGbUVRG5PG4zimh8Iam1L4wpkY05GFMKVf0GOBV4VkRmA3fy22I3rxGN\nGXxN1HX0YHy1cBzRUpazgZHAsaq6QlWnEq3Z8T7wEfAu8Fgufx5jysJuZTWmnOJupU9V9dqksxhT\n2ezKwRhjTDFWHIwxxhRj3UrGGGOKsSsHY4wxxVhxMMYYU4wVB2OMMcVYcTDGGFOMFQdjjDHF/D8W\nPvjcKjzSKAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e958bb390>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(HistoryCNNLSTM.history['loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "q9UbfuT9L9vB"
   },
   "source": [
    "### CNN + LSTM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "C4NgLs7HL3TB"
   },
   "outputs": [],
   "source": [
    "modelCNNLSTM.save('CNNLSTM_Model.h5')  # creates a HDF5 file 'my_model.h5'\n",
    "del modelCNNLSTM  # deletes the existing model\n",
    "\n",
    "# returns a compiled model\n",
    "# identical to the previous one\n",
    "modelCNNLSTM = load_model('CNNLSTM_Model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "xVfb_cdfMH_7"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.download('CNNLSTM_Model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8itRFvUFM1iK"
   },
   "source": [
    "## bidirectional LSTM\n",
    "-  The idea of Bidirectional Recurrent Neural Networks (RNNs) is straightforward.\n",
    "-  It involves duplicating the first recurrent layer in the network so that there are now two layers side-by-side, then providing the input sequence as-is as input to the first layer and providing a reversed copy of the input sequence to the second."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 819,
     "output_extras": [
      {
       "item_id": 107
      },
      {
       "item_id": 530
      },
      {
       "item_id": 948
      },
      {
       "item_id": 1362
      },
      {
       "item_id": 1775
      },
      {
       "item_id": 2191
      },
      {
       "item_id": 2610
      },
      {
       "item_id": 3029
      },
      {
       "item_id": 3121
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1122670,
     "status": "ok",
     "timestamp": 1521565041659,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "FzEODoc9MLzt",
    "outputId": "bcc151a6-30cb-4f21-8975-11c894ca1759"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "bidirectional_4 (Bidirection (None, 128)               84480     \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 84,609\n",
      "Trainable params: 84,609\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Train...\n",
      "Train on 27636 samples, validate on 4878 samples\n",
      "Epoch 1/15\n",
      "14080/27636 [==============>...............] - ETA: 36s - loss: 0.6319 - acc: 0.638127636/27636 [==============================] - 76s 3ms/step - loss: 0.6107 - acc: 0.6597 - val_loss: 0.5810 - val_acc: 0.6835\n",
      "Epoch 2/15\n",
      "27636/27636 [==============================] - 74s 3ms/step - loss: 0.5611 - acc: 0.7044 - val_loss: 0.5664 - val_acc: 0.6855\n",
      "Epoch 3/15\n",
      "15232/27636 [===============>..............] - ETA: 32s - loss: 0.5402 - acc: 0.723827636/27636 [==============================] - 75s 3ms/step - loss: 0.5394 - acc: 0.7222 - val_loss: 0.5256 - val_acc: 0.7308\n",
      "Epoch 4/15\n",
      "27636/27636 [==============================] - 75s 3ms/step - loss: 0.5222 - acc: 0.7372 - val_loss: 0.5186 - val_acc: 0.7308\n",
      "Epoch 5/15\n",
      "15488/27636 [===============>..............] - ETA: 31s - loss: 0.5076 - acc: 0.745927636/27636 [==============================] - 75s 3ms/step - loss: 0.5074 - acc: 0.7465 - val_loss: 0.4986 - val_acc: 0.7450\n",
      "Epoch 6/15\n",
      "27636/27636 [==============================] - 75s 3ms/step - loss: 0.4963 - acc: 0.7525 - val_loss: 0.5008 - val_acc: 0.7431\n",
      "Epoch 7/15\n",
      "15616/27636 [===============>..............] - ETA: 31s - loss: 0.4833 - acc: 0.762927636/27636 [==============================] - 75s 3ms/step - loss: 0.4819 - acc: 0.7630 - val_loss: 0.4873 - val_acc: 0.7558\n",
      "Epoch 8/15\n",
      "27636/27636 [==============================] - 75s 3ms/step - loss: 0.4706 - acc: 0.7726 - val_loss: 0.4875 - val_acc: 0.7536\n",
      "Epoch 9/15\n",
      "15616/27636 [===============>..............] - ETA: 31s - loss: 0.4618 - acc: 0.779327636/27636 [==============================] - 75s 3ms/step - loss: 0.4617 - acc: 0.7778 - val_loss: 0.4912 - val_acc: 0.7573\n",
      "Epoch 10/15\n",
      "27636/27636 [==============================] - 75s 3ms/step - loss: 0.4512 - acc: 0.7835 - val_loss: 0.4855 - val_acc: 0.7575\n",
      "Epoch 11/15\n",
      "15360/27636 [===============>..............] - ETA: 31s - loss: 0.4342 - acc: 0.795427636/27636 [==============================] - 75s 3ms/step - loss: 0.4365 - acc: 0.7936 - val_loss: 0.4825 - val_acc: 0.7565\n",
      "Epoch 12/15\n",
      "27636/27636 [==============================] - 75s 3ms/step - loss: 0.4260 - acc: 0.8004 - val_loss: 0.4814 - val_acc: 0.7608\n",
      "Epoch 13/15\n",
      "15360/27636 [===============>..............] - ETA: 31s - loss: 0.4193 - acc: 0.805127636/27636 [==============================] - 74s 3ms/step - loss: 0.4200 - acc: 0.8055 - val_loss: 0.4758 - val_acc: 0.7690\n",
      "Epoch 14/15\n",
      "27636/27636 [==============================] - 74s 3ms/step - loss: 0.4067 - acc: 0.8102 - val_loss: 0.4774 - val_acc: 0.7692\n",
      "Epoch 15/15\n",
      "15360/27636 [===============>..............] - ETA: 31s - loss: 0.3912 - acc: 0.818927636/27636 [==============================] - 75s 3ms/step - loss: 0.4004 - acc: 0.8134 - val_loss: 0.4781 - val_acc: 0.7661\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional\n",
    "from keras.datasets import imdb\n",
    "\n",
    "\n",
    "\n",
    "maxlen = 100\n",
    "batch_size = 128\n",
    "\n",
    "\n",
    "\n",
    "modelBiLstm = Sequential()\n",
    "modelBiLstm.add(Bidirectional(LSTM(64), input_shape=(X_a.shape[1], X_a.shape[2])))\n",
    "modelBiLstm.add(Dropout(0.5))\n",
    "modelBiLstm.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "modelBiLstm.compile('adam', 'binary_crossentropy', metrics=['accuracy'])\n",
    "print(modelBiLstm.summary())\n",
    "print('Train...')\n",
    "historyBilstm = modelBiLstm.fit(X_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=15,\n",
    "validation_data=[X_val, y_val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 298,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 827,
     "status": "ok",
     "timestamp": 1521565577972,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "EqTXBVv3OSKz",
    "outputId": "a91d2e70-eecb-4576-ba52-83bd0fff32f1"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEZCAYAAAB8culNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4FEX+x/F39eQ+OQOEcAoOpwiI\ngoCcKireUuKNt7vqoru6rroeeOvqoqzX+lPR1V21EEG8AEEF5VA5RBAYReUMEO4kQM6u3x89QCBA\nQshkJsn39Tw8zHR3zXwSZb7TVd1VylqLEEIIUZIT7gBCCCEijxQHIYQQpUhxEEIIUYoUByGEEKVI\ncRBCCFGKFAchhBClSHEQ4ij4/f7+fr/f+v3+BuU83vr9/osOse8rv9//fOUmFKJipDgIIYQoRYqD\nEEKIUqLCHUCIUPL7/Ra4AvgD0BX4CRgG3AlcBuwEbgsEAuOCxzcBngP6AYnAbGBkIBBYFtzfDXgF\naA8sAcYe8H5NgDFAXyAJ+Aa4JRAIrKhA9quAO4BjgE3Ay8ATgUDA+v3+hsCLwAAgDlgG/DUQCHzp\n9/sd4PHgz1cPyARGBwKBF440g6i95MxB1AZ/Ai4HWgGNgZnA10AaMAnvw3yPD4BovA//dCALmOT3\n+53gh+54YB7QABgB3HzAe30I7Ab8QBNgTfA1j4jf7x+CVwzuAFLwCtzdwb8BHg1ubw3UAd4E3vb7\n/VHAcOBKoA9egbsKeMzv93c+0hyi9pLiIGqD9wKBwO+BQGAjMAfYFggE3g0EAgXAR0Bjv9+f5Pf7\nuwA9gTsDgcDWQCCQDdwDtAFOAHoALYFHAoHA7uDZxGt73iR4VtEj2H5HIBDIwftw7+j3+084wsw3\nAuMDgcCUQCBQFAgEZgLv433wg1cQCoBdwf3PAxmBQKAouK8YyA0EAjYQCMwC6gYCgcVHmEHUYlIc\nRG2wpsTjXcC6A56D1zXTGigo2QUUCARW430IHwNkBPevLdH+pxKPjw3+vcrv9+f5/f48YCPg4hWV\nI9EaWHrAthXBHABPAMcD6/x+/9t+v/8SwBfc9w7wO7Da7/d/7Pf7/4RXMIQoNxlzELWBW8bzPWIB\ndZDtCrDB/Qd+oSr5fHfwtRMDgUBxBXIemOVgLEAgEFjg9/tbA4OBs/C6xm72+/39AoHANqCv3+/v\nAQzFG2+51+/3nxgIBFYdZS5RS8iZgxD7/ApE+/1+/54Nfr//WLwxiF/wzjii/H5/eok2Jfvxf8H7\nN3VcifbK7/e3rGCWA8cIOgXfA7/fXwcgEAh8FggEbgFOAnoDXfx+f6zf708OBALfBwKBB4J5coEL\nK5BD1FJSHITYZx6wGHjC7/en+v3+unjdN4uABcC3wBbgHr/fH+/3+zvhDUoDEAgElgJfAaP9fn+6\n3++PA+4D5gQfH4nXgQv8fv9gv98f5ff7BwMXBLcDzAUe8fv9icGB8pOAfGAV3lnEB8ErpwDaAXWB\nn48wg6jFpDgIERQIBCxwDl536wq8y0MLgCHBgd084Gy8q4C24H1QP37Ay1we3Lcc2ACcApwebHsk\nWcbjDWaPAbYBo4HrAoHAhOAhw4ATgfXAduDPwAWBQGALcBfeWMdiv9+/C+9qqccCgcDHR5JB1G5K\nVoITQghxIDlzEEIIUYoUByGEEKVIcRBCCFGKFAchhBCl1KSb4GRkXQghKqbUzZ81qTiQmZlZoXbp\n6ekVbhsO1SmvZA2d6pS3OmWF6pX3aLOmp6cfdLt0KwkhhChFioMQQohSpDgIIYQoRYqDEEKIUqQ4\nCCGEKEWKgxBCiFKkOAghhCilRt3nUBF2wWx2/hQLHbuHO4oQQkSMWn/m4M6YzNYxD2OzQnPDy4wZ\nM8p13PPPP8/69etDkkEIIY5UrS8Oqs+p4LrYKRMr/bU3bNjAF198Ua5jb7nlFpo0aVL2gUIIUQVq\nfbeS6nYyviYZFM2ejj3nElRq3Up77WeffZbly5czcOBABg8ezIYNG3j66ad56qmn2LRpE3l5eYwY\nMYJevXpx2223MXLkSGbMmMHOnTtZs2YNmZmZ3HzzzZx00kmVlkkIIcqj1hQHd9xY7PxZB99ZkAdF\nhbj3/QESksr9mqp7b5xhVx9y//Dhw5kwYQKtWrVi9erVjBkzhm3btnHCCScwZMgQMjMzefDBB+nV\nq9d+7bKysnjiiSf47rvvmDRpkhQHIUSVqzXF4XBUXALk5kDebohPAFX5vW3t2rUDIDk5mUAgwMcf\nf4zjOGRnZ5c6tnPnzgA0bNiQnTt3VnoWIYQoS60pDs6wq+EQ3/LT09NZ++pz2AlvoQachTPkwkp/\n/+joaACmTZtGdnY2Y8aMIScnhxtvvLHUsT6fb+9jWeNbCBEOtX5Aeg/V/wyIi8dO+whbWFg5r6kU\nxcXF+23Lzs6mSZMmOI7DzJkzKSoqqpT3EkKIylQlZw5a69FAT7wFeUYaY74vsa8Z8A4QAywwxtxU\nVptQUAlJqH5DsFMmYOd+iep72lG/ZosWLfjll19o0qQJqampAJxyyince++9LF26lDPOOIOGDRvy\n5ptvHvV7CSFEZQp5cdBa9wPaGmN6aa3bA68DJUdgnwGeMcZM0Fq/oLVuDrQqo01IqMHnYKd/hJ38\nAbb3IJTjK7vRYdSpU4f33ntvv22NGzfmtdde2/v81FNPBeCqq64CoFWrVnv3tWrVimefffaoMggh\nREVURbfSIGAigDFmGVBXa50CoLV2gL7ApOD+m40xqw/XJpRUnfqoXgMhKxMWfhvqtxNCiIhVFd1K\njYH5JZ5vCm7LBhoCOcBorXU34GtjzN1ltDmkQy13Vx572hZecSMbvvkc3/RJNBp6IUqVWlo1IhzN\nz1rVJGvoVKe81SkrVK+8ocgajquV1AGPmwLPASuBT7TWZ5XR5pAqZQ1pFQ1de1G4YDaZX0xGte9S\nodcMpdq0vm1Vqk5ZoXrlrU5ZoXrlrc5rSGfifevfmwXYM4nQZmCVMeZXY0wxMB3oWEabkNtzKas7\neXxVvaUQQkSUqigOU4GLAIJdR5nGmBwAY0wR8JvWum3w2O5A4HBtqoJq1Rb8nWHpD9hVv1bV2woh\nRMQIeXEwxswG5mutZwNjgJu11iO01ucHD7kNGBvcvwP46GBtQp3zQM4ZFwFgp3xQ1W8thBBhVyVj\nDsaYvx2waVGJfSuAPuVoU7U6HA/NW2PnzcKedzkqLXQzpg4fPpyxY8cSHx8fsvcQQogjIXdIH4JS\nCjXkQrAuduqEcMcRQogqJcXhMFS3k6FhY+ys6dgd2464/Q033MDGjRsBb22H66+/nrvvvpvbb7+d\nP/zhDyxbtqyyIwshRKWoNRPvjV2QxezVB79NwudbSXHxwec4ssfdCjtz4JPVqIQt++07uXkKV3dL\nO+R79unTh9mzZ3P++ecza9Ys+vTpwzHHHEOfPn1YsGAB77zzDg899FDFfyghhAgROXMoS2y8N4V3\n3u4jniG1b9++zJkzB4BZs2bRu3dvZs6cya233sorr7xy0Om6hRAiEtSaM4eru6Ud8lt+WTeRuJ8s\nwE58G3XRCJzTLyj3e7Zq1YotW7aQlZVFbm4us2bNokGDBtxzzz0EAgFeeumlI/45hBCiKsiZQzmo\nAWd603l/PumIp/Pu2bMnr776Kr1792bHjh1770b8+uuvZbpuIUTEkuJQDiohCXXKENixFTv3yyNq\n27dvX6ZPn06/fv047bTTGDduHHfeeSft27dn69atfPbZZyFKLYQQFVdrupWOljr1HOwXH3nrPRzB\ndN7t2rVj+vTpe5+XXLuhd+/eAJxxxhmVG1YIIY6SnDmUk6pTH9VzAGxcJ9N5CyFqPCkOR0Cdfj4o\nhTt5vKztLISo0aQ4HAHVOAO69oSVv8DyH8MdRwghQqbWF4cpv2znua9WkJtfXK7j903nLRPyCSFq\nrlpfHH7K2sXb36/m5o9/45tV2WV2F6lWxwan816IXS3TeQshaqZaXxz+1KsJN5/Sml2FLv/4JpNH\nvlpLVu7h72XYc/Zg5exBCFFD1friEOUoRpzUkjFnteK4xgnMy9zJrZ/8xqTlWyl2D3EW0bErNGvl\nTeedVWUL1AkhRJWp9cVhjybJMTw0sBkjezUh2ufw2vws7pyyit+25pU6VqbzFkLUdFIcSlBKMbB1\nKi8MbUX/Vin8ujWPv0xeyRsLssgrcvc/tnvvfdN5Zx/5dN5CCBHJpDgcRGpcFLefnM6ogc1IS4xm\nwrKt/OmT31mQmbv3GOXzoU47D4oKsdM+CmNaIYSofFIcDuP4JomMOasVF3Sox6adhYz6ci3/nJXJ\n9jxvwjx18iBITsV+9Rl2964wpxVCiMojxaEMsVEOV3VN459ntKRt/ThmrMzmlo9+Y9qv2yE6BjXo\nbNi9EztzcrijCiFEpZHiUE6t6sbx5GktuK57GoWu5V9zN3Df9DWsP+FUiK3YdN5CCBGppDgcAZ+j\nOLtdPZ4f2poeTRNZvHEXI7/I4v0+11KYveOIp/MWQohIJcWhAhomRnNvvwz+2jedpBiH/xU3584T\nRrJ85hysW75pOIQQIpJJcaggpRS9m6fw/NmtOb1NHVYnNuaeFsP49+TF7CyQAiGEqN6kOBylpBgf\nfzypMY91j6fprk18ti2OWz7+nTlrcsIdTQghKkyKQyXp2K4FzxTPZfjvU8nOK+SJmev476JN4Y4l\nhBAVIsWhEsUOOR+9ahr/3PwJaYnRjP9pC2uz88MdSwghjpgUh0q0ZzrvjMUzubq5S7GFNxbI2YMQ\novqR4lDJ9kznfdK8iXRKi+f7dbks2rAzzKmEEOLISHGobMHpvJk3i6tb+VDA6/OzDj39txBCRCAp\nDpWs5HTerT59nQEtk1m5PZ/pv+0IdzQhhCg3KQ4hoLr3hnbHweJ5XDrvLeJ8ircXbWJXodz/IISo\nHqQ4hIDy+XBuvQ86n0C9JbM5f+t8duQVM27JlnBHE0KIcpHiECIqJhbnj/egevTlnEXv06Aoh0nL\nt7IxtyDc0YQQokxSHEJIRUWhrvszcb0HcvnPH1Hkwhvfrg13LCGEKJMUhxBTjg91xc2c0qUVbbNX\nMXtDAT8FVoc7lhBCHJYUhyqglMIZNoJr0r31Hl6b8SvFa34PcyohhDg0KQ5VRClFh3OH0jcuh18T\nm/Dl2Pewv/8c7lhCCHFQUVXxJlrr0UBPwAIjjTHfl9i3ElgD7LnO8zKgLTAO+Cm4bbEx5taqyBpq\nVw05nm8/XMF/MwbQa/RDJPzxr6h2x4U7lhBC7CfkxUFr3Q9oa4zppbVuD7wO9DrgsDOMMbkl2rQF\nZhhjLgp1vqrWMDGaczs0YNxPiomNezH8uVE4N92F6nJiuKMJIcReVdGtNAiYCGCMWQbU1VqnVMH7\nRqwLO9anbpyPiS0HsjmuDu5Lj+N+OyPcsYQQYq+q6FZqDMwv8XxTcFt2iW0va61bAt8Adwe3ddBa\nTwLqAaOMMZ+X9Ubp6ekVDnk0bSvilv4OD09ezvhz7+WmiX/HvvZPUuJiSDrjwnK1r+q8R0Oyhk51\nyludskL1yhuKrFUy5nAAdcDz+4HJwFa8M4wLgTnAKMAArYEvtdZtjDGHvYMsMzOzQoHS09Mr3Lai\nuta1tKoby5R1+Qy6fhRtXnuIbc8/zvb1mXtndj2UcOStKMkaOtUpb3XKCtUr79FmPVRhqYrikIl3\nprA3C7B+zxNjzH/2PNZafwp0Nsa8D7wX3Pyr1noD0BSoMdd/+hzFtd3T+Pu0NYxdF8NjdzyGffYB\n7Pg3cXfvQp13OUodWEeFEKJqVMWYw1TgIgCtdTcg0xiTE3yeqrWeorWOCR7bD1iitb5Ma31H8JjG\nQCNgXRVkrVKdGyXSs1kSyzbtZnZhKs5dT0BaE+yn47DvvIJ13XBHFELUUiEvDsaY2cB8rfVsYAxw\ns9Z6hNb6fGPMDuBTYK7WehbeeMT7wCSgn9b6a+BD4A9ldSlVVyO6phHlwJsLN1FYpwHOX5+Api2w\nX36CHfsctlhmchVCVL0qGXMwxvztgE2LSux7DnjugP05wNmhzhUJmiTHMNRfj4nLtjJp+TYu6lgf\n587HcJ8bhZ37JTZvN84Nd6Kio8MdVQhRi8gd0hFgWKf6pMT6eH/JFrbtLkIlJuP8+SFvTYgf5uL+\n6yFsfl64YwohahEpDhEgKcbHJcc1YHeRy38XbQJAxSXg/Ol+6HIiLFuEO/p+7K7cMl5JCCEqhxSH\nCHF6mzo0S41h2q87+H2bd5agomNwbvob6sR+8Oty3H/ci83eHuakQojaQIpDhPA5imu6pWGB1+dn\nYa0FgmtCXHs7qt8QWPs77lN3U7Re1oQQQoSWFIcI0i09ie7pify4cRffrdvXhaQcB3XZH1BDLoSN\n69hw62W4338TxqRCiJpOikOEubpbGo6CNxZkUVhs925XSuFceBXqmtvButhXnsJ960VsQX4Y0woh\naiopDhGmWWosQ9rWITOnkE9/3lZqv9NrAI2eewsyWmFnTsZ97A6sdDMJISqZFIcIdEnnBiRGO7y3\nZDPZ+aVvgovOaIlzzz9Q/c+EdatwH7kdd/b0MCQVQtRUUhwiUEpcFBd3bsDOApd3f9x00GNUdAzO\nZTfh3HQX+HzYsc/hvj4am7e7itMKIWoiKQ4R6sxj65KeHM1nv2xnzY5Djyuo7r1x7nsWWrbFzvkS\n99E/Y2V9aiHEUZLiEKGifYoRXdNwLYxdkHXYY1XDxjh3PYE69VzYsA73sTtwv/ps7+WwQghxpKQ4\nRLATM5Lo3CiB+Zk7WZB5+LujVVQ0jr4W55b7IDYO+9+XsP9+CrtrZxWlFULUJFIcIphS3o1xCnh9\nQRbFbtlnAqpLD5z7n4U2HbDzZ+E+fBv2919CH1YIUaNIcYhwrevFMeiYVNbsKGDqivJNnaHqNcS5\n41HUmRq2ZOE+eRfu5x9KN5MQotykOFQDl3dpSFyUw/9+3ExuQfnWd1A+H875l+Pc9iAkJGLNa7jP\nP4LNzS6zrRBCSHGoBurGRzGsY32y84sZt2TLEbVVHbriPDAG2neBH7/Hfeg27C9LQ5RUCFFTSHGo\nJs5pX5e0xCg+Dmxl4o+ZbN9dVO62KrUuzm0Pos67HLZvxX36HtxPjCxDKoQ4pCpZCU4cvRifw7Xd\nG/Hk1+t4dMpyFHBsgzh6NE3ixIxkmqfGoJQ6ZHvl+FBnaWzbjrj/9zR24tvYn5fgXHs7KqVu1f0g\nQohqQYpDNdKzWTIvDG3N8hyH6UvXsXTTbgKb83h70WbSEqPpkZHEiU2T6JiWQLTv4IVCHdsR5/7n\ncMc+C4vn4Y4aiXPdX1Dtu1TxTyOEiGRSHKqZ9JQYTmiXzsCmUeTkFzM/M5fv1+WyIHMnnwS28Ulg\nGwnRDl2bJHJiRhLd0pNIifXt9xoqOQXn1vuwn3+I/eBN3NH3o84Yhjr7YlSUrFUthJDiUK0lx/ro\n3yqV/q1SKSy2LN20i+/X5vLdulxmrc5h1uocHAXtGsRzYkYSPTKSyEiJBbx7KNRp52HbdsD991PY\nTw120bc4V9yMOqZdmH8yIUS4SXGoIaJ9ii6NE+nSOJFru6exZkcB363L5bu1uSzbtJulm3bzxsJN\npCdHc2JGMj2aJtG+YTy+Vsfi3P8cdvyb3hTgT96FGjgUdd5lqLiEcP9YQogwkeJQAymlaF4nluZ1\nYrmoY3225xUxf53X/bRw/U4mLtvKxGVbSYpx6JbujVN0u/hGEk46BfetF7DTP8IunINz2R9Qx/UI\n948jhAgDKQ61QJ24KAYdU4dBx9ShoNhlycZdfBfsfpq5MpuZK7OJi3K4o3dLTrj/OewnBjt5PO6/\nHkb16Isafj0qpU64fwwhRBWS4lDLxPi8s4Vu6UncaC2/b8vn27U5fLB0K4/NXMt13Rtx1nmXY0/o\ng/uf57Hff41d+gNKX4PqNfCwl8sKIWoOuQmuFlNK0bpeHJcc15BHBzcnOdbHK/M28tr8jbjpLXD+\n9iRq+PVQVOgtJjT6fmzW+nDHFkJUASkOAoBjG8Tzj9NbkJESw6Tl23jqm3UUuApn0Nk4o56HzifA\nskW4o27FnTIBW1y+OZ6EENWTFAexV6OkGJ48rQWdGyUwd00u905bzfbdRaj6aTi33oe67i8QG499\nfyzuY3dgV/0a7shCiBCR4iD2kxTr44EBzRjQKoVftuRx55RVrNmRj1IK56R+OA+9gOo1EFb/ivvY\nX3DfH4vNP/QypkKI6qlcA9Ja6/bAWcaYp7XWnYCXARcYaYxZGMqAoupF+xQjezWhcXIM7/y4mbum\nruJvfZtyXONEVFIK6prbsD374b71InbKBOyCOd7NczIFhxA1RnnPHMYCe1atfwH4DHg0+FjUQEop\nhnduwG29mpBf5DLqyzV88duOffs7dMV58HnU6efD5izcf96HO/Y5WS9CiBqivMUh1RgzXmudBnQB\nnjTGTAGSQxdNRIIBrVN5cGAzYqMcnpuznnd+3LR3RTkVG4tz0dU49z4NzVtjZ0/Hvf9m3O9myqpz\nQlRz5S0OVmudAAwHphpjirTW0UBs6KKJSNG5USJPntaCRknRvLt4C8/OWU9h8b4Pf9WiDc49z6Au\nGgF5u7H/9zTuvx7GbtkUvtBCiKNS3uLwIrAWuB+vOwngv8CEUIQSkadZaixPnd6CY+vH8dXv2Tz4\n5Rpy8/ddzqp8PpzTL8B5cAy0O86bDvyBW8ge/xY2Py+MyYUQFVGu4mCMeR5oDjQ2xiwKbn7IGHNX\nyJKJiFMnLopHBjenV7MklmzcxV1TV7Ext2C/Y1RaOs6fH0aNGAk+Hztefw73b9d5K8/t2hmm5EKI\nI1Wu4hC8WummYHdSJ631N8CLWuvjQxtPRJrYKIe/9m3Kee3rsTa7gDunrOLnzbv3O0YphdN7EM6j\nL5My/Dpwi7ET38b927W4E97C5uw4xKsLISLF0V6t9GIoQonI5ijF1d3SuLFHI3Lyi7l32mrmrMkp\ndZxKSiH1iptwnngNdcGVEBWN/XScdybx3qvYbVvCkF4IUR5ytZKosDOPrcu9/TJwFDw5cx0fLtt6\n0KuUVHwCzhkX4Tz+Kmr4DZCYjJ02Cfee670pwjdtCEN6IcThlHdW1qO6WklrPRroCVi8G+e+L7Fv\nJbAG2DO6eZkxZt3h2ojIcULTJB4/tQUPf7WW1xdksSG3gOu6N8LnlJ69VcXGogYNxfY7HTvnS+zk\n8diZU7DffI468RTUGReh0puH4acQQhyovMVhz9VKLjAouK1cVytprfsBbY0xvYJjF68DvQ447Axj\nTO4RthERonW9OP4xpAUPf7mWT3/ezqadhfyld1Piow9+YqqiolF9T8OePAg77xvsZ+9j536FnfsV\ndOuFc+YwVIs2+7XJK3LZtruIbbuLyM4vxt8gnrrxMuO8EKFSrn9dxpjntdZvAHnGmKLg5oeMMUvK\n0XwQMDH4Osu01nW11inGmMPdSluRNiKMGiRE8/hpzXnq60y+X7eTez5fxd/7Z5B+mDbWccju0put\nbU5k+7Kf2DJ/Hts372Lbu1+xrcGvbG/QnO0qhq27i8krcvdrGxflcHGn+pzdri7RPpkiTIjKVt65\nlRRwDnBacNxhI/AJUJ7i0BiYX+L5puC2kh/0L2utWwLfAHeXs00p6emH+yg6vKNpGw6RmvfFjKY8\nNe1nJvyYyd3T1vLH/Fh2Frhs3lnAlp35bMktYMvOAjbvzGfrzkKK945RJEGD/tBg32upHJdUm01G\nSgING9SjflIMDRJjifYpxi1cx5s/bOLLVbn8eWBberducLA4RyxSf6+HUp3yVqesUL3yhiJrec/L\n/wH0xetK2gbUB+7WWnc0xow6wvc8sDP6fmAysBXvbOHCcrQ5qMzMzCOM4klPT69w23CI9LxXdUom\nxWnImz9s4oFPl5baH+NT1IuPom39OOrGR1E33uf9HRcVfB5FnQ2/kfz5+/iWzPMatWiDc+YwaH0S\nynHon96SdxZv5tOft3Hb+B/pnp7Itd0b0TQlpsK5I/33eqDqlLc6ZYXqlfdosx6qsJS3OJwBdDfG\n7L3VVWv9f8B3QFnFIRPvW//eLMDe5cSMMf8p8ZqfAp3LaiMim1KKCzrW59gG8WwujsEp2LVfAUiI\ndspebrReB+hwP3b1r7ifjoMFc3BfehyaNEOdeRGJPU7h+hMacVqbOrw6byPzM3eyaMNvnNOuHsM6\n1Sch2lc1P6wQNVR5i4MPOHDS/t2U71LYqXgF5N9a625ApjEmB0BrnQoY4GxjTAHQD3gfWHeoNqL6\n6NQo4ai/1ajmx+C76W/Y9Wu8getvZ2BfG42dMhHnyptp0epYHhrUjDlrchi7IIsPlm7ly992cGXX\nNPq3SsGRNa+FqBBVntkztdYvA02BV9jXrXQdsMYY88dytH8COAXvaqebga7ADmPMBK31SOAqvGKz\nELjVGGMPbFNi2o5DsdKtFHkqO6vdtAH70bvYOV+AUqj+Z6LOvwIVn0B+kcuEZVsZ/9MWCoot/gZx\nXH9CI9rWjw9L1lCrTnmrU1aoXnkrqVup1Leo8haHGOB24EygId6A9MfAv4Lf+COBFIcIFKqsNrAE\n9+0XYMM6qFMPZ/gN0K0XSimycgt5Y2EWs1bnoIBBx6RyxfENqRN3+BPl6vR7heqVtzplheqVN1TF\n4bD/WrTWv+DdhHZg4wzgJuBG4NgKpxKigpS/E879Y7wb6T41uC8/Acf1wLn0RtLqp/HXvk1ZvHEn\n/zcvi2m/7mD26hyGd27AWf66RB3kBj0hxP7KGnO4rkpSCFEBKjoadfZwbI8+uG+/BD9+j7v8R9S5\nl6IGnUPnRomMPqMlk3/Zzv9+3MTrC7KYumI7153QiK5NEsMdv0KstWzLK2btjny2qWySXSvFToRE\nubqVqgnpVopAVZXVWutNyTHuNcjNgWatcK64BdWqLQDZeUX878fNTFmxHdfCSRlJXNMtjcbJ+y59\njaTfq2stWbmFrM0uYM2OfNbsKGBtdj5rdxSws3DfDYFxUYp2DRPolBZPx7QE2taPi8ibAiPpd1se\n1SlvWLqVhKgulFKokwdiO5+AfX+st2Tp43egBpyFOu9yUuITuOnExpzWpg7/N28j367NZUHmTs5r\nX4+LOtUnLio8H6iFxZb1uV4oJXlIAAAbEUlEQVQBWLujgLU7CliTnc+67AIKivf/4uZT0CQ5hs6N\nY8hIicVGxfHt75v4Yf1OfljvrZUR41P4G8TTKS2Bjo3i8TeIJyYCi4WIfFIcRI2iklNQV4/EnjwQ\n9+0XsV98jF0wG+eSG6BrL1rXi+OxU5vzzaocxi7MYtxPW/jitx2M6JbGxU2ahCxXXpHrffhne2cB\na3bksza7gPU5BbgHnLzH+BQZKTFkpMbSLCWGjNQYmqXG0jgphmjfvi943jfGZLbnFfFT1i5+2riL\nJVm7WbxxF4s37oLFEOUojq0fR6dGCXRMS6Bdw/iwFUJRvUi3EtXrFBKqV95wZrWFhd69EZ+Ng6Ii\n6HIiziU3ouo3BLwP7PE/bWHC0q0UupaOTVJoEOt16bjWuxLDlnjsutb72wa3l3hsbfAx3vF7twO5\n+cVs2lVUKl9ijENGSizNUmNolhqz93HDxOhy3Z9xqN9tdn4xS7N2sSRYMH7flr/3qhKfgjb14+mU\nFk+nRl6xqIobBqvT/7NQvfJKt5IQR0hFR6POuQR7Yl9vwHrRd96A9TmXogadTVyUj8u6NGRQ61TG\nLsxi7pqKz+voKO9fl1IKR+157j2OjXI4rlECGSUKQLPUWOrE+cq+U7wCUmJ99GyWTM9m3nIruQXF\nLMva7RWLrF38smU3gc27Gb90K46CY+rF0TEtgU5pCbRPiycpRu4uF1IcRC2gGmfg/OUR7JwvsONe\n9/58+xXOFTejWralcXIMd5+SQVxqA9atX+99wANKsfex4+z7sFclPvid4DGRLCnGR4+MJHpkJAGw\nq7CY5Zt281PWbpZs3MWKrbv5ZUseE5dtRQENE6NIS4qhcVI0jRKjaZQUTePkGBolRpMaooImIo8U\nB1EreAPWg7Cde+wbsH7sTtTAs1DnXoaKT6BeYgx5CdHhjhpyCdE+uqUn0S3dKxb5RS7LN3uFYumm\n3azPLmDJxl0s2Vi6baxP0TgphrSkaK94BP/s2SbjGTWHFAdRq+wdsO41APftl7DTP8LO9wasbZML\nwh0vLGKjHLo0TqRL4333fhQUu2TlFrIxt5ANuYVk7SxkQ26B9zynkFU7DpxqzVMnzhcsGDH7FQ+b\nsJvc/GLiopz9BtVF5JLiIGol1e44nAeeCw5Yv4/70uNsXjALe9ZwVJOMcMcLuxifQ0ZqLBmppVcC\nttaSU+CycU+xyC0kK3df8VixJY/A5rwDWq3Z+yjK8RZrio1yiI9yiItyiIt2iI9Sh9juEBfct/82\nh9Q4H4kyRhISUhxEraWiY1DnXIrtcQru2y+S9+1M+O5rVPfeqLM0KqNluCNGJKUUKbE+UmLjDzqp\nYbFr2bzLO+vYUzx22Wi2ZOeSV+R6fwotecUuOfnFZO0sLHVPx5FoUSd2702AHdMSqCPLx1YK+S2K\nWk81ycC541Hqrv6FLf95yVvXet43cHxPnKEXo1ocE+6I1YrPUTRKiqFRUvnvPi92LfnFLrsLXfKL\nLHlFLruLXPIK3X2Pi1zyiux+2zbkFvLz5t2s2p7PJz9vByAjJYaOaQl0CF6u26AWjCOFghQHIfC+\nDSf06s+25m1hyXzcj9+DH+bi/jAXOp+Ac5ZGHdMu3DFrLJ+jSHB8FbrnorDYZcWWvOClurtZtmk3\nU1ZsZ8oKr1g0SooOnlV4d443SoquUVdchepeNSkOQpSglPKKQafusGwR7ifvweJ5uIvnQfsu3pnE\nsZ3CHVOUEO1zaJ+WQPu0BIbhnYX8ti2Pn7J2sWTjbpZu2sUXv+3gi992AFA/PoqOjfYVi6YpMSEp\nFoXFLrkFLjkFxeTmF5NbUEx+kaXQtRQUuxQUWwqKLYX7PS5jn+tSUGQpcPftS4hZwT+HNN/vTK0y\nSHEQ4iCUUtDheHwdjvfWjvjkPa9YLFsEx3bCGXoxtDuuRn0DrSl8jqJtfW885Lz23h3vq7bn7ysW\nWbuYuTKbmSu9mx5T43z7nVk0r7NvEN5ay65Cl9yCYu+DPvghn1tQTG5+8IN/7/NicgrcvY/zj2Ic\n5UAxPkWMTxHtc4j1KRJiHGJ8UcT4HNLrJoVkUF6KgxBlUP5O+PydsL8u97qblszH/ecSOKYdzlkX\nQ6duUiQimKMUrerG0apuHEP93gf+2uyC4HxU3p3js1fnMHu1txJxUoxDvcQ1bN+VT25Bcam5rw4n\nMdohMcZHRmoMSTG+vX+SY30kxnhXWMX4FNGOIibKIcZRRPsUsVGOt80XfO7zLvmN8SmiHHXY/79C\nNdWHFAchykkd0w7fyAewK3/B/cTAD9/ijhkFLdrgDNXQ5SQpEtWAUopmqbE0S41lSNu6WGvZkFvo\nFYusXSzN2k1ufiEpsT7Sk2NIjvU+8JNjfCTFen8nxjh7nyfF+EiO8Y7x1aC1NaQ4CHGEVMu2+G6+\nF7v2d+zHBrtgNu4Lj0FGS5yzNHQ7GeXIncLVhVKKJskxNEmOYfAxdYDqNfFeqEhxEKKCVEYr1E13\nYTNXYz8dh/3ua9x/PwVNmqHOHIbq0Rflkxu0RPUkX2+EOEoqvTnOdX/BefhFVO9BkJWJfe2fuPf/\nEXfWNGxR6em6hYh0UhyEqCSqUTrOiJE4j7yMOmUIbNmEfWMM7r034E76H3bzQWayEyJCSbeSEJVM\nNWiEuuKP2LOGYSd/gJ39Bfajd7EfvQvtu6B6D0Z17YmKKT1vkRCRQoqDECGi6jVEXXoj9oIrsfNn\nY2d9DssWYZctwsYnok46BdV7MLRoI1c5iYgjxUGIEFNx8d5YRO9B2A3rsLOnewsPffUZ9qvPoGkL\nVJ/BqJP6o5JTwx1XCECKgxBVSjVuirrgSuy5l8HSH3BnfQ4/fId97zXs+29661z3GQwdusqVTiKs\npDgIEQbK54PO3fF17o7NycZ++xX2m89hwWzcBbMhtR7q5AGokwejGjcNd1xRC0lxECLMVHIKavA5\n2EFnw+pfsd9Mw343A/vZeOxn46FNB6/bqXtvVFzp9ROECAUpDkJECKWUNzjdog122NXYhXOxs6bB\n8h+xK5Zi33kFdUIfVJ/BcEz7cMcVNZwUByEikIqJRZ3UD07qh9280bscdvZ07KxpXsFo1JTsocOw\nXXqi4hPCHVfUQFIchIhwqkEj1DmXYIdeDIHFXoFYMIcdrz0L8QmovqejBg1F1WsY7qiiBpHiIEQ1\noRzHu4mufRfszlySF85mx8T/YqdOwE6f5M3ldNr5qGatwh1V1ABSHISohlRiEil6BDknDfAGr6dM\nwM79Cjv3K+hwPM5p50OH4+XmOlFhUhyEqMZUdDSq92DsyYNgyQLcKR94908s/QEyWnpnEj36oKKi\nwx1VVDNSHISoAby1r4P3Taxa4Z1JzJ+FfX009oP/oAaf7Y1NJCSGO6qoJmRWViFqGNWiDc4Nd+I8\n+m/U4HNg907s+2/g3nUN7rjXsVs3hTuiqAbkzEGIGko1aIS6+Drs0OHYmZOx0z/GTp2Inf6Rd7/E\naeejmrcOd0wRoaQ4CFHDqcQk1BkXYQefi/1upnd107czsN/OgPZdvMHrjl1l8Frsp0qKg9Z6NNAT\nsMBIY8z3BznmcaCXMaa/1ro/MA74Kbh7sTHm1qrIKkRN5Q1eD8KePNAbvJ46AZYtwl22yJsZ9rTz\nUCeeIoPXAqiC4qC17ge0Ncb00lq3B14Heh1wTAfgFKCwxOYZxpiLQp1PiNqm1OD11InYed9gxz6H\nnfAW6pxLUX1OlTOJWq4qBqQHARMBjDHLgLpa65QDjnkGuLcKsgghSlAt2uBcfwfOY6+gBp8Lu3dh\n//M87jN/x27MDHc8EUZV0a3UGJhf4vmm4LZsAK31CGAGsPKAdh201pOAesAoY8znZb1Renp6hUMe\nTdtwqE55JWvoVFre9HTofDxFV9zAthefJO/bmbij/kTqpdeTfMEVqKij/6iotb/bKhCKrOEYkN57\nrqq1rgdcDQwGSk5a/wswCjBAa+BLrXUbY0zB4V44M7Ni33TS09Mr3DYcqlNeyRo6ocprr/0LTtde\nuO+8wo43X2DH9E9wrrwV1apthV9Tfrehc7RZD1VYqqI4ZOKdKezNAqwPPh4INAS+BmKBY7TWo40x\ntwPvBY/5VWu9Aa94/F4FeYWo1ZRS0L03Trsu2PFvYL+eivv4nd7kfudeJmtK1BJVURym4p0F/Ftr\n3Q3INMbkABhj3gfeB9BatwTeMMbcrrW+DGhijHlaa90YaASsq4KsQogglZiEuvIW7En9cP/zAnba\nJOzCuTiX/wHVqXu444kQC/mAtDFmNjBfaz0bGAPcrLUeobU+/zDNJgH9tNZfAx8CfyirS0kIERrK\n3xnngedQZw6D7VtwnxuF++oz2Jwd4Y4mQkhZa8OdobJYGXOIPJI1dMKR1675Hfc/z8PKXyApGaWv\nQ/XsX+Zlr/K7DZ1KGnMo9R9Q5lYSQpSbatYK5+6nUBdfCwUF2NdH4z77IHbThnBHE5VMioMQ4ogo\nx4cz+FycUc9Dp26wdCHug7fiTp2ILS4OdzxRSaQ4CCEqRDVohPOnB1DX/QViYrHjXsd9/E7s6t/C\nHU1UAikOQogKU0rhnNQP56EXUb0GwKoVuI/+GXf8m9iC/HDHE0dBioMQ4qip5BSca27HuW0U1GuI\nnTwe98FbscsWhTuaqCApDkKISqM6dsV58F+o086DzVm4/7wP940xFMtlr9WOFAchRKVSsXE4w67B\nufdpaNYKO2saG24ahjvlA+yuneGOJ8pJioMQIiRUizY49zyDuvAqbH6et1TpX6/Bfe817JascMcT\nZZDiIIQIGRUVhTPkQtLf+AR14VUQH4+d9iHuPTfgvvIP7Mpfwh1RHIIsEyqECDknKRlnyIXYwedg\nv//GW2Do+6+x338Nx3b0lirtfALKke+rkUKKgxCiyqioaFSvAdie/b0lSj+f6C1Z+vNP0Lgp6tRz\nUT0HoGJiwx211pPiIISockop6HA8vg7HY9etwn4+ETt3BvatF7ET/4vqfyZqwJmo5NRwR6215BxO\nCBFWqmkLnBEjcZ541Zv5tbgY+9E7uHddi/vWC9gNa8MdsVaSMwchRERQdeqhzr8Ce+Yw7Kxp2M8/\nxM6cgp05BbqciHPqeXBsxzJngBWVQ4qDECKiqNg41MCh2P5nwMJvcadOgEXf4S76Dlq0QZ1+Pqrb\nySifL9xRazQpDkKIiKQcH3Q/GV/3k7ErlnmD1wvnYl/5B7Z+GmrQ2ai+p6LiEsIdtUaS4iCEiHiq\nTXt8bdpjszK95UpnTcOa17AT3oI27VHtjkO17wItjvGKijhqUhyEENWGSktHXXoT9pxLsTMme/dJ\nLFuEXbbIKxTxieDvhGrXBdX+OGjSTMYoKkiKgxCi2lFJKaizNJylsdnbsct/hOU/en//8C32h2+x\nAKl1Uf7joL13ZqHqp4U7erUhxUEIUa2plDqoE0+BE08BwG7e6E0VHiwW9rsZ8N0Mr1g0bOx1P7Xr\ngmrXWe6jOAwpDkKIGkU1aITqexr0PQ1rLWSu9orEskXw8xLv0tiZU7xikdFyXxfUsR1lcLsEKQ5C\niBpLKQVNW6CatoBBZ3trXK9a4Y1RLP8RVizDrl2JnfYh+HzQsi2qfRcKz7wAouPDHT+spDgIIWoN\n5fNBaz+qtd8brygs8ArEnjOL337G/rqcDZ+OQ/U5FXXuZaiUOuGOHRZSHIQQtZaKjoH2XbxxiPOv\n8BYjWroQ57P3KZo5Bfv916ihF6MGDkVFRYc7bpWSuZWEECJIJSSiTuhD4+f/h7r0RnB82HFjcR+4\nxbsCytpwR6wyUhyEEOIAyheFM+AsnEdfRg06GzZvxH3hUdzR92PXrgx3vCohxUEIIQ5BJSbjDL8e\n58F/Qafu3hoUD92G+/aL2Jwd4Y4XUlIchBCiDKpJM3wjH8D50wPQKB07YzLuvTfhTp2ILSoMd7yQ\nkOIghBDlpDp3x3lgDGr4DaAUdtzruA/cil30XY0bj5DiIIQQR0BFReEMGuqNRwwcCps34D7/CO6z\nD2DXrQp3vEojxUEIISpAJaXgXHIDzgNjoGNXWPoD7qiRuP99GZuTHe54R02KgxBCHAWV3hxn5IM4\nf7ofGjXBfvUp7t9vxJ32YbUej5DiIIQQR0kphep8As4D/0JdfB0A9r3XcB/8E/bH76vleIQUByGE\nqCQqKgpn8Dk4j/wbNeBM2LQe918P4z77IHbd6nDHOyIyfYYQQlQylZziLUrU70xc8yosXYj70J+g\n3XGoDl1RHbt6EwJG8EJEUhyEECJEVNPmOLeNgh/n4U76Hyz9Abv0B+z7Y72FiDocDx26ojocH3ET\n/ElxEEKIEFJKQZce+Lr0wG7f6s3+unQh9qeF2DlfwpwvvbUlmrf2zio6HA9tOqCiwzvRnxQHIYSo\nIqpOPVSvAdBrANZ1Ye1K7NKF2KU/wC8/YVf/hp08HmJiwd/ZO6Po2BUaZ1R5F1SVFAet9WigJ2CB\nkcaY7w9yzONAL2NM//K2EUKI6ko5jne20Lw1DLkQm5/vrVQXPKtg8Tzs4nneWUW9BqgOXb0uqPbH\noZJSQp4v5MVBa90PaGuM6aW1bg+8DvQ64JgOwClAYXnbCCFETaJiY6Fzd1Tn7gDYrZuxy36An7wz\nC/vN5/DN51iloEWbvQPbNi0tJHmq4lLWQcBEAGPMMqCu1vrAsvcMcO8RthFCiBpL1WuA03swzg13\n4vzzPzj3PoM673Jo2wHW/Ib91OD+424yrzwDu2VTpb9/VXQrNQbml3i+KbgtG0BrPQKYAawsb5tD\nSU9Pr3DIo2kbDtUpr2QNneqUtzplhQjMm9EMTu4HgLtrJ/mL55O3YC5FWeup17wFvtTKvdopHAPS\ne0dVtNb1gKuBwUDT8rQ5nMzMzAoFSk9Pr3DbcKhOeSVr6FSnvNUpK1STvM3aQLM2+7Lu3FWhlzlU\nEayK4pCJ961/bxZgffDxQKAh8DUQCxwTHIg+XBshhBAhVhVjDlOBiwC01t2ATGNMDoAx5n1jTAdj\nTE/gfGCBMeb2w7URQggReiEvDsaY2cB8rfVsYAxws9Z6hNb6/CNpE+qcQggh9qmSMQdjzN8O2LTo\nIMesBPofpo0QQogqIrOyCiGEKEWKgxBCiFKkOAghhChFioMQQohSVHVcvu4QaswPIoQQVazUjcY1\nacruyF1SSQghqhnpVhJCCFGKFAchhBClSHEQQghRihQHIYQQpUhxEEIIUYoUByGEEKXUpEtZKyS4\nfkRPvPskRhpjvg9zpEPSWj8F9MX77/a4MeaDMEc6LK11PLAEeNgY80aY4xyW1voy4K9AEXC/MeaT\nMEc6JK11EvAfoC7eOiijjDFTwpuqNK11J+BDYLQx5nmtdTPgLcCHtz7LFcaY/HBm3OMQWccC0Xhr\n219ujNkQzowlHZi3xPbTgcnGmKO+tL9WnzlorfsBbY0xvYBr8aYHj0ha6wFAp2DWIcCzYY5UHn8H\ntoY7RFm01vWBB4A+wFDg3PAmKtMIIGCMGYC37slz4Y1TmtY6EfgXML3E5oeAF4wxfYEVwDXhyHag\nQ2R9BHjFGNMPmAD8ORzZDuYQedFaxwF3U0kLo9Xq4gAMAiYCGGOWAXW11inhjXRIM4FhwcfbgUSt\ntS+MeQ5La90O6ABE7DfwEgYD04wxOcaY9caYG8IdqAybgfrBx3WDzyNNPnAm3qqOe/QHJgUff4T3\ne48EB8v6R2B88PEm9v2+I8HB8gLcA7wAFFTGm9T24tAY7z/8HpvYf3nSiGGMKTbG7Aw+vRb41BhT\nHM5MZXiGCPq2VYaWQILWepLW+mut9aBwBzocY8y7QHOt9Qq8Lw13hDlSKcaYImPM7gM2J5boRsoC\nmlRxrIM6WFZjzE5jTHHwC9jNwP/Ck660g+XVWh8LdDHGjKus96ntxeFAET8Fh9b6XLzicEu4sxyK\n1vpKYI4x5vdwZyknhffN8AK8LpuxWuuI/X9Ba305sNoY0wZvHfbny2gSiSL297tHsDC8BXxhjJle\n1vFhNppK/jJW24tDJvufKaRTSf11oRAcbLoXOMMYsyPceQ7jLOBcrfVc4DrgPq11pHQhHMxGYHbw\nG9mvQA7QMMyZDqc3MAXAGLMISI/kLsYScoMXKQA0pXS3SKQZC/xijBkV7iCHo7VuCrQD/hv8N9dE\naz3jaF+3tl+tNBUYBfxba90NyDTG5IQ500FprVOBfwCDjTERPchrjLl4z2Ot9YPASmPMtPAlKtNU\n4A2t9ZN4ffhJRGY//h4rgJOA8VrrFkBuhHcx7jENuBB4O/j35PDGObTg1WsFxpgHwp2lLMaYdcAx\ne55rrVcGB9KPSk2asrtCtNZPAKcALnBz8JtYxNFa3wA8CPxcYvOVxpjV4UlUPiWKwxthjnJYWusb\n8brrAB4xxkw63PHhFLyU9XWgEd4XvPuMMV+EN9X+tNbd8cadWuJdCroOuAx4A4gDVgFXG2MKwxRx\nr0NkTQPygOzgYUuNMX8MS8ADHCLvBXu+NAaLQ8ujfZ9aXxyEEEKUVtvHHIQQQhyEFAchhBClSHEQ\nQghRihQHIYQQpUhxEEIIUYoUByEigNa6f3A6DCEighQHIYQQpch9DkKUQ3BOq0eARLw7lC8Fnga2\nAccDxwLzgeHGmF1a6+OAl/DmbMoD7tqz5oLW+i7gRry1Iz4G/gL0A17Fm7LhciAGuM4Y82VV/YxC\nlCRnDkKUQWvdGm8CtkuMMa2BL4GXg7vPx1tToRmQClyvtXaAd4HnjTHt8OaXekdrnay17hN83gXo\nhLeGxEXB18oAFhtj2uMVlr9Xxc8nxMFIcRCibEOAr4wxS4LPXwbOwVvR7ENjzBZjjIu3NsjJQCu8\nCR3fBTDGzMObLqIH3jz8nwTXjijAW+Ngz4p+2SWm7ViIVyyECIvaPvGeEOVRBzhFa728xLYdeF1G\nJQeRt+FN3NcQ2G6MsQfsSwMaUGI2UmPMLgCtNeybxwegGK/4CBEWUhyEKFsm3kpxF5XcqLV+A+/D\nfo96eMuibgTqaa1ViQJRP7h9c8k2wSVKhYg4UhyEKNsU4EmtdWtjzG9a6xPxZhgFGKK1roO3BsR5\nwHvASmAtcDHwrtb6ZLxupu+A3YDRWt8XbDMRb3wh0tc2ELWMjDkIUQZjzHrgemCC1noZ3spr7wV3\nT8cbM1iL13X0evBsYThwS/D4McCw4NKTc/HW5fgBWAosAN6pyp9HiPKQS1mFqKBgt9IKY8wj4c4i\nRGWTMwchhBClSHEQQghRinQrCSGEKEXOHIQQQpQixUEIIUQpUhyEEEKUIsVBCCFEKVIchBBClPL/\nt41zc9wkuWUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e90ef6898>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(historyBilstm.history['loss'])\n",
    "plt.plot(historyBilstm.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 298,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1625,
     "status": "ok",
     "timestamp": 1521565593082,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "WG4-CXXXRosK",
    "outputId": "7d6886c7-84b2-459a-fe32-a6e22038e4a1"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEZCAYAAAB8culNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4XNWd//H3HTWrWLZcJbnbscfd\nxl3YxhVCy9JPyCabNQubZGN+gc1mEwKbDaSRhCQkQArZEJOEEDgBTByq44qNMTZugMu4V9mSLFu9\njeae3x/3Sh6rW9I06/t6Hj2auWXmIxnmq3vOPedYxhiEEEKIYJ5IBxBCCBF9pDgIIYRoRIqDEEKI\nRqQ4CCGEaESKgxBCiEakOAghhGhEioMQHeD1eud7vV7j9Xr7tPF44/V6b29m3zqv1/tU5yYUon2k\nOAghhGhEioMQQohG4iMdQIhQ8nq9BvgX4D+AK4DdwB3AfwOfBcqB+30+31/d47OAXwDzgFRgE3Cf\nz+fb6+6fAvwWGAN8DCxr8H5ZwBPAXCAN2Ajc6/P5DrYj+78CXwNGAAXAb4Af+nw+4/V6+wK/AhYA\n3YC9wNd9Pt9ar9frAR51f75eQC7wuM/n++WlZhBdl1w5iK7gK8DngGFAJvAOsAHoB6zA+TCv8wqQ\ngPPhnw3kAyu8Xq/H/dB9GfgA6AMsAZY2eK+/AZWAF8gCTriveUm8Xu+1OMXga0A6ToH7pvsd4Pvu\n9uFAT+APwHNerzceuBP4PDAHp8D9K/ADr9c74VJziK5LioPoCl70+XxHfD5fHvAecN7n873g8/lq\ngL8DmV6vN83r9U4CZgH/7fP5zvl8vhLgQeATwDRgOjAU+J7P56t0ryaeqXsT96piunt+sc/nK8X5\ncB/n9XqnXWLmLwIv+3y+t30+X63P53sHeAnngx+cglADVLj7nwIG+ny+WndfACjz+XzG5/O9C2T4\nfL6PLjGD6MKkOIiu4ETQ4wrgVIPn4DTNDAdqgpuAfD7fcZwP4RHAQHf/yaDzdwc9HuV+P+b1equ8\nXm8VkAfYOEXlUgwH9jTYdtDNAfBDYDJwyuv1Puf1ej8DxLn7/gIcAY57vd7XvF7vV3AKhhBtJn0O\noiuwW3leJwmwmthuAcbd3/APquDnle5rp/p8vkA7cjbM0hQD4PP5tnu93uHAYuAGnKaxpV6vd57P\n5zsPzPV6vdOBG3H6Wx7yer0zfD7fsQ7mEl2EXDkIccEhIMHr9XrrNni93lE4fRAHcK444r1eb3bQ\nOcHt+Adw/p+aGHS+5fV6h7YzS8M+gvHue+D1ensC+Hy+N30+373ATGA2MMnr9SZ5vd7uPp9vq8/n\n+7abpwy4rR05RBclxUGICz4APgJ+6PV6e3i93gyc5ptdwHbgfaAQeNDr9SZ7vd7xOJ3SAPh8vj3A\nOuBxr9eb7fV6uwHfAt5zH1+K3wO3er3exV6vN97r9S4GbnW3A2wGvuf1elPdjvKZQDVwDOcq4hX3\nzimA0UAGsP8SM4guTIqDEC6fz2eAf8Jpbj2Ic3toDXCt27FbBXwK5y6gQpwP6kcbvMzn3H37gDPA\nVcAn3XMvJcvLOJ3ZTwDngceBe3w+33L3kDuAGcBpoAj4KnCrz+crBL6B09fxkdfrrcC5W+oHPp/v\ntUvJILo2S1aCE0II0ZBcOQghhGhEioMQQohGpDgIIYRoJCzjHJRSj+OMPDXAfVrrrUH7luJ04gWA\nD7TW9yul4nFGno5wM35Na70xHFmFEEKE4cpBKTUPGKm1zgHuJmgeG6VUOs4EaHO11nOAsUqpWTjz\nx5S72+4GftaGtzLyJV/yJV/y1a6vRsJx5bAIeBVAa71XKZWhlErXWpfg3CZYA6QppcqAFOAc8BzO\nFADgzEbZuy1vlJub266A2dnZ7T43EmIpr2QNnVjKG0tZIbbydjRrdnZ2k9vDURwygW1BzwvcbSVa\n6yql1CPAYZypB17QWtcN1PG73+8Hnm/LGzX3Q4b63EiIpbySNXRiKW8sZYXYyhuKrJGYW6l+7hq3\nWelBnAnLSoA1SqlJWutd7v6lwBScgUetkiuH6CNZQyeW8sZSVoitvKG6cgjH3Uq5OFcK9VlwRnWC\nM2f+Ya31Wa11Dc4c+1MBlFJ34xSFm7XWfoQQQoRNOIrDSuB2AKXUFCBXa13q7jsKjFFKJbvPpwEH\nlFLDgS8Bt2qtL2naASGEEB0X8uKgtd4EbFNKbcKdVlgptUQpdYvWOg94DFirlNoI7NBabwDuwemE\nfkMptc79Sgx1ViGEEI6w9DlorR9osGlX0L6ngacbHP8gTl+EEEKICJAR0kIIIRqRleCEECLGmKJz\ncOIw5vhhiiwwi2/CSmxu8cD2keIQYuvXr2fevHmtHvfUU09x2223kZWV1eqxQoiuwdg2FJzBHD8M\nJw65349ASVH9MaXx8XiuuBKyBnbqe0txCKEzZ86wZs2aNhWHe++9NwyJhBDRyvj9cPq4UwCOH8ac\nOAwnjkJ15cUH9u4Hk2diDRqONXg4mbPmkldV0+l5pDiE0M9//nP27dvHwoULWbx4MWfOnOEnP/kJ\nP/7xjykoKKCqqoolS5aQk5PD/fffz3333cf69espLy/nxIkT5ObmsnTpUmbOnBnpH0UI0YlMZYXT\nLHTiiFMIjh+G0ycgUHvhIMsDWQOxBg2DwcOxBg13vqd2v+i14nr1gRAM2OsyxcH+6zLMtneb3Jcb\nF0cgELjk17SmzsZzx13N7r/zzjtZvnw5w4YN4/jx4zzxxBOcP3+eadOmce2115Kbm8vDDz9MTk7O\nRefl5+fzwx/+kC1btrBixQopDkLEMGMH4NhhzL5dmGMH4fhhKDhz8UGJic4H/+Dh4F4RMGBIp/cj\nXIouUxwibfTo0QB0794dn8/Ha6+9hsfjoaSkpNGxEyZMAKBv376Ul5eHNacQouPMuQLMnp2wZydm\n704oK72wM7U7jJnkXAkMGuYUgv4DsOLiIhe4CV2mOHjuuAua+Ss/HPOoJCQkALBq1SpKSkp44okn\nKC0t5Ytf/GKjY+OC/iORNb6FiH6mugr278bs3u4UhdMnLuzM6IM1ZxaMnYw1fDT06oNlWc2/WJTo\nMsUhEizLatRcVVJSQlZWFh6Ph3feeYfa2tpmzhZCRCtj23DyCGb3TsyeHXBwD9T9v5yYBBOmYY2d\njDVuCmQOiIli0JAUhxAaMmQIBw4cICsrix49egBw1VVX8dBDD7Fnzx6uu+46+vbtyx/+8IcIJxVC\ntMYUnXObinY430uLL+wcPAJr3GSssVfAiDFYbktBLLMuo2YLI1N2Rx/JGjqxlDeWsoKT99TRI3Bw\nD2b3DszuHXDq2IUDevTCGjsZxl2BNWYSVnrPiGbthCm7G13ayJWDEEK4TF4u5sOtFBzcjf3RdvC7\n4wcSEp1CMPYKrHFXQPbgmGwquhRSHIQQXZYJBODQPsyHWzC7tsKZkwBUAQwc6haDyTByHFZC15oY\nWoqDEKJLMZUVsHs7ZtdWzMcfXLjNNDHJGXk8aQZZi28IyajjWCLFQQhx2TNn85xi8OEW8H18YSRy\nz95Y867FmjQDvBPqB52FatRxLJHiIIS47BjbhiP7MR9uxezacnFn8uARWJNmOAVh8PDLvu+gvcJS\nHJRSjwOzAAPcp7XeGrRvKfA5IAB8oLW+XymVADwLDHG336W1PhyOrEKI2GSqq5wRybu2YD7ceuFW\n04REZ9zBpBlYE6djZfSObNAYEfLioJSaB4zUWucopcYAvwdy3H3pwH8Dn9Ba1yqlViqlZgFeoEhr\n/Vml1DXAo8CnQ501Uu68806WLVtGcnJy6wcLIeqZc2edq4MPt8LeXVDrd3ak98SaczXWpOkwZjJW\nUrfIBo1B4bhyWAS8CqC13quUylBKpWutS4Aa9ytNKVUGpADn3HP+6J6/CqegCCEExl+D2bYJs/4t\nZ2RynYFDsSbOcArC0JFYHlnosiPCURwygW1BzwvcbSVa6yql1CPAYaASeEFrvV8plekeh9baVkoZ\npVSi1rrF2wfcwRzt0pFzm3PLLbfwy1/+0hlQc+oUS5cupX///lRUVFBVVcW3vvUtJk6cSHx8PJmZ\nmaSmpkY0b6hI1tCJpbwdzerPPUH5m69QvmoFpsRpMkqaNIPknHkkz7yK+H6du1BWV/rdNiUSHdL1\nvT9us9KDwCigBFijlJrU0jktaWmU4LLt+Ww63ngGVIC4uHgCgUuf4+jKwencNaVfs/tnzpzJ8uXL\nueWWW3jllVeYOXMmI0aMYM6cOWzfvp0nnniC73znO9TW1nLmzJk2NyvF0mhTyRo6sZS3vVlNbS18\nuAV7/VuwZ6ezMS0d69rbsOZeQ22/LEqB0lrTqXcXdYXfbfD5TQlHccjFuVKozwKcdh+PAQ5rrc8C\nKKU2AFODztnldk5brV01RKO5c+fy61//mltuuYV3332XL3/5y2itefHFF/H7/XTrJu2gQjTFnCvA\nbFiJ2fAPKD7nbBw1Duuqa7GmXHlZzF0U7cJRHFYCjwBPK6WmALla67rJzY8CY5RSyVrrSmAa8AbO\nAMU7gLeBTwFrOxririn9mv0rP1R/JQwbNozCwkLy8/MpKyvj3XffpU+fPjz44IP4fD5+/etfd/p7\nChGrjB2A3Tucq4QPPwBjQ3Iq1qJPYV31SazswZGO2KWEvDhorTcppbYppTYBNrBUKbUEKNZaL1dK\nPQasVUrVApu01huUUnHA1UqpjUA1sCTUOUNl1qxZ/O53v2P27NkUFxczfPhwADZs2CDTdQsBmJLz\nmI2rMO+8DYX5zsZho5zBadPmYiVFbjW0riwsfQ5a6wcabNoVtO9p4OkGxweA5tffjCFz585l6dKl\nPPPMM1RVVfHoo4+yfv16br75ZtasWcObb74Z6YhChJ0xBnwfYda/hdnxHgQCkJiENfcapygM+USk\nI3Z5MkI6xEaPHs3q1avrnwev3TB79mwArrvuurDnEiISTHkpZtMa5zbUvFPOxgFDsOZdhzVzHlZK\n2+/YE6ElxUEIEVLGGKr3fYT90h8xH7zrTIMdn4A1awHWvGthxGiZwiIKSXEQQoSMycvFfuH/yP/Y\nHerUL8tpNspZhNU9PbLhRIukOAghOp2prsK88VfMyuVQW0vSpOn4F/2TM/OpjFyOCVIchBCdxhgD\n2zdh62fg3Fno1QePupu+N97O6dOnW38BETWkOAghOoU5fQL7L791JsCLj8e6XmFdfztWUjfpU4hB\nUhyEEB1iqiowf38Rs3qFc0vq+Kl47vx3rP6xMzeRaEyKgxCiXYwxmPfXY1561pniok9/PJ++BybN\nkCuFy4AUByHEJTMnj2A//zQc2AMJiVif+gzWtbfWL7MpYp8UByFEm5mKMsyKv2DWvg62DZNn4VH/\nhtU3s/WTRUyR4iCEaJWxbcx7azAv/8FZfrNfNp7P/DvW+KmRjiZCRIqDEKJF5thBpwnpsM+Z/+iW\nf8G6+maZNvsyJ8VBCNEkU1aCefU5Z7ZUY7CmzcG64y6sXn0jHU2EgRQHIcRFjB3AbPgHZvmfoLwU\nsgbh+cwXsMY0tUijuFxJcRBC1DOH9jkD2Y4dhG7JWHf8G9bCG7Hi5aOiq5F/cSG6OHPmJGbrRswH\nGyH3OADWrPlYty3B6tkrwulEpEhxEKILMvm5FwrCyaPOxvgE59bUq2/CGjUuovlE5IWlOCilHgdm\nAQa4T2u91d0+APhz0KHDgQeAdcDvgSQgDvhPrfW2cGQV4nJlCs5gtr2L2boRjh9yNsbFOyOap83G\nmjQTKzklsiFF1Ah5cVBKzQNGaq1zlFJjcD70cwC01qeA+e5x8ThFYQXwMLBca/20UupK4PvAtaHO\nKsTlxpwrwHyw0Vlk58h+Z2NcHIyfijV9DtbkmVgpaZENKaJSOK4cFgGvAmit9yqlMpRS6VrrkgbH\nLQFe1lqXKaXOAr3d7RnA2TDkFOKyYIoKMR+86zQZHdrnbPR4YOxk53bUK2ZhpclCO6Jl4SgOmUBw\nk1CBu61hcbgHuMZ9/DiwRSn1eSAdmNOWN8rObv8skB05NxJiKa9kDZ26vIHzhVS8u5rKd/5B9Z6d\nYAx4PCRNnEbKVVeTfOVC4npkREXWWBFLeUORNRId0o2ma1RK5QD7gq4m/hvQWuvvK6VuBH4C3Nra\nC+fm5rYrUHZ2drvPjYRYyitZQ6d/ajKn31iO2boB9u8GY4NlwcixzhXClCup7ZFBCVBSXgnllRHL\nGmu/21jK29GszRWWcBSHXJwrhfosQMMloW4EVgU9nw38j/v4H8CvQpZOiBhiCvMxO7dgdr1Pru9j\nsAPOjhGjsabPxZpyJVZG75ZfRIg2CEdxWAk8AjytlJoC5GqtSxscMx14Iej5QWAmTnPUdOBAGHIK\nEXWMMXDsIGbXFszOLXDySP2+xFHj8E+aiTV1NlZvmdJCdK6QFwet9Sal1Dal1CbABpYqpZYAxVrr\n5e5hWUB+0Gk/AJ5RSin3+VdCnVOIaGH8fvB9iNn5PmbXVigqdHbExzt3GU2agTVpBv3HTYiZpg8R\ne8LS56C1fqDBpl0N9k9o8Pw0cH2ocwkRLUxZCeajbZid78PuHVDt9g+kdsfKWYA1aSaMm4zVTcYh\niPCQEdJCRIjJz63vP+DAXqdDGaBfFtbkT2JNmgEjxmDFxUU2qOiSpDgIESbGtuHIfsyu953+g9Mn\nnB2WBcO9zgjlyTMgc6CswSwiToqDECFk/H7Ys8PtP9jirKIGkJjoTFsxaQbWpOlY6ZEdgyBEQ1Ic\nhAgBYwxs34T912VQ6N5r0b0H1pyrneaiMZOxkpIiG1KIFkhxEKKTmWMHsV/8HRzYA3HxWAtuwJo5\nD4aNxPJI/4GIDVIchOgkpqgQs/w5zHtrnOkrJs/Ec/tdWP1jZxoGIepIcRCig0xNNeYff8O8+RJU\nV8HAoXjU3bKspohpUhyEaCdjDGbrBszLf4BzBU6fgroba85iaT6KUbW2YdPxUsqOVlNcUoJtcL8M\ntoGAbeof2wYC9Y+D9xN0zIXvBuifmsDQjCSG9ezGkIwk0hKj978TKQ5CtIM5st/pVzi0D+Ljsa69\nDev6O2SxnBjlD9isPlzMy7vPkV/uD9v79k2JZ2hGN4b2TGJYRhJDMpLISkskzhP5W5mlOAhxCcy5\ns5jlf8JsXutsmHIlntuXYPXNbPlEEZWqa21WHixi+Z5zFFbWkuCxuGFUT66bNJTz5wqJsyw8HvBY\nFh4L57l14bnHsojzXPy8uf3GQG5pDUfOV3GsqJoj56s5WlTN1lNlbD1VVp8pMc5iSM8khvZMiuhV\nhhQHIdrAVFdj3n4F8/YrUFMNg4fjUfdgecdHOppohwp/gLf2F/HqvnMUVwXoFm9x85he3DSmF72S\n48nO7kVuYlWnv++QnkkM6XnxLcxFVbUcPV/tFowqjrqF40Dhxe/f3FVGqEhxEKIFxrYxW97BvPJH\nOH8WemRgfeYLWFculH6FGFRWHeC1/ef5+75zlNXYpCR4UON78ylvBundIvNx2LNbPJOz4pmclVq/\nrdY2nCx2riyOnq/7XtXkVcbEAXncN71Pp+eX4iBEM8yhfU6/wpH9EJ/g9Clcd5tMfheDiqtqWbHv\nPG/sP0+F36Z7oofPTuzD9d6MqOwUjvdYzlVCRjcYdmF73VXG0aKq+qJxsqiS6oDp/Ayd/opCxDhT\nWIB55Q+YLe8AOIvo3Pp5rD79I5xMXKpzlbW8uqeQtw4UUR0w9OwWhxrfl2tHZpCc4Il0vEvW1FVG\nqFatk+IghMuurMD+258xby8Hfw0M+QSeT9+DNXJspKOJS5Rf5ueVPYWsOlSM3zb0Tonn82N7cfWI\nniTFx15RiAQpDqLLqxuvcOaVP2AKC6BnL6xbPo81az6WRz5IYsnp0hpe2l3I2sPFBAz0T0vg9nG9\nWTAsnYQ4+be8FFIcRJdm8nOx//w07NmBnZiEdeOdWNfeipXULdLRxCU4XlzNSx8XsuGYM3BtQHoi\nd4zrzVVD06NizEAsCktxUEo9DswCDHCf1nqru30A8OegQ4cDD2itn1dKfQ34HOAHvlx3jhCdwfj9\nzq2pr2uo9cP4KWTe/7/kG/nrMpYcPleF/riQzSdKMTi3iqrxvckZ1F2KQgeFvDgopeYBI7XWOUqp\nMcDvgRwArfUpYL57XDywDlihlBoH3AlMAyYCNwFSHESnMPs+xP7zr+HMKejRC8+d98DU2cRnDQBZ\nkzmqVfpt9hZU8FGe81U3FuATvbqhJvRm+oA0PLJQUqcIx5XDIuBVAK31XqVUhlIqXWtd0uC4JcDL\nWusypdSNzuG6FtjufgnRIaakCPPSMsx7a8GysBbeiHXz52TKiyhWXWuz72wlH52pKwaV1N21GWfB\nxP4p3DK2F1dkpcrqeZ0sHMUhE9gW9LzA3dawONwDXOM+HgoElFJvAQnAV7XWu1p7o+zs9k+N3JFz\nIyGW8kY6q7Ftylf+jeJlT2LKSkgYMZpe/+9BEpu4CynSWS9VLOVtS9aaWpuPTxez7XgRH5w4z0e5\nxfjdauCxYExmOlMH9WTa4AwmDehBSmLoPsIut9/tpYpEh3Sj8q6UygH2BV1NWEAccB0wG/gdML21\nF27vvb6huk84VGIpb6SzmpNHsZ/7lTNBXrdkrDu/QGDBdZz1xDVqQop01ksVS3mby1prGw4WVvFR\nXjkf5VWwt6CSGrcYWMCwjCQm9E9hQv9UxvZLJrV+wFo1RWfzKQpz3mjU0azNFZZwFIdcnCuF+izA\n6QbH3AisCnqeh1MsDLBRKTU0pAnFZcdUV2H+/gJm1d8gEMCaOhvrznuwevaOdLQOMcaQW+pn/9lK\n7JN+zheXXDw9tO18DzQzbXTwcQ2nm677nhTvITXBQ2piHKkJHlISPaQmxJGS6CEtIY7URA8p7vfk\nBM8ltfEHbMPh81V8lFfBx3kV7M6vpKrWrt8/pEcS4zNTmNA/hfH9UuieFH2jl7uKcBSHlcAjwNNK\nqSlArta6tMEx04EXgp6/CXwJ+ItSajRwIgw5xWXC7NqC/fzTzhoLffrj+ecvYU2YGulY7VJWHWB/\nYSX7z1bhO1vJ/sJKymrs1k8MEwtISfCQ4haTlKCiElxE0k/62Xwoj915FZT7L+QfkJ7IhP4pTOyf\nwrj+KfSM0PxGorGQ/0torTcppbYppTYBNrBUKbUEKNZaL3cPywLyg87ZrJS6Tin1nrtpaahzithn\nzhVgv/B/sGOzs3bz9XdgXa+wkpJaPfd0aQ0vrTuIqamgf1oimWkJZHZPpHuiJ2wdnQHbcLy4Gt/Z\nSnxnq9h/tpKTJTUXHZOZlsDU7DS8fZIZPbg/RefOEedpfrpo5zt4PFbQdNPNTEHtAQuL6oBNeY1N\nhT9AeY1NeU2Acr/zvcIf/PzCMRX+AAXlfir8Nk3P8lNQnz9ncHcm9k9hfP8UeqckhPz3KtrHMqbz\nJ2yKECN9DtEnHFlNIIBZ/XfMiuedZTpHjsXzuS9jZQ9u0/kbj5Xw1OYzVNY2/os8JcFD/7QEMtMS\n6otG/7QEsron0iclgYS49heOc5W17D9b6VwRnK3kQGHVRROoJcd7GNWnG94+yYzqncyoPt3oEfSX\ndTT+d2AbQ6XfblBEAqT3yKC3p5K+qbFRDKLxd9ucTupzaPQfslzDiZhmDvuw//QrOHkE0rpjfeaL\nznTabfhrvyZg8/tt+bx5oIhu8RYPfXI0nuoy8sr8nC6rIa/MT16pn9ySGo6cr250vseCPinx9E9L\nvKiAZHV3vgdfddQEbA6fq7sqqORAYSX55bX1r2UBg3sk1RcDb59kBqRHx4pgl8JjWU6zUmLcRYUg\nO7tfzHzYCocUBxGTTEWZsyLb+rfAGKzZi7FuX4KVlt6m80+X1vDYxlMcOlfNkB5JfH1uNjPGNP0X\nmDGG4qpAfcE4U+Ynr6yGM6V+8sr87oCsxu9Rd9URZ1kcLaoi+MKkR1Ic0wek4XWLwSd6dyMlQTpf\nRfSQ4iBiivH7MVs3YF5+FkqKIGuQ04Q0alybX+Pd404zUoXfZvGIHnxhWv8WZ+q0LIueyfH0TI5n\nTN/G+6trbfLL/W7hqHGLh58zpTWcKqnBNoZhGd0Y1ScZb2+nGPRPS5BBWyKqSXEQUc/YNhzah9m8\nDvPBRqgog8REZ42Fq2/Cim9bO7Y/YLNsez6v7y8iKc7i/pwsFgzv0eF8SfEeBvVIYlCPxh3fxr2t\nND7GmoeEkOIgopY5fdIpCO+vg0L3ZrYevbCuuRlr/vVYfTNbPD/YmdIafrwxl0PnqhjcI5Gvzx3Q\n5Id5Z7Msi3ipCyIGSXEQUcUUn8dsfQezeT0cO+hsTErGylmINWs+jJ5wyWs3v3e8lCc3n6bcb7No\neA++OL3lZiQhhBQHEQVMdRVmx2bM5rWwZxcYGzwemDDNWXBn0sw2jVVoyB8wPLsjn9d850mKs7gv\nJ4uFndCMJERXIMVBRIQJBGDvLsz76zA7NjvjEwCGjcKaOR9r+hys9J7tfv28shoe25jLgcIqBrnN\nSIPD0IwkxOVCioMIG2MMHD/k9CNsece52wigb6ZTEGbOw8oc0OH32XyilCc2n6a8xmbBsHS+NCOT\nbtKMJMQlkeIgQs6czcO8vx6zeR2cOelsTOvudCrPmg/DvZ1yW6c/YPjjznxW7DtPYpzFV2ZlsmhE\n+68+hOjKpDiIkDB2APP+O+S9vxZ7905nY3yCMztqzgIYd0Wbb0Fti+BmpIHpTjPSkJ7SjCREe0lx\nEJ3KGAO7t2O/9CycOkaNZYF3gtOxPOVKrJTUTn/P90+W8ov3nGak+UOdZqTkBGlGEqIjpDiITmOO\nH3KKwt5dGMti9ZzPkzduDinJCfRJiad3KfQJ1NArOb5TbiWttQ1/2lnAq3vPkRhnce/MTBaP6CEj\nj4XoBFIcRIeZwnzMq885fQpAzbip/HrcZ1ifb4Ov6bW6uid66J2SQO+UePq43xs+bmmuoYJyP49t\nPIXvbBUD0hP5+pxshmZ0C8WPJ0SXJMVBtJupKMO88VfM6teg1g+DhlFy0xIePdMLX34lo3p34xuf\nHMfR3DzOVvgprKh1v/ycrajlTJmfo0WNZzutk5LgcQuFe+XhFg9j4I878ymrsZk3NJ3/kGYkITqd\nFAdxyYzfj1n3BuZ1DeWl0Kt9bJGLAAAcq0lEQVQP1s3/wrFRs/j+O6coqKjkqiHp3Dsrk2EDetDH\nKm/2tSr8Ac42KBqFbiGpe3yiuKbReQkei6UzM7lampGECImwFAel1OPALMAA92mtt7rbBwB/Djp0\nOPCA1vp5d39/YB9wi9Z6XTiyiuYZ28Z8sBGz/E9wNg+SU7Fu+1esRZ9ia141P111gqpam89O7MMd\n43u36UM7JSGOwT3iWhygVlVr1xePwopaiqpqmZqdxmC5G0mIkGlTcVBKjQFu0Fr/RCk1HvgNzpKf\n92mtd7Ry7jxgpNY6x32d3wM5AFrrU8B897h4YB2wIuj0x4DDl/IDidAwvo+w/7rMme8oLh5r8U1Y\nN9wBqd15de85/rCjgIQ4i6/PzWb24LatqdBW3eI9DEhPZEB6Yqe+rhCieW1tqF0GHHEf/xJ4E/i+\n+7g1i4BXAbTWe4EMpVRTnx5LgJe11mUASqmFQCnwURszihAwp44TePK72D95CI4dxJpxFZ7v/grP\np++mNrk7T24+w7M7CshIjucHVw/u9MIghIiMtjYr9dBav6yU6gdMAhZprWuVUj9pw7mZwLag5wXu\ntpIGx90DXAOglEoEvg3cBPy8jRlFJzJFhZgVf8FsXOVMhDdqPJ7b78IaNhKAkqpaHn3nFHsKKhnR\nqxsPzRsgi8ULcRlpa3EwSqkU4E5gpVsYEoD2NPo2aohWSuUA+7TWdQXjAeD/tNZFSqk2v7C7UHa7\ndOTcSAhVXruinNKX/0Tp8ucw1VXEDx5Oz7u+Qrfps+v7EA6fLecbr+0it7iKRaP68vD1Y+nWwm2n\nsfS7jaWsEFt5YykrxFbeUGRta3H4FXASp59hkbvtz8DyNpybi3OlUCcbON3gmBuBVUHPPwnEKaXu\nBUYAM5RSd2itd7f4Ru1cwDw7u+m1g6NVKPKa2lrMxpWYFX+B0mJnUZ1P34N95SLOx8XBaeefbHtu\nGY9tzKXCb/PpCb25c0IvzhU0sYByCLOGSixlhdjKG0tZIbbydjRrc4WlTcVBa/2UUupZoEprXetu\n/o7W+uM2nL4SeAR4Wik1BcjVWpc2OGY68ELQ+82ue+y+77OtFQbRfmb/x9h//CXknXIW1rnps87y\nm0kXBpUZY3jNd57fb88nzrL4r9nZXDVU+heEuFx16G4lpdRXtNY7WzpXa71JKbVNKbUJ58pjqVJq\nCVCsta678sgC8tv9U4h2s7e8g/n9z8HYziypn/o0VnrGRcfU2obfbs3j7YNF9OwWx4PzBuLtkxyh\nxEKIcGhrs9IynNtK4cLdSh/gNDdd2drJWusHGmza1WD/hBbOXdLGjOISGGMw/3gV89dlkJyKZ+mD\nWN7G/wyl1QF+vOEUH+ZVMCwjiYfmDaRvqnQ8C3G5a+utrA3vVvqR1vptoHvooolQMbaNefF3TmHo\n2RvP1x9tsjCcLKnm628f5cO8CmYOTOPRq4dIYRCii4jE3Uoigoy/BvuZn8G2TZA9GM9938bq1bfR\ncTtPl/Pjjacor7G5fVxvPjupDx6ZpkKILiMcdyuJKGHKy7B/+T04sMcZt7D0QayUtEbHvbn/PL/9\nIA+PZXF/ThYLhveIQFohRCS1qVlJa/0UMBjI1FrX9Rd8R2v9jZAlE53KFBZg/+gbcGAP1rQ5eO5/\npFFhCNiG336Qx2+25tE9MY7vLRokhUGILqqtdytZwD8B17j9DnnA60BbbmUVEWZOHsH+xSNQdM6Z\nE+mOu7A8F/9dUFYT4LGNuew8Xc6QHkk8NH8A/dNkLiMhuqq2Nis9BszFaUo6D/QGvqmUGqe1fiRU\n4cTF/AGbdUdKSDodoLi4uG0n5eViNv4DUsfAnFlOx/P+xgvwvHWgiJMlNUwfkMpXZ2e3uNCOEOLy\n19bicB0wVWtdVbdBKfV/wBacAW4iDN46UMTvtuUDZy7hrHgYep3zsAzY1vxwkpvH9OLzk/sS55GO\nZyG6urYWhzig4ZJdlbT9VljRCVYfLibOgkduGEtZC1cOBoPZ+T68txYSk7Cuuw0re0iLr90rJV4G\ntgkh6rW1OKwDViilfsuFZqV7gLUhyiUaOHK+iiPnq5kxMI1PjskkN9du8jhjBzAvPoNZ85ozhuHe\nh7EGtFwYhBCiobYWh68A/wl8DeiL0yH9GvBkiHKJBtYcdq4UFg5r/u4h46/B/t3PYPsmGDAEz1e+\njdWrT7giCiEuIy0WB6XUAZylPeHiqbYHAl8CvgiMCk00UafWNqw/WkL3pDimDWg8LgHAlJdi//L7\nrY5hEEKItmjtyuGesKQQLdqeW0ZxVYAbRvUkIa5xZ7EpLMD+xcNw+gTWtDlY//afWAkyzYUQov1a\nLA5a6/XhCiKat+awswbSwuE9G+0zJ45gP+GOYbj6JqzbG49hEEKIS9XWPgcRISXVAbaeKmVIjyRG\n9Lp4Kiuzdxf2r34AVZVY6m48V98UoZRCiMuNFIcot+FoCbU2LBieXr9MJ4C9eR3m2SfAAusL/41n\n+twIphRCXG6kOES5NYeL8Vgwz71LyRhDyUt/wCx7ssV1GIQQoiOkOESx40XVHDxXxdTsVHolx9eP\nYShe8xpk9HGm25YxDEKIEAhLcVBKPQ7Mwrkt9j6t9VZ3+wCc+ZrqDAceADTwDDDCzfg1rfXGcGSN\nJnVjGxYN7+Gs3PbcrzEbVpIwZASBLz8kYxiEECET8ttalFLzgJFa6xzgbuCJun1a61Na6/la6/nA\nYuA4sAL4F6Bcaz3HPednoc4ZbQK2Yd3RElITPUwbkIrRz2A2rITBI+j3499JYRBChFQ47nlcBLwK\noLXeC2QopdKbOG4J8LLWugx4Dviqu70AZ7qOLmXn6XLOV9Zy1ZB0El5/AbNqBWQNwnP/I3jSZHVW\nIURohaNZKRPYFvS8wN1W0uC4e4BrALTWfsDvbr8feL4tb5Sdnd3ukB05NxSe/MBZKuOG8v2Y114k\nPmsg/X70W+J6O0t6RlvelkjW0ImlvLGUFWIrbyiyRqJDutEQX6VUDrBPa13SYPtSYArwqba8cG5u\nbrsCZWdnt/vcUCirDrD+QAED4mvIev4nkNEH+yvfJq/aD7m5UZe3JZI1dGIpbyxlhdjK29GszRWW\ncBSHXJwrhfoswOkGx9wIrAreoJS6G6co3OxeSXQZG46V4LcNCw6uwureA89Xv4PVp3+kYwkhupBw\n9DmsBG4HUEpNAXK11qUNjpkO1K1NjVJqOM7EfrcGLzDUVaz9+BQeYzOveB+e//wOVubASEcSQnQx\nIb9y0FpvUkptU0ptAmxgqVJqCVCstV7uHpYFBC9Rdg9OJ/QbSqm6bddorWtCnTfSTn6wHV9lCpOL\nD9L3y/+FNWhYpCMJIbqgsPQ5aK0faLBpV4P9Exo8fxB4MNS5oo3Z/zGrV22FQfNYeMVQrOHeSEcS\nQnRRMn1nlDBHDuB/8nus6zeZFI9h1oyxkY4khOjCpDhEAXPyKPbPv81HKQM5l9SDOcN7khQv/zRC\niMiRT6AIM2dOYT/+v1BRxrqcfwZaXgpUCCHCQYpDBJnCfOzHvwUlRVTc+R9srkolq3sCo/smRzqa\nEKKLk+IQIaboHPbPvgXnzmLd+q+8NySHmoBh4bAeF63bIIQQkSDFIQJMaYnTlJR/Gut6hee621h7\nuBgLWDBcmpSEEJEnxSHMTEU59i8ehtzjWIs+hXXzZzldWsOegkom9E+hb2pCpCMKIYQUh3Ay1VXY\nT34Xjh3Emr0YS92NZVn16zYslKsGIUSUkOIQJsbvx/7VD+DgHqzpc7E+vxTL48E2hrWHi+kW7yFn\nsEzFLYSIDlIcwsDU1mL/9jHYsxMmTsf6t//E8sQB8HFeBQUVtcwe3J1uMrZBCBEl5NMoxIxtY579\nBezcDKMn4vnSN7DiL8xaIk1KQohoJMUhhIwxmD//BvP+ehgxGs/Sh7ASEuv3V/pt3jtRSv+0BMb2\nk7ENQojoIcUhRIwxmJeWYd55CwYNw/OV/8XqdnEB2HS8hKpaw4Jh6XhkbIMQIopIcQgR89qLmJWv\nQuZAZ02GlLRGx6w54ix8t0CmyxBCRBkpDiFgco9jVjwPvfvh+ep3sbo3/vDPK6vh47wKxvVLJrN7\nYhOvIoQQkSPFIQTMu6sB8Ny+BCujd5PHrHWvGqQjWggRjaQ4dDJTW4vZvBZS0mDSzKaPccc2JMVZ\nXCljG4QQUSgsK8EppR4HZgEGuE9rvdXdPgD4c9Chw4EHgL8CzwJDgABwl9b6cDiydtjuHVBShLXg\neqyEpqfC2FNQyZkyP/OHpZOSEBfmgEII0bqQXzkopeYBI7XWOcDdwBN1+7TWp7TW87XW84HFwHFg\nBfDPQJHWeg7wfeDRUOfsLPamVQBYsxc3e0zd2IZF0qQkhIhS4WhWWgS8CqC13gtkKKXSmzhuCfCy\n1rrMPWe5u30VMDsMOTvMlJbArq0wYAgMHtHkMVW1Nu8eK6VvSjzj+6eEOaEQQrRNOJqVMoFtQc8L\n3G0lDY67B7gm6JwCAK21rZQySqlErXVNS2+UnZ3d7pAdObdO6d/WUxSoped1t9J9wIAmj3lj9xkq\na20+M30wA5s5pi06I2+4SNbQiaW8sZQVYitvKLKGpc+hgUajvZRSOcA+rXXDgtHsOU3Jzc1tV6Ds\n7Ox2nxss8MYrEBdHydgplDbzeq9sPw7A9D6eiOcNB8kaOrGUN5ayQmzl7WjW5gpLOJqVcnGuBOqz\nAKcbHHMjTvNRo3OUUgmA1dpVQ6SZ44fg5BGYML3JcQ0ABeV+PjxTwZi+yWSny9gGIUT0CkdxWAnc\nDqCUmgLkaq1LGxwzHdjV4Jw73MefAtaGOmRH1Y9tmL2o2WPWHSnGIGMbhBDRL+TFQWu9CdimlNqE\nc6fSUqXUEqXULUGHZQH5Qc9fBOKUUhuBpcA3Q52zI4zf70yu170HjJ/a9DHGsOZwMYlxFrNlbIMQ\nIsqFpc9Ba/1Ag027Guyf0OB5ALgr1Lk6zYdboLwU65qbL5qOO5jvbBW5pX6uGpJOaqKMbRBCRDcZ\nId0JbLdJybqy+Sal+nUbRkiTkhAi+klx6CBTdA4+3g5DPoE1YEiTx1TX2mw8VkLv5HgmytgGIUQM\nkOLQQWbzWjB2iyOi3z9ZRrnfZv6wdOI8sm6DECL6SXHoAGOMc5dSfALWjKuaPW6tLAUqhIgxUhw6\n4rAPzpzEumIWVmrjxXwACiv87DxTzqje3RjYIynMAYUQon2kOHSA2dR6R/T6IyXYRq4ahBCxRYpD\nO5nqaszWDZDRB8ZOavoYY1h9uJh4j8XcIU3NNSiEENFJikM7mR3vQWUFVs4CLE/T4xYOnqviZEkN\nMwemkZYkYxuEELFDikM7taVJafUh6YgWQsQmKQ7tYArzYd+H8ImxWP2bntHQH7DZcKyEjG5xXJGV\nGuaEQgjRMVIc2sG8twaMwbpyYbPHbDlVRlmNzbxhPWRsgxAi5khxuETGtjGb1kBiEtb0OU0e4w8Y\naVISQsS0SCz2E9sO7KGqsJCzsz5J4TmbvLLzFJT7KSivJb/cT0G5n3OVtRhgRK9uDOkpYxuEELFH\nikMDxhjKamwKyv31H/YXvtdSUGhTctX3nYPXnrzoXI8FvZPjGdM3mX6pCVzvzYjATyCEEB3X5YvD\n7rwK/rTbx5H8ovoCUFVrN3lsggf6VJUwLFBGv0kT6ZeaSN/UBPqlJtA3NYHeKfHSvyCEuCx0+eLw\n0u5Ctp8uByAlwUP/tAT6pcbT1/3A7xf0vfsHa7H++CTWP/0znlmxs/i4EEJcqi5fHL42JxtSMjAV\n50lrZRGewKbVYFkt3qUkhBCXg7AUB6XU48AswAD3aa23Bu0bBPwFSAS2a62/pJRKA/4IZABJwCNa\n67dDkS01MY7sfmnk5pa0eJzJy4WDe2DMJKze/UIRRQghokbIb2VVSs0DRmqtc4C7cdaRDvZT4Kda\n6xlAQCk1GFgC+LTWC4DbgV+EOmdr2jIiWgghLhfhGOewCHgVQGu9F8hQSqUDKKU8wFxghbt/qdb6\nOHAW6O2en+E+jxhjBzDvrYXkFKwrciIZRQghwiIczUqZwLag5wXuthKgL1AKPK6UmgJs0Fp/U2v9\nglJqiVLqIE5xuKEtb5Sd3f5O4pbOrdq+mYLzZ0n95C30Gjas3e/RmTrys4abZA2dWMobS1khtvKG\nImskOqStBo8H4DQbHQVeV0rdgFMQjmutr1VKTQKeAaa19sK5ubntCpSdnd3iufaKFwGovCKn3e/R\nmVrLG00ka+jEUt5YygqxlbejWZsrLOFoVsrFuVKozwKcdh+fBY5prQ9prQPAamAcMBt4G0BrvQvI\nVkpFZM5rU16G2bEZMgfAcG8kIgghRNiFozisxOlUxm06ytValwJorWuBw0qpke6xUwEfcBCY6Z4z\nBChzi0fYma3vQK0f68rFWJYMcBNCdA0hLw5a603ANqXUJpw7lZa6/Qm3uIfcDyxz9xcDfweeBoYq\npdYDzwNfCnXO5ph3V4PlwcqZH6kIQggRdmHpc9BaP9Bg066gfQeBhtOblgEq1LlaY04dh6MHYMI0\nrJ69Wz9BCCEuEzJldwvMplUAeGbL2AYhRNcixaEZprbWGduQ2h0mzoh0HCGECCspDs35eBuUFmPN\nuAorISHSaYQQIqykODTDrpsuY/biCCcRQojwk+LQBFNaDB9uhYFDYfDwSMcRQoiwk+LQBPP+OggE\nsGYvkrENQoguSYpDA8YYZ2xDXBzWzPmRjiOEEBEhxaGh44fh5FGYOB2re49IpxFCiIiQ4tCAebdu\nbIN0RAshui4pDkGM34/Z8g6k94TxUyMdRwghIkaKQ7Bd70N5KdasBVhxEZkEVgghooIUhyD2u7IU\nqBBCgBSHeqaoEHbvgKEjsQYMjnQcIYSIKCkOLvPeOjA2lkyyJ4QQUhzAHduwaRXEJ2BNvyrScYQQ\nIuKkOAA1+z6CM6ewrpiFlZoW6ThCCBFxYVnsRyn1ODALMMB9WuutQfsGAX8BEoHtWusvuds/C3wd\nqAX+V2v9eqjylf/j74BMsieEEHVCfuWglJoHjNRa5wB34ywVGuynwE+11jOAgFJqsFKqN/BtnBXi\nbgRuClU+U11NxTsrIaMPjJkYqrcRQoiYEo5mpUXAqwBa671AhlIqHUAp5QHmAivc/Uu11seBxcAq\nrXWp1vq01voLoQpndmzCVJZj5SzE8sjYBiGEgPA0K2UC24KeF7jbSoC+QCnwuFJqCrBBa/1NYCiQ\nopRaAWQAD2utV4cinNmyAQBr9sJQvLwQQsSksPQ5NGA1eDwA+AVwFHhdKXWDu703cAswBFirlBqi\ntTYtvXB2dvYlhymdOYfAmAn0nDztks+NpPb8rJEiWUMnlvLGUlaIrbyhyBqO4pCLc6VQJxs47T4+\nCxzTWh8CUEqtBsYBecAmrXUtcEgpVYpzlZHf4hvl5l56upkLyc7Obt+5ERJLeSVr6MRS3ljKCrGV\nt6NZmyss4ehzWAncDuA2HeVqrUsB3A//w0qpke6xUwGfe85CpZTH7ZxOwykkQgghwiDkxUFrvQnY\nppTahHOn0lKl1BKl1C3uIfcDy9z9xcDftdangJeAzcCbwP/TWtuhziqEEMIRlj4HrfUDDTbtCtp3\nEOeW1YbnPA08HeJoQgghmiAjpIUQQjQixUEIIUQjUhyEEEI0IsVBCCFEI1IchBBCNGIZ0+Kg41hy\n2fwgQggRZlbDDZGYPiNUGv1wQggh2kealYQQQjQixUEIIUQjUhyEEEI0IsVBCCFEI1IchBBCNCLF\nQQghRCOX062s7aKUehyYhTNO4j6t9dYIR2qWUurHOGtuxwOPaq1fiXCkFimlkoGPge9qrZ+NcJwW\nKaU+C3wdqAX+V2v9eoQjNUsplQb8EWcJ3STgEa3125FN1ZhSajzwN+BxrfVTSqlBwJ+AOJwFv/5F\na10dyYx1msm6DEgA/MDntNZnIpkxWMO8Qds/Cbylte7wrf1d+spBKTUPGKm1zgHuxllvIioppRYA\n492s1wI/j3Cktvgf4FykQ7TGXVDq2zhTx98I3BTZRK1aAvi01gtwFtL6RWTjNKaUSgWeBILXfv8O\n8Eut9VzgIPBvkcjWUDNZvwf8Vms9D1gOfDUS2ZrSTF6UUt2Ab3Jhpc0O6dLFAVgEvAqgtd4LZCil\n0iMbqVnvAHe4j4uAVKVUXATztEgpNRoYC0TtX+BBFgOrtNalWuvTWusvRDpQK87irLEOztVDNK6S\nWA1cj7NMcJ35wAr38d9xfu/RoKmsXwZedh8XcOH3HQ2aygvwIPBLoKYz3qSrF4dMnH/4OgVcvN51\n1NBaB7TW5e7Tu4E3tNaBSGZqxU+Jor+2WjEUSFFKrVBKbVBKLYp0oJZorV8ABiulDuL80fC1CEdq\nRGtdq7WubLA5NagZKR/ICnOsJjWVVWtdrrUOuH+ALQWej0y6xprKq5QaBUzSWv+1s96nqxeHhqJ+\nCg6l1E04xeHeSGdpjlLq88B7Wusjkc7SRhbOX4a34jTZLFNKRe1/C0qpzwHHtdafABYCT7VySjSK\n2t9vHbcw/AlYo7Ve3drxEfY4nfzHWFcvDrlcfKWQTSe114WC29n0EHCd1ro40nlacANwk1JqM3AP\n8C2lVLQ0ITQlD9jk/kV2CCgF+kY4U0tmA28DaK13AdnR3MQYpMy9SQFgAI2bRaLNMuCA1vqRSAdp\niVJqADAa+LP7/1yWUmp9R1+3q9+ttBJ4BHhaKTUFyNVal0Y4U5OUUj2Ax4DFWuuo7uTVWn+67rFS\n6mHgqNZ6VeQStWol8KxS6kc4bfhpRGc7fp2DwEzgZaXUEKAsypsY66wCbgOec7+/Fdk4zXPvXqvR\nWn870llao7U+BYyoe66UOup2pHfI5TRld7sopX4IXAXYwFL3L7Goo5T6AvAwsD9o8+e11scjk6ht\ngorDsxGO0iKl1BdxmusAvqe1XtHS8ZHk3sr6e6A/zh9439Jar4lsqosppabi9DsNxbkV9BTwWeBZ\noBtwDLhLa+2PUMR6zWTtB1QBJe5he7TWX45IwAaayXtr3R+NbnEY2tH36fLFQQghRGNdvc9BCCFE\nE6Q4CCGEaESKgxBCiEakOAghhGhEioMQQohGpDgIEQWUUvPd6TCEiApSHIQQQjQi4xyEaAN3Tqvv\nAak4I5T/GfgJcB6YDIwCtgF3aq0rlFITgV/jzNlUBXyjbs0FpdQ3gC/irB3xGvBfwDzgdzhTNnwO\nSATu0VqvDdfPKEQwuXIQohVKqeE4E7B9Rms9HFgL/MbdfQvOmgqDgB7AvyulPMALwFNa69E480v9\nRSnVXSk1x30+CRiPs4bE7e5rDQQ+0lqPwSks/xOOn0+IpkhxEKJ11wLrtNYfu89/A/wTzopmf9Na\nF2qtbZy1Qa4EhuFM6PgCgNb6A5zpIqbjzMP/urt2RA3OGgd1K/qVBE3bsQOnWAgREV194j0h2qIn\ncJVSal/QtmKcJqPgTuTzOBP39QWKtNamwb5+QB+CZiPVWlcAKKXgwjw+AAGc4iNEREhxEKJ1uTgr\nxd0evFEp9SzOh32dXjjLouYBvZRSVlCB6O1uPxt8jrtEqRBRR4qDEK17G/iRUmq41vqwUmoGzgyj\nANcqpXrirAFxM/AicBQ4CXwaeEEpdSVOM9MWoBLQSqlvuee8itO/EO1rG4guRvochGiF1vo08O/A\ncqXUXpyV1150d6/G6TM4idN09Hv3auFO4F73+CeAO9ylJzfjrMuxE9gDbAf+Es6fR4i2kFtZhWgn\nt1npoNb6e5HOIkRnkysHIYQQjUhxEEII0Yg0KwkhhGhErhyEEEI0IsVBCCFEI1IchBBCNCLFQQgh\nRCNSHIQQQjTy/wF6gOr6ypmU0gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e90f4bdd8>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(historyBilstm.history['acc'])\n",
    "plt.plot(historyBilstm.history['val_acc'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 38
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12680,
     "status": "ok",
     "timestamp": 1521567144667,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "e3jkV40Gdz-2",
    "outputId": "723242be-b73d-4125-a5f0-f7692917a343"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4878/4878 [==============================] - 12s 2ms/step\n"
     ]
    }
   ],
   "source": [
    "scoreBiLSTM = modelBiLstm.evaluate(np.array(X_val),y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 586,
     "status": "ok",
     "timestamp": 1521567146212,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "ZhjREsFKd_tP",
    "outputId": "3b08a727-d6d0-4a8c-af7d-a7c2e34564bc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.4781140277453161, 0.7660926609266092]"
      ]
     },
     "execution_count": 58,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scoreBiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 51,
     "output_extras": [
      {
       "item_id": 59
      },
      {
       "item_id": 60
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 15143,
     "status": "ok",
     "timestamp": 1521567185072,
     "user": {
      "displayName": "Ala Bayoudh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "112506902794522806974"
     },
     "user_tz": -120
    },
    "id": "bLztrfyheDqB",
    "outputId": "9a85ac2c-5d8a-408f-caf4-b6c4aed99153"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5738/5738 [==============================] - 15s 3ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.4881193633277311, 0.7689090275565021]"
      ]
     },
     "execution_count": 59,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modelBiLstm.evaluate(np.array(X_test),y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "-I7iGtJQYGcW"
   },
   "outputs": [],
   "source": [
    "modelBiLstm.save('BILSTM_Model.h5')  # creates a HDF5 file 'my_model.h5'\n",
    "del modelBiLstm  # deletes the existing model\n",
    "\n",
    "# returns a compiled model\n",
    "# identical to the previous one\n",
    "modelBiLstm = load_model('BILSTM_Model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "0a1uByd9dy96"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.download('BILSTM_Model.h5')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "SentimentAnalysisClassificationWithLSTM.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}