LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 603,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading utils module\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import concatenate, Activation, GlobalAveragePooling1D, GlobalMaxPooling1D, Layer, Dense, Embedding, LSTM, GRU, Dropout, SpatialDropout1D, Input, Average, Bidirectional, BatchNormalization\n",
    "from keras.callbacks import Callback\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "\n",
    "import sys, importlib\n",
    "importlib.reload(sys.modules['utils'])\n",
    "import utils\n",
    "\n",
    "from keras.models import load_model\n",
    "import json, argparse, os\n",
    "import re\n",
    "import io\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Don't hog GPU\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)\n",
    "keras.backend.set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to training and testing data file. This data can be downloaded from a link, details of which will be provided.\n",
    "trainDataPath = \"./train.txt\"\n",
    "testDataPath = \"./dev.txt\"\n",
    "evalDataPath = \"./evaluate.txt\"\n",
    "# Output file that will be generated. This file can be directly submitted.\n",
    "solutionPath = \"./test.txt\"\n",
    "\n",
    "label2emotion = {0:\"others\", 1:\"happy\", 2: \"sad\", 3:\"angry\"}\n",
    "emotion2label = {\"others\":0, \"happy\":1, \"sad\":2, \"angry\":3}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_CLASSES = 4                 # Number of classes - Happy, Sad, Angry, Others\n",
    "MAX_NB_WORDS = 15000                # To set the upper limit on the number of tokens extracted using keras.preprocessing.text.Tokenizer \n",
    "MAX_SEQUENCE_LENGTH = 35         # All sentences having lesser number of words than this will be padded\n",
    "EMBEDDING_DIM = 300               # The dimension of the word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessData(dataFilePath, mode):\n",
    "    \"\"\"Load data from a file, process and return indices, conversations and labels in separate lists\n",
    "    Input:\n",
    "        dataFilePath : Path to train/test file to be processed\n",
    "        mode : \"train\" mode returns labels. \"test\" mode doesn't return labels.\n",
    "    Output:\n",
    "        indices : Unique conversation ID list\n",
    "        conversations : List of 3 turn conversations, processed and each turn separated by the <eos> tag\n",
    "        raw_conversations : All conversations together\n",
    "        labels : [Only available in \"train\" mode] List of labels\n",
    "    \"\"\"\n",
    "    indices = []\n",
    "    conversations = []\n",
    "    raw_conversations = []\n",
    "    labels = []\n",
    "    \n",
    "    importlib.reload(sys.modules['regex'])\n",
    "    import regex\n",
    "    \n",
    "    with io.open(dataFilePath, encoding=\"utf8\") as finput:\n",
    "        finput.readline()\n",
    "        for line in finput:\n",
    "            # Convert multiple instances of . ? ! , to single instance\n",
    "            # okay...sure -> okay . sure\n",
    "            # okay???sure -> okay ? sure\n",
    "            # Add whitespace around such punctuation\n",
    "            # okay!sure -> okay ! sure\n",
    "            raw_conv = ' '.join(line[:].strip().split('\\t')[1:4])\n",
    "            repeatedChars = ['.', '?', '!', ',']\n",
    "            for c in repeatedChars:\n",
    "                lineSplit = line.split(c)\n",
    "                while True:\n",
    "                    try:\n",
    "                        lineSplit.remove('')\n",
    "                    except:\n",
    "                        break\n",
    "                cSpace = ' ' + c + ' '    \n",
    "                line = cSpace.join(lineSplit)\n",
    "            \n",
    "            line = line.strip().split('\\t')\n",
    "            if mode == \"train\":\n",
    "                # Train data contains id, 3 turns and label\n",
    "                label = emotion2label[line[4]]\n",
    "                labels.append(label)\n",
    "            \n",
    "            conv = ' <eos> '.join(line[1:4])\n",
    "            \n",
    "            # Remove any duplicate spaces\n",
    "            duplicateSpacePattern = re.compile(r'\\ +')\n",
    "            conv = re.sub(duplicateSpacePattern, ' ', conv)\n",
    "            \n",
    "            indices.append(int(line[0]))\n",
    "            # Remove stray punctuation\n",
    "            stray_punct = ['‑', '-', \"^\", \":\",\n",
    "                           \";\", \"#\", \")\", \"(\", \"*\", \"=\", \"\\\\\", \"/\"]\n",
    "            for punct in stray_punct:\n",
    "                    conv = conv.replace(punct, \"\")\n",
    "    \n",
    "            processedData = regex.cleanText(conv.lower()).lower() #.rstrip()\n",
    "            processedData = processedData.replace(\"'\", \"\")\n",
    "            # Remove numbers\n",
    "            processedData = ''.join([i for i in processedData if not i.isdigit()])\n",
    "\n",
    "            conversations.append(processedData)\n",
    "            raw_conversations.append(raw_conv)\n",
    "    \n",
    "    if mode == \"train\":\n",
    "        return indices, conversations, raw_conversations, labels\n",
    "    else:\n",
    "        return indices, conversations, raw_conversations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing training data...\n",
      "Loading utils module\n",
      "Processing test data...\n",
      "Loading utils module\n",
      "Processing evaluation data...\n",
      "Loading utils module\n",
      "Extracting tokens...\n",
      "Found 15752 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "print(\"Processing training data...\")\n",
    "trainIndices, trainTexts, rawtrainTexts, labels = preprocessData(trainDataPath, mode=\"train\")\n",
    "labels = to_categorical(np.asarray(labels), NUM_CLASSES)\n",
    "print(\"Processing test data...\")\n",
    "testIndices, testTexts, rawtestTexts, testLabels = preprocessData(testDataPath, mode=\"train\")\n",
    "testLabels = to_categorical(np.asarray(testLabels), NUM_CLASSES)\n",
    "print(\"Processing evaluation data...\")\n",
    "evalIndices, evalTexts, rawevalTexts = preprocessData(evalDataPath, mode=\"test\")\n",
    "\n",
    "print(\"Extracting tokens...\")\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "# tokenizer.fit_on_texts(trainTexts)\n",
    "tokenizer.fit_on_texts(trainTexts + testTexts + evalTexts)\n",
    "trainSequences = tokenizer.texts_to_sequences(trainTexts)\n",
    "testSequences = tokenizer.texts_to_sequences(testTexts)\n",
    "evalSequences = tokenizer.texts_to_sequences(evalTexts)\n",
    "\n",
    "wordIndex = tokenizer.word_index\n",
    "print(\"Found %s unique tokens.\" % len(wordIndex))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train-data Coverage (cutoff length): 0.9904509283819629\n",
      "Test-data Coverage (cutoff length): 0.992377495462795\n",
      "Eval-data Coverage (cutoff length): 0.9920130695225994\n"
     ]
    }
   ],
   "source": [
    "lens = [len(x) for x in trainSequences]\n",
    "print(\"Train-data Coverage (cutoff length):\", np.sum(np.array(lens) <= MAX_SEQUENCE_LENGTH) / len(trainSequences))\n",
    "lens = [len(x) for x in testSequences]\n",
    "print(\"Test-data Coverage (cutoff length):\", np.sum(np.array(lens) <= MAX_SEQUENCE_LENGTH) / len(testSequences))\n",
    "lens = [len(x) for x in evalSequences]\n",
    "print(\"Eval-data Coverage (cutoff length):\", np.sum(np.array(lens) <= MAX_SEQUENCE_LENGTH) / len(evalSequences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage with 15000 words: 0.9987336272485694\n"
     ]
    }
   ],
   "source": [
    "sorted_wordcounts = sorted(tokenizer.word_counts.items(), key=lambda x: x[1], reverse=True)\n",
    "words_covered, total_words = 0, 0\n",
    "for i, tup in enumerate(sorted_wordcounts):\n",
    "    total_words += tup[1]\n",
    "    if i < MAX_NB_WORDS:\n",
    "        words_covered += tup[1]\n",
    "print(\"Coverage with %d words:\" % MAX_NB_WORDS, words_covered/total_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_wordcounts = sorted(tokenizer.word_counts.items(), key=lambda x: x[1], reverse=True)\n",
    "with open('./need_sswe.txt', 'w') as f:\n",
    "    for i, tup in enumerate(sorted_wordcounts):\n",
    "        f.write(tup[0] + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_l, train_m, train_r = utils.split_into_three(trainTexts, tokenizer)\n",
    "test_l, test_m, test_r = utils.split_into_three(testTexts, tokenizer)\n",
    "eval_l, eval_m, eval_r = utils.split_into_three(evalTexts, tokenizer)\n",
    "\n",
    "train_all = tokenizer.texts_to_sequences(trainTexts)\n",
    "test_all = tokenizer.texts_to_sequences(testTexts)\n",
    "eval_all = tokenizer.texts_to_sequences(evalTexts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wont you ask my age  <eos> hey at least i age well  <eos> can you tell me how can we get closer  \n",
      "Wont u ask my age?? hey at least I age well! Can u tell me how can we get closer??\n",
      "\n",
      "i said yes <eos> what if i told you iam not  <eos> go to hell\n",
      "I said yes What if I told you I'm not? Go to hell\n",
      "\n",
      "where i ll check <eos> why tomorrow  <eos> no i want now\n",
      "Where I ll check why tomorrow? No I want now\n",
      "\n",
      "shall we meet <eos> you say you are leaving soon  anywhere you wanna go before you head  <eos>  \n",
      "Shall we meet you say- you're leaving soon...anywhere you wanna go before you head? ?\n",
      "\n",
      "let us change the subject <eos> i just did it  l  <eos> you are broken\n",
      "Let's change the subject I just did it .l. You're broken\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(5, 10):\n",
    "    print(testTexts[i])\n",
    "    print(rawtestTexts[i])\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating embedding matrix...\n",
      "Found 400000 word vectors.\n",
      "Found embedding for 76.16175723717623 % embeddings\n"
     ]
    }
   ],
   "source": [
    "print(\"Populating embedding matrix...\")\n",
    "embeddingMatrix, oov = utils.getEmbeddingMatrix(wordIndex, EMBEDDING_DIM)\n",
    "oov = [(x, tokenizer.word_counts.get(x, 0)) for x in oov]\n",
    "oov.sort(key=lambda tup: tup[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading utils module\n",
      "Populating SSWE embedding matrix...\n",
      "Found unreadable 144 word vectors\n",
      "Found 15936 word vectors.\n",
      "Found embedding for 98.99060436769933 % embeddings\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(sys.modules['utils'])\n",
    "import utils\n",
    "\n",
    "print(\"Populating SSWE embedding matrix...\")\n",
    "embeddingMatrix_sswe, oov_sswe = utils.get_sswe_embeddings(wordIndex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 498,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['≠', 'afterdays', '…it', 'more…lasagna', 'family…', 'görlitz', 'rşch', 'ilm', 'whére', 'ω', 'girlfriend\\u200b', 'iş', 'wai', '⊄', 'referrel', 'iscoming', 'me…', '¡so', '¿did', 'aadu', 'face…', 'n', 'xx', 'nan', 'libspill', 'serious…just', 'mbile', 'bred', 'ounces', 'r', '••', 'yyi', 'song¿', 'elhamdülillah', '\\u200d\\u200d', 'okay\\u200b', 'what¡', 'well…', 'one—', 'i̇', 'àapka', 'kha\\u200d\\u200d', 'ı', 'friendsqa', '–', 'vho', 'not\\u200b', 'kķkkkk', 'vmyhorbs', 'driver', 'it¿', 'ĺove', '—', 'cross\\u200b', 'still…', '¶', 'party¿', 'sad…', 'will…', 'cbillion', '͡°', 'alented', 'snt', 'itӳ', 'false', 'któw', 'morrow', 'i…', '͜ʖ', '×', 'wuakd', '฿', 'yaaaaà', 'what¿¿', 'ixm', 'is…', '\\u200d\\u200d\\u200d\\u200d', 'netflixs\\u200b', 'doğng', 'mnts', 'seelfi', 'sone', 'cliché', 'same…', 'or…', 'beşiktaş', 'vsit', 'gudni', 'yr', 'want…', 'north\\u200b', 'rknow', '¿', 'su', 'kiu', 'y', '∑oo', 'çuz', 'fi̇nd', 'gr']\n",
      "159\n"
     ]
    }
   ],
   "source": [
    "print(oov_sswe[:100])\n",
    "print(len(oov_sswe))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('emoji', 39), ('i̇', 18), ('emojis', 12), ('friendzoned', 11), ('friendzone', 10), ('tajmahal', 9), ('hmmzoning', 9), ('oho', 9), ('donapost', 8), ('lolz', 8), ('rted', 7), ('bangaram', 7), ('everyones', 7), ('ehh', 7), ('gng', 7), ('everythings', 6), ('useropenreflink', 6), ('nonveg', 6), ('whatsaap', 6), ('iand', 5), ('thatll', 5), ('ftfy', 5), ('dafaq', 5), ('facepalm', 5), ('whatt', 5), ('oww', 5), ('ddlj', 5), ('selfies', 5), ('freecharge', 5), ('chatbots', 5), ('\\u200d', 5), ('emojisong', 4), ('himher', 4), ('begar', 4), ('happies', 4), ('thnk', 4), ('wiil', 4), ('lololol', 4), ('playcreepypedia', 4), ('flipkart', 4), ('temme', 4), ('bdw', 4), ('youve', 4), ('brokeup', 4), ('oclock', 4), ('lolzzz', 4), ('whyre', 4), ('habbit', 4), ('iaposm', 4), ('whts', 4), ('xams', 4), ('medam', 4), ('arjit', 4), ('bhindi', 3), ('lve', 3), ('dobt', 3), ('theyll', 3), ('padmavat', 3), ('hihihi', 3), ('scuffletown', 3), ('katachi', 3), ('tysm', 3), ('baaghi', 3), ('okz', 3), ('brozoned', 3), ('hinglish', 3), ('fucken', 3), ('skys', 3), ('watsaap', 3), ('goid', 3), ('ranbeer', 3), ('tshirts', 3), ('hyd', 3), ('typelaugh', 3), ('maam', 3), ('wheather', 3), ('ohhhooo', 3), ('nothng', 3), ('thatd', 3), ('tonights', 3), ('halfgirlfriend', 3), ('werent', 3), ('huhhhh', 3), ('ssup', 3), ('wch', 3), ('anyones', 3), ('okiee', 3), ('howd', 3), ('besties', 3), ('sned', 3), ('offf', 3), ('youself', 3), ('itll', 3), ('selfi', 3), ('frind', 3), ('uve', 3), ('awwh', 3), ('lols', 3), ('nightcore', 3), ('shits', 3)]\n",
      "3755\n"
     ]
    }
   ],
   "source": [
    "print(oov[:100])\n",
    "print(len(oov))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {},
   "outputs": [],
   "source": [
    "ooh = {}\n",
    "\n",
    "relevant_oov = [x[0] for x in oov]\n",
    "actual_words = [k for (k,v) in tokenizer.word_index.items()]\n",
    "for datum in testTexts:\n",
    "    words = datum.split(' ')\n",
    "    for word in words:\n",
    "        if word in relevant_oov :\n",
    "            ooh[word] = ooh.get(word, 0) + 1\n",
    "\n",
    "ooh_items = [(k, v) for (k,v) in ooh.items()]\n",
    "ooh_items.sort(key=lambda tup: tup[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('donapost', 6), ('iaposm', 3), ('friendzoned', 3), ('emojis', 3), ('nightcore', 3), ('frnz', 2), ('wowe', 2), ('whataposs', 2), ('ahow', 2), ('bachhe', 2), ('tajmahal', 2), ('playcreepypedia', 2), ('emojisong', 2), ('oclock', 2), ('lolz', 2), ('emoji', 2), ('kewlwst', 2), ('baaghi', 2), ('lamakaan', 2), ('swiggy', 1), ('dunp', 1), ('howaposs', 1), ('hmmzoning', 1), ('oopsy', 1), ('wokayyy', 1), ('vakola', 1), ('lolzzz', 1), ('tindering', 1), ('dhyat', 1), ('besties', 1), ('ocourse', 1), ('whoaposs', 1), ('tnq', 1), ('sharukh', 1), ('yaantey', 1), ('juxt', 1), ('thks', 1), ('milovie', 1), ('xwhat', 1), ('muhhhh', 1), ('higherres', 1), ('megalol', 1), ('areee', 1), ('pshhhh', 1), ('aahh', 1), ('powerliftingsmh', 1), ('tranwendy', 1), ('ivve', 1), ('one—', 1), ('temme', 1), ('saxy', 1), ('fucken', 1), ('googlin', 1), ('dped', 1), ('aaare', 1), ('introducted', 1), ('sircasm', 1), ('favoriet', 1), ('scense', 1), ('gafe', 1), ('gulthfriend', 1), ('whatsaap', 1), ('fidha', 1), ('berozgaari', 1), ('pizzaaaass', 1), ('onely', 1), ('sabji', 1), ('pulka', 1), ('littl', 1), ('frenching', 1), ('kisse', 1), ('caming', 1), ('pokiri', 1), ('interperted', 1), ('soported', 1), ('whate', 1), ('yediiiii', 1), ('unaru', 1), ('nect', 1), ('sharaddha', 1), ('lovery', 1), ('strtd', 1), ('bthing', 1), ('ahve', 1), ('sagarwcam', 1), ('vandetta', 1), ('aiyyo', 1), ('wtsup', 1), ('deepikas', 1), ('friendzoning', 1), ('agian', 1), ('dubstub', 1), ('everyones', 1), ('mondegar', 1), ('cortanan', 1), ('tattos', 1), ('zonbie', 1), ('friendzone', 1), ('everyword', 1), ('ledis', 1), ('hahahahaja', 1), ('botking', 1), ('spritiual', 1), ('cuntry', 1), ('dcac', 1), ('happene', 1), ('mny', 1), ('iaposll', 1), ('ugghh', 1), ('krbo', 1), ('rolins', 1), ('charg', 1), ('voicemsg', 1), ('tihri', 1), ('pohot', 1), ('srimathi', 1), ('cryp', 1), ('karunga', 1), ('⃣⃣⃣', 1), ('maam', 1), ('ahaan', 1), ('boujee', 1), ('thataposs', 1), ('plox', 1), ('whatt', 1), ('babyd', 1), ('seld', 1), ('ssup', 1), ('sposed', 1), ('listion', 1), ('ehy', 1), ('siliconbased', 1), ('ruuch', 1), ('youself', 1), ('tumhe', 1), ('≠', 1), ('sovashvssjxj', 1), ('eakada', 1), ('achcha', 1), ('youve', 1), ('ohkaayyy', 1), ('ghoonghat', 1), ('shld', 1), ('spenkig', 1), ('hurman', 1), ('hadnt', 1), ('agyooo', 1), ('cutw', 1), ('yuo', 1), ('bhosdi', 1), ('contect', 1), ('flep', 1), ('ingish', 1), ('whattay', 1), ('zarvise', 1), ('alri', 1), ('afgalgunj', 1), ('referrel', 1), ('skys', 1), ('bdw', 1), ('baggi', 1), ('smouch', 1), ('tatto', 1), ('gve', 1), ('vaoov', 1), ('eminems', 1), ('nabr', 1), ('dout', 1), ('vegita', 1), ('aayojan', 1), ('yyi', 1), ('ishaqbaaz', 1), ('publicuty', 1), ('brozone', 1), ('sunnyleone', 1), ('atrect', 1), ('dued', 1), ('gustion', 1), ('i̇', 1), ('programmin', 1), ('daynight', 1), ('behenchood', 1), ('becauce', 1), ('duckface', 1), ('fuckbuddy', 1), ('lololol', 1), ('hounomkee', 1), ('almigthy', 1), ('sillly', 1), ('opss', 1), ('lovevu', 1), ('answerer', 1), ('whhat', 1), ('whatevr', 1), ('wduwtta', 1), ('edsheerans', 1), ('ironmaiden', 1), ('panipiri', 1), ('mhy', 1), ('nsmer', 1), ('woohooo', 1), ('dafaq', 1), ('rockzz', 1), ('joim', 1), ('ookh', 1), ('lovear', 1), ('kudaellu', 1), ('vettai', 1), ('wuv', 1), ('pleae', 1), ('childo', 1), ('flipkart', 1), ('bakwaas', 1), ('swewty', 1), ('thatd', 1), ('mrunal', 1), ('egyup', 1), ('thatll', 1), ('plutos', 1), ('iscoming', 1), ('wnaa', 1), ('uve', 1), ('underdiving', 1), ('xams', 1), ('adsa', 1), ('partywonderful', 1), ('bhaskars', 1), ('cross\\u200b', 1), ('pming', 1), ('scuffletown', 1), ('bhakkk', 1), ('gujurati', 1), ('tysm', 1), ('àapka', 1), ('fuking', 1), ('mams', 1), ('begar', 1)]\n",
      "237\n"
     ]
    }
   ],
   "source": [
    "print(ooh_items)\n",
    "print(len(ooh_items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "metadata": {},
   "outputs": [],
   "source": [
    "ooh = {}\n",
    "\n",
    "relevant_oov = [x[0] for x in oov]\n",
    "actual_words = [k for (k,v) in tokenizer.word_index.items()]\n",
    "for datum in evalTexts:\n",
    "    words = datum.split(' ')\n",
    "    for word in words:\n",
    "        if word in relevant_oov:\n",
    "            ooh[word] = ooh.get(word, 0) + 1\n",
    "\n",
    "ooh_items = [(k, v) for (k,v) in ooh.items()]\n",
    "ooh_items.sort(key=lambda tup: tup[1], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('emoji', 5), ('useropenreflink', 3), ('allout', 2), ('everythings', 2), ('friendzoned', 2), ('donapost', 2), ('okiee', 2), ('briyani', 2), ('chatbots', 2), ('halfgirlfriend', 2), ('besties', 2), ('shole', 2), ('iand', 2), ('lolz', 2), ('ftfy', 2), ('panipuri', 2), ('lohiya', 2), ('wachapp', 2), ('choclati', 2), ('i̇', 2), ('hihihi', 2), ('sachhi', 2), ('nigt', 1), ('duuuuude', 1), ('sherikkum', 1), ('whatsa', 1), ('oclock', 1), ('cutti', 1), ('chillll', 1), ('sherlyns', 1), ('friendzone', 1), ('bzy', 1), ('bainchod', 1), ('tgnxmh', 1), ('twilightt', 1), ('shype', 1), ('ihddit', 1), ('shold', 1), ('mattee', 1), ('bwaha', 1), ('kilpauk', 1), ('langwege', 1), ('dhonis', 1), ('ruwho', 1), ('matvh', 1), ('delect', 1), ('directot', 1), ('voicesg', 1), ('blowjob', 1), ('thnk', 1), ('thenn', 1), ('spiling', 1), ('trappp', 1), ('messagee', 1), ('besharam', 1), ('namber', 1), ('llallu', 1), ('aswu', 1), ('toask', 1), ('conformd', 1), ('adhuri', 1), ('heartfull', 1), ('hutiya', 1), ('huhhhh', 1), ('quotfemalequot', 1), ('messanger', 1), ('joffery', 1), ('knowp', 1), ('perioda', 1), ('uask', 1), ('defghi', 1), ('bhenkelode', 1), ('yrah', 1), ('titanfall', 1), ('uhhhmm', 1), ('biddable', 1), ('blockchain', 1), ('iaposm', 1), ('hahaing', 1), ('niceyekyahai', 1), ('sharadha', 1), ('fuuuuuuukkkhhhhh', 1), ('bluddy', 1), ('jantarmantar', 1), ('noshad', 1), ('fuddi', 1), ('arobot', 1), ('kixs', 1), ('wifes', 1), ('jigarthanda', 1), ('okhy', 1), ('aupport', 1), ('virtualasstftw', 1), ('countery', 1), ('fuvvked', 1), ('maledive', 1), ('pffft', 1), ('aritifical', 1), ('simble', 1), ('poguthey', 1)]\n",
      "617\n"
     ]
    }
   ],
   "source": [
    "print(ooh_items[:100])\n",
    "print(len(ooh_items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 587,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of training data tensor:  (30160, 35)\n",
      "Shape of training label tensor:  (30160, 4)\n"
     ]
    }
   ],
   "source": [
    "train_l = pad_sequences(train_l, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "train_m = pad_sequences(train_m, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "train_r = pad_sequences(train_r, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "train_all = pad_sequences(train_all, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "\n",
    "test_l = pad_sequences(test_l, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "test_m = pad_sequences(test_m, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "test_r = pad_sequences(test_r, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "test_all = pad_sequences(test_all, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "\n",
    "eval_l = pad_sequences(eval_l, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "eval_m = pad_sequences(eval_m, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "eval_r = pad_sequences(eval_r, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "eval_all = pad_sequences(eval_all, maxlen=MAX_SEQUENCE_LENGTH, padding='post', truncating='post')\n",
    "\n",
    "print(\"Shape of training data tensor: \", train_l.shape)\n",
    "print(\"Shape of training label tensor: \", labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading utils module\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import average\n",
    "from keras.regularizers import l1, l2, l1_l2\n",
    "from keras.layers import RepeatVector\n",
    "import keras.backend as K\n",
    "\n",
    "importlib.reload(sys.modules['utils'])\n",
    "import utils\n",
    "\n",
    "def buildSingleModel(embeddingMatrix, hidDim=128, dropout=0.2, first_type='lstm', second_type='gru', multitask=False):\n",
    "    embeddingLayer = Embedding(embeddingMatrix.shape[0],\n",
    "                                    embeddingMatrix.shape[1],\n",
    "                                    weights=[embeddingMatrix],\n",
    "                                    input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                    trainable=False)\n",
    "    \n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "    x = embeddingLayer(inp)\n",
    "    x = SpatialDropout1D(0.2)(x)\n",
    "    if first_type == 'lstm':\n",
    "        x = Bidirectional(LSTM(hidDim, return_sequences=True,\n",
    "                           dropout=0.2, recurrent_dropout=0.2,\n",
    "                           bias_regularizer=l1_l2(0.01, 0.01),\n",
    "                           recurrent_regularizer=l1_l2(0.01, 0.01),\n",
    "                          ))(x)\n",
    "    else:\n",
    "        x = Bidirectional(GRU(hidDim, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(x)\n",
    "    if second_type == 'gru':\n",
    "        y = Bidirectional(GRU(hidDim, return_sequences=True, dropout=0.2, recurrent_dropout=0.2))(x)\n",
    "    else:\n",
    "        y = Bidirectional(LSTM(hidDim, return_sequences=True,\n",
    "                           dropout=0.2, recurrent_dropout=0.2,\n",
    "                           bias_regularizer=l1_l2(0.01, 0.01),\n",
    "                           recurrent_regularizer=l1_l2(0.01, 0.01),\n",
    "                          ))(x)\n",
    "\n",
    "    atten_1 = utils.Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "    atten_2 = utils.Attention(MAX_SEQUENCE_LENGTH)(y)\n",
    "    avg_pool_1 = GlobalAveragePooling1D()(x)\n",
    "    max_pool_1 = GlobalMaxPooling1D()(x)\n",
    "    avg_pool_2 = GlobalAveragePooling1D()(y)\n",
    "    max_pool_2 = GlobalMaxPooling1D()(y)\n",
    "    \n",
    "    feature_comb_1 = concatenate([atten_1, avg_pool_1, max_pool_1])\n",
    "    feature_comb_1 = Dropout(dropout)(feature_comb_1)\n",
    "    conc_1 = Dense(hidDim, activation=\"relu\")(feature_comb_1)\n",
    "\n",
    "    feature_comb_2 = concatenate([atten_2, avg_pool_2, max_pool_2])\n",
    "    feature_comb_2 = Dropout(dropout)(feature_comb_2)\n",
    "    conc_2 = Dense(hidDim, activation=\"relu\")(feature_comb_2)\n",
    "\n",
    "    conc = concatenate([conc_1, conc_2])\n",
    "    conc = Dropout(dropout)(conc)\n",
    "    conc = Dense(hidDim, activation=\"relu\")(conc)\n",
    "    conc = Dropout(dropout)(conc)\n",
    "    output = Dense(NUM_CLASSES, activation='softmax')(conc)\n",
    "    \n",
    "    if multitask:\n",
    "        text_encoding = RepeatVector(MAX_SEQUENCE_LENGTH)(conc)\n",
    "        decoded = LSTM(hidDim, return_sequences=True)(text_encoding)\n",
    "        decoded = LSTM(hidDim, return_sequences=True)(decoded)\n",
    "        decoded_words = Dense(embeddingMatrix.shape[0], activation='softmax')(decoded)\n",
    "        \n",
    "        def multitask_loss(y_true, y_pred):\n",
    "            cross_entropy_loss = K.categorical_crossentropy(y_true[1], y_pred[1])\n",
    "            autoencoder_loss = K.mean(K.square(y_true[0] - y_pred[0]))\n",
    "            combined_loss = cross_entropy_loss + autoencoder_loss\n",
    "            return combined_loss\n",
    "        \n",
    "        auto_model = Model(inputs=inp, outputs=decoded_words)\n",
    "        main_model = Model(inputs=inp, outputs=output)\n",
    "        auto_model.compile(loss='mse',\n",
    "                  optimizer=optimizers.Adam(1e-3),\n",
    "                  metrics=['accuracy'])\n",
    "        main_model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizers.Adam(1e-3),\n",
    "                  metrics=[utils.microF1Loss])\n",
    "        return auto_model, main_model\n",
    "    else:\n",
    "        model = Model(inputs=inp, outputs=output)\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizers.Adam(1e-3),\n",
    "                  metrics=[utils.microF1Loss])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 571,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import average\n",
    "from keras.regularizers import l1, l2, l1_l2\n",
    "\n",
    "def buildSingleDEEPModel(embeddingMatrix, hidDim=128, learnEmbs=False, dropout=0.2):\n",
    "    if learnEmbs:\n",
    "        embeddingLayer = Embedding(embeddingMatrix.shape[0],\n",
    "                                    embeddingMatrix.shape[1],\n",
    "                                    input_length=MAX_SEQUENCE_LENGTH)\n",
    "    else:\n",
    "        embeddingLayer = Embedding(embeddingMatrix.shape[0],\n",
    "                                    embeddingMatrix.shape[1],\n",
    "                                    weights=[embeddingMatrix],\n",
    "                                    input_length=MAX_SEQUENCE_LENGTH,\n",
    "                                    trainable=False)\n",
    "    \n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "    x = embeddingLayer(inp)\n",
    "    x = SpatialDropout1D(0.1)(x)\n",
    "    \n",
    "#     x = Bidirectional(LSTM(hidDim, return_sequences=True,\n",
    "#                            dropout=0.1, recurrent_dropout=0.1,\n",
    "#                            bias_regularizer=l1(0.01),\n",
    "#                            recurrent_regularizer=l2(0.01),\n",
    "#                           ))(x)\n",
    "#     x = Bidirectional(LSTM(hidDim, return_sequences=True,\n",
    "#                            dropout=0.1, recurrent_dropout=0.1,\n",
    "#                            bias_regularizer=l1(0.01),\n",
    "#                            recurrent_regularizer=l2(0.01),\n",
    "#                           ))(x)\n",
    "    \n",
    "    x = LSTM(hidDim, return_sequences=False,\n",
    "                           dropout=0.1, recurrent_dropout=0.1,\n",
    "                           bias_regularizer=l1_l2(0.01, 0.01),\n",
    "                           recurrent_regularizer=l1_l2(0.01, 0.01),\n",
    "                          )(x)\n",
    "#     x = Bidirectional(LSTM(hidDim, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n",
    "#     x = Bidirectional(LSTM(hidDim, return_sequences=False, dropout=0.1, recurrent_dropout=0.1))(x)    \n",
    "#     x = utils.Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "    \n",
    "    conc = Dropout(dropout)(x)\n",
    "    conc = Dense(hidDim, activation=\"relu\")(conc)\n",
    "    conc = Dropout(dropout)(conc)\n",
    "    output = Dense(NUM_CLASSES, activation='softmax')(conc)\n",
    "    model = Model(inputs=inp, outputs=output)\n",
    "    \n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=optimizers.Adam(1e-3),\n",
    "#                   optimizer=optimizers.Adam(1e-4),\n",
    "#                   optimizer=optimizers.SGD(5e-3),\n",
    "                  metrics=[utils.microF1Loss])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 674,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 0.09015739790282193, 1: 0.35540724746822805, 2: 0.27889419830855045, 3: 0.27554115632039955}\n"
     ]
    }
   ],
   "source": [
    "class_weight = {0: 0.25,\n",
    "                1: 0.25,\n",
    "                2: 0.25,\n",
    "                3: 0.25}\n",
    "total = 0\n",
    "for i in range(4):\n",
    "    class_weight[i] = (len(labels) + len(testLabels))/(np.sum(np.argmax(labels, 1)==i)+np.sum(np.argmax(testLabels, 1)==i))\n",
    "    total += class_weight[i]\n",
    "\n",
    "for i in range(4):\n",
    "    class_weight[i] /= total\n",
    "\n",
    "print(class_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 675,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30160\n"
     ]
    }
   ],
   "source": [
    "print(len(labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 684,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-684-259aa9d639cb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m model = buildSingleModel(embeddingMatrix_sswe, hidDim=32,\n\u001b[1;32m      7\u001b[0m                          \u001b[0mdropout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmultitask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m                         first_type='lstm', second_type='gru')\n\u001b[0m\u001b[1;32m      9\u001b[0m model.fit(train_all, labels,\n\u001b[1;32m     10\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2048\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m150\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-608-2d0f6896489f>\u001b[0m in \u001b[0;36mbuildSingleModel\u001b[0;34m(embeddingMatrix, hidDim, dropout, first_type, second_type, multitask)\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0minp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMAX_SEQUENCE_LENGTH\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membeddingLayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSpatialDropout1D\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfirst_type\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'lstm'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, **kwargs)\u001b[0m\n\u001b[1;32m    437\u001b[0m                 \u001b[0;31m# Load weights that were specified at layer instantiation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    438\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initial_weights\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 439\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initial_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    440\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    441\u001b[0m             \u001b[0;31m# Raise exceptions in case the input is not compatible\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36mset_weights\u001b[0;34m(self, weights)\u001b[0m\n\u001b[1;32m   1064\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1065\u001b[0m         \u001b[0mweight_value_tuples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1066\u001b[0;31m         \u001b[0mparam_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_get_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1067\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mpv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mw\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam_values\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1068\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mbatch_get_value\u001b[0;34m(ops)\u001b[0m\n\u001b[1;32m   2383\u001b[0m     \"\"\"\n\u001b[1;32m   2384\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2385\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2386\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2387\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m()\u001b[0m\n\u001b[1;32m    194\u001b[0m                 \u001b[0;31m# not already marked as initialized.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m                 is_initialized = session.run(\n\u001b[0;32m--> 196\u001b[0;31m                     [tf.is_variable_initialized(v) for v in candidate_vars])\n\u001b[0m\u001b[1;32m    197\u001b[0m                 \u001b[0muninitialized_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mflag\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_initialized\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcandidate_vars\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    885\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 887\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    888\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    889\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1108\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1109\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1110\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1111\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1112\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1284\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1285\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1286\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1287\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1288\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1290\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1291\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1292\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1293\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1294\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1273\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1274\u001b[0m       \u001b[0;31m# Ensure any changes to the graph are reflected in the runtime.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1275\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1276\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m   1277\u001b[0m           options, feed_dict, fetch_list, target_list, run_metadata)\n",
      "\u001b[0;32m~/persona/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_extend_graph\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1310\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1311\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session_run_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1312\u001b[0;31m       \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExtendSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1313\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1314\u001b[0m   \u001b[0;31m# The threshold to run garbage collection to delete dead tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from livelossplot.keras import PlotLossesCallback\n",
    "\n",
    "clr = utils.CyclicLR(base_lr=0.001, max_lr=0.005,\n",
    "               step_size=300., mode='exp_range',\n",
    "               gamma=0.99994)\n",
    "model = buildSingleModel(embeddingMatrix_sswe, hidDim=32,\n",
    "                         dropout=0.25, multitask=False,\n",
    "                        first_type='lstm', second_type='gru')\n",
    "model.fit(train_all, labels,\n",
    "          batch_size=2048, epochs=150,\n",
    "          class_weight=class_weight,\n",
    "          callbacks=[PlotLossesCallback(), clr],\n",
    "          verbose=0,\n",
    "          validation_data=(test_all, testLabels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 688,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10000 samples, validate on 2755 samples\n",
      "Epoch 1/50\n",
      "10000/10000 [==============================] - 71s 7ms/step - loss: 8.9474 - microF1Loss: 0.1788 - val_loss: 8.3421 - val_microF1Loss: nan\n",
      "Epoch 2/50\n",
      "10000/10000 [==============================] - 2s 153us/step - loss: 8.4274 - microF1Loss: 0.0265 - val_loss: 7.7955 - val_microF1Loss: nan\n",
      "Epoch 3/50\n",
      "10000/10000 [==============================] - 2s 157us/step - loss: 7.9839 - microF1Loss: 0.0062 - val_loss: 7.4208 - val_microF1Loss: nan\n",
      "Epoch 4/50\n",
      "10000/10000 [==============================] - 2s 155us/step - loss: 7.5532 - microF1Loss: 0.0043 - val_loss: 7.0119 - val_microF1Loss: nan\n",
      "Epoch 5/50\n",
      "10000/10000 [==============================] - 1s 148us/step - loss: 7.1520 - microF1Loss: 0.0055 - val_loss: 6.5742 - val_microF1Loss: nan\n",
      "Epoch 6/50\n",
      "10000/10000 [==============================] - 1s 146us/step - loss: 6.7661 - microF1Loss: 0.0027 - val_loss: 6.1990 - val_microF1Loss: nan\n",
      "Epoch 7/50\n",
      "10000/10000 [==============================] - 1s 149us/step - loss: 6.3974 - microF1Loss: 0.0207 - val_loss: 5.8636 - val_microF1Loss: 0.0899\n",
      "Epoch 8/50\n",
      "10000/10000 [==============================] - 2s 153us/step - loss: 6.0469 - microF1Loss: 0.0801 - val_loss: 5.5371 - val_microF1Loss: 0.1602\n",
      "Epoch 9/50\n",
      "10000/10000 [==============================] - 2s 154us/step - loss: 5.6991 - microF1Loss: 0.1527 - val_loss: 5.1873 - val_microF1Loss: 0.1706\n",
      "Epoch 10/50\n",
      "10000/10000 [==============================] - 2s 159us/step - loss: 5.3806 - microF1Loss: 0.2147 - val_loss: 4.9308 - val_microF1Loss: 0.1712\n",
      "Epoch 11/50\n",
      "10000/10000 [==============================] - 2s 164us/step - loss: 5.0841 - microF1Loss: 0.2428 - val_loss: 4.6592 - val_microF1Loss: 0.1820\n",
      "Epoch 12/50\n",
      "10000/10000 [==============================] - 2s 153us/step - loss: 4.8008 - microF1Loss: 0.2733 - val_loss: 4.4127 - val_microF1Loss: 0.1867\n",
      "Epoch 13/50\n",
      "10000/10000 [==============================] - 2s 174us/step - loss: 4.5409 - microF1Loss: 0.2759 - val_loss: 4.1585 - val_microF1Loss: 0.2104\n",
      "Epoch 14/50\n",
      "10000/10000 [==============================] - 2s 160us/step - loss: 4.3083 - microF1Loss: 0.3029 - val_loss: 3.9307 - val_microF1Loss: 0.2252\n",
      "Epoch 15/50\n",
      "10000/10000 [==============================] - 2s 158us/step - loss: 4.0802 - microF1Loss: 0.3251 - val_loss: 3.7016 - val_microF1Loss: 0.2343\n",
      "Epoch 16/50\n",
      "10000/10000 [==============================] - 2s 157us/step - loss: 3.8746 - microF1Loss: 0.3321 - val_loss: 3.5314 - val_microF1Loss: 0.2588\n",
      "Epoch 17/50\n",
      "10000/10000 [==============================] - 2s 155us/step - loss: 3.6833 - microF1Loss: 0.3669 - val_loss: 3.3128 - val_microF1Loss: 0.3034\n",
      "Epoch 18/50\n",
      "10000/10000 [==============================] - 2s 153us/step - loss: 3.5040 - microF1Loss: 0.4003 - val_loss: 3.1769 - val_microF1Loss: 0.3315\n",
      "Epoch 19/50\n",
      "10000/10000 [==============================] - 2s 150us/step - loss: 3.3479 - microF1Loss: 0.4098 - val_loss: 2.9856 - val_microF1Loss: 0.3436\n",
      "Epoch 20/50\n",
      "10000/10000 [==============================] - 2s 164us/step - loss: 3.1981 - microF1Loss: 0.4293 - val_loss: 2.9051 - val_microF1Loss: 0.3328\n",
      "Epoch 21/50\n",
      "10000/10000 [==============================] - 2s 163us/step - loss: 3.0576 - microF1Loss: 0.4581 - val_loss: 2.7093 - val_microF1Loss: 0.3788\n",
      "Epoch 22/50\n",
      "10000/10000 [==============================] - 2s 154us/step - loss: 2.9289 - microF1Loss: 0.4815 - val_loss: 2.6229 - val_microF1Loss: 0.3784\n",
      "Epoch 23/50\n",
      "10000/10000 [==============================] - 2s 169us/step - loss: 2.8120 - microF1Loss: 0.4775 - val_loss: 2.5007 - val_microF1Loss: 0.3868\n",
      "Epoch 24/50\n",
      "10000/10000 [==============================] - 2s 164us/step - loss: 2.6989 - microF1Loss: 0.5098 - val_loss: 2.3950 - val_microF1Loss: 0.4131\n",
      "Epoch 25/50\n",
      "10000/10000 [==============================] - 2s 169us/step - loss: 2.6020 - microF1Loss: 0.5181 - val_loss: 2.3608 - val_microF1Loss: 0.3933\n",
      "Epoch 26/50\n",
      "10000/10000 [==============================] - 2s 157us/step - loss: 2.5050 - microF1Loss: 0.5485 - val_loss: 2.1876 - val_microF1Loss: 0.4273\n",
      "Epoch 27/50\n",
      "10000/10000 [==============================] - 2s 153us/step - loss: 2.4269 - microF1Loss: 0.5432 - val_loss: 2.1955 - val_microF1Loss: 0.4073\n",
      "Epoch 28/50\n",
      "10000/10000 [==============================] - 2s 157us/step - loss: 2.3396 - microF1Loss: 0.5524 - val_loss: 2.0560 - val_microF1Loss: 0.4360\n",
      "Epoch 29/50\n",
      "10000/10000 [==============================] - 2s 159us/step - loss: 2.2737 - microF1Loss: 0.5614 - val_loss: 2.0381 - val_microF1Loss: 0.4247\n",
      "Epoch 30/50\n",
      "10000/10000 [==============================] - 2s 158us/step - loss: 2.1987 - microF1Loss: 0.5767 - val_loss: 1.9315 - val_microF1Loss: 0.4480\n",
      "Epoch 31/50\n",
      "10000/10000 [==============================] - 2s 152us/step - loss: 2.1376 - microF1Loss: 0.5883 - val_loss: 1.9165 - val_microF1Loss: 0.4394\n",
      "Epoch 32/50\n",
      "10000/10000 [==============================] - 2s 157us/step - loss: 2.0801 - microF1Loss: 0.5939 - val_loss: 1.8393 - val_microF1Loss: 0.4530\n",
      "Epoch 33/50\n",
      "10000/10000 [==============================] - 2s 162us/step - loss: 2.0236 - microF1Loss: 0.6025 - val_loss: 1.7937 - val_microF1Loss: 0.4562\n",
      "Epoch 34/50\n",
      "10000/10000 [==============================] - 2s 166us/step - loss: 1.9815 - microF1Loss: 0.6146 - val_loss: 1.7636 - val_microF1Loss: 0.4560\n",
      "Epoch 35/50\n",
      "10000/10000 [==============================] - 2s 163us/step - loss: 1.9329 - microF1Loss: 0.6219 - val_loss: 1.6930 - val_microF1Loss: 0.4734\n",
      "Epoch 36/50\n",
      "10000/10000 [==============================] - 2s 151us/step - loss: 1.8911 - microF1Loss: 0.6213 - val_loss: 1.6996 - val_microF1Loss: 0.4710\n",
      "Epoch 37/50\n",
      "10000/10000 [==============================] - 2s 154us/step - loss: 1.8532 - microF1Loss: 0.6347 - val_loss: 1.6240 - val_microF1Loss: 0.4835\n",
      "Epoch 38/50\n",
      "10000/10000 [==============================] - 2s 160us/step - loss: 1.8152 - microF1Loss: 0.6405 - val_loss: 1.6092 - val_microF1Loss: 0.4869\n",
      "Epoch 39/50\n",
      "10000/10000 [==============================] - 2s 165us/step - loss: 1.7865 - microF1Loss: 0.6462 - val_loss: 1.5818 - val_microF1Loss: 0.4974\n",
      "Epoch 40/50\n",
      "10000/10000 [==============================] - 2s 154us/step - loss: 1.7603 - microF1Loss: 0.6501 - val_loss: 1.5586 - val_microF1Loss: 0.4932\n",
      "Epoch 41/50\n",
      "10000/10000 [==============================] - 2s 157us/step - loss: 1.7291 - microF1Loss: 0.6546 - val_loss: 1.5521 - val_microF1Loss: 0.4843\n",
      "Epoch 42/50\n",
      "10000/10000 [==============================] - 2s 159us/step - loss: 1.7008 - microF1Loss: 0.6608 - val_loss: 1.4830 - val_microF1Loss: 0.5113\n",
      "Epoch 43/50\n",
      "10000/10000 [==============================] - 2s 154us/step - loss: 1.6738 - microF1Loss: 0.6692 - val_loss: 1.5031 - val_microF1Loss: 0.4996\n",
      "Epoch 44/50\n",
      "10000/10000 [==============================] - 2s 158us/step - loss: 1.6556 - microF1Loss: 0.6726 - val_loss: 1.4414 - val_microF1Loss: 0.5211\n",
      "Epoch 45/50\n",
      "10000/10000 [==============================] - 1s 147us/step - loss: 1.6296 - microF1Loss: 0.6792 - val_loss: 1.4393 - val_microF1Loss: 0.5214\n",
      "Epoch 46/50\n",
      "10000/10000 [==============================] - 2s 152us/step - loss: 1.6030 - microF1Loss: 0.6753 - val_loss: 1.4487 - val_microF1Loss: 0.5082\n",
      "Epoch 47/50\n",
      "10000/10000 [==============================] - 2s 155us/step - loss: 1.5932 - microF1Loss: 0.6888 - val_loss: 1.3870 - val_microF1Loss: 0.5393\n",
      "Epoch 48/50\n",
      "10000/10000 [==============================] - 2s 156us/step - loss: 1.5798 - microF1Loss: 0.6762 - val_loss: 1.4067 - val_microF1Loss: 0.5231\n",
      "Epoch 49/50\n",
      "10000/10000 [==============================] - 2s 153us/step - loss: 1.5563 - microF1Loss: 0.6819 - val_loss: 1.3784 - val_microF1Loss: 0.5366\n",
      "Epoch 50/50\n",
      "10000/10000 [==============================] - 1s 149us/step - loss: 1.5387 - microF1Loss: 0.6999 - val_loss: 1.3507 - val_microF1Loss: 0.5423\n",
      "Query no. 1\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 525us/step - loss: 2.2143 - microF1Loss: 0.2928 - val_loss: 1.3542 - val_microF1Loss: 0.5450\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 0s 489us/step - loss: 2.2080 - microF1Loss: 0.2889 - val_loss: 1.3865 - val_microF1Loss: 0.5345\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 531us/step - loss: 2.1765 - microF1Loss: 0.3516 - val_loss: 1.4213 - val_microF1Loss: 0.5133\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 514us/step - loss: 2.1775 - microF1Loss: 0.3796 - val_loss: 1.4377 - val_microF1Loss: 0.5107\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 510us/step - loss: 2.1621 - microF1Loss: 0.3731 - val_loss: 1.4308 - val_microF1Loss: 0.5269\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 521us/step - loss: 2.1445 - microF1Loss: 0.3710 - val_loss: 1.4192 - val_microF1Loss: 0.5470\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 554us/step - loss: 2.1201 - microF1Loss: 0.3559 - val_loss: 1.4120 - val_microF1Loss: 0.5704\n",
      "Query no. 2\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 546us/step - loss: 2.1402 - microF1Loss: 0.3306 - val_loss: 1.4265 - val_microF1Loss: 0.5745\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 545us/step - loss: 2.1445 - microF1Loss: 0.3096 - val_loss: 1.4660 - val_microF1Loss: 0.5615\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 534us/step - loss: 2.1349 - microF1Loss: 0.3203 - val_loss: 1.5138 - val_microF1Loss: 0.5504\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 589us/step - loss: 2.1360 - microF1Loss: 0.3281 - val_loss: 1.5660 - val_microF1Loss: 0.5357\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 519us/step - loss: 2.1362 - microF1Loss: 0.3423 - val_loss: 1.6174 - val_microF1Loss: 0.5138\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 505us/step - loss: 2.1264 - microF1Loss: 0.3652 - val_loss: 1.6623 - val_microF1Loss: 0.5068\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 527us/step - loss: 2.1192 - microF1Loss: 0.3742 - val_loss: 1.6983 - val_microF1Loss: 0.4808\n",
      "Query no. 3\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 556us/step - loss: 2.0803 - microF1Loss: 0.2024 - val_loss: 1.6510 - val_microF1Loss: 0.5070\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 568us/step - loss: 2.0567 - microF1Loss: 0.2116 - val_loss: 1.5464 - val_microF1Loss: 0.5498\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 551us/step - loss: 1.9662 - microF1Loss: 0.1847 - val_loss: 1.4041 - val_microF1Loss: 0.5876\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 548us/step - loss: 1.9130 - microF1Loss: 0.0814 - val_loss: 1.2806 - val_microF1Loss: 0.5568\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 544us/step - loss: 1.8570 - microF1Loss: 0.0399 - val_loss: 1.2155 - val_microF1Loss: 0.4886\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 566us/step - loss: 1.8621 - microF1Loss: 0.0237 - val_loss: 1.1993 - val_microF1Loss: 0.4382\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 580us/step - loss: 1.8881 - microF1Loss: 0.0060 - val_loss: 1.2023 - val_microF1Loss: 0.4258\n",
      "Query no. 4\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 528us/step - loss: 2.2087 - microF1Loss: 0.3047 - val_loss: 1.2244 - val_microF1Loss: 0.5101\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 555us/step - loss: 1.9941 - microF1Loss: 0.5031 - val_loss: 1.2815 - val_microF1Loss: 0.5712\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 554us/step - loss: 1.8001 - microF1Loss: 0.6967 - val_loss: 1.3736 - val_microF1Loss: 0.5556\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 561us/step - loss: 1.6616 - microF1Loss: 0.7703 - val_loss: 1.4932 - val_microF1Loss: 0.4553\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 593us/step - loss: 1.5988 - microF1Loss: 0.7837 - val_loss: 1.6330 - val_microF1Loss: 0.3469\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 580us/step - loss: 1.6004 - microF1Loss: 0.7848 - val_loss: 1.7739 - val_microF1Loss: 0.2740\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 597us/step - loss: 1.6247 - microF1Loss: 0.7785 - val_loss: 1.8933 - val_microF1Loss: 0.2426\n",
      "Query no. 5\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 563us/step - loss: 2.0987 - microF1Loss: 0.1176 - val_loss: 1.9252 - val_microF1Loss: 0.2485\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 568us/step - loss: 2.0735 - microF1Loss: 0.1335 - val_loss: 1.8922 - val_microF1Loss: 0.2796\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 545us/step - loss: 2.0316 - microF1Loss: 0.1489 - val_loss: 1.8111 - val_microF1Loss: 0.3197\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 537us/step - loss: 1.9614 - microF1Loss: 0.1618 - val_loss: 1.7018 - val_microF1Loss: 0.3823\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 511us/step - loss: 1.8351 - microF1Loss: 0.2625 - val_loss: 1.5794 - val_microF1Loss: 0.4549\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 517us/step - loss: 1.7306 - microF1Loss: 0.3081 - val_loss: 1.4603 - val_microF1Loss: 0.4877\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 565us/step - loss: 1.6570 - microF1Loss: 0.2882 - val_loss: 1.3676 - val_microF1Loss: 0.5069\n",
      "Query no. 6\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 546us/step - loss: 2.0291 - microF1Loss: 0.3176 - val_loss: 1.2903 - val_microF1Loss: 0.5094\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 501us/step - loss: 2.0101 - microF1Loss: 0.2472 - val_loss: 1.2417 - val_microF1Loss: 0.5028\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 581us/step - loss: 2.0420 - microF1Loss: 0.1596 - val_loss: 1.2223 - val_microF1Loss: 0.4896\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 0s 494us/step - loss: 2.0213 - microF1Loss: 0.1279 - val_loss: 1.2231 - val_microF1Loss: 0.4798\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 513us/step - loss: 2.0275 - microF1Loss: 0.1638 - val_loss: 1.2383 - val_microF1Loss: 0.4802\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 522us/step - loss: 1.9916 - microF1Loss: 0.2114 - val_loss: 1.2639 - val_microF1Loss: 0.4683\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 528us/step - loss: 1.9326 - microF1Loss: 0.2757 - val_loss: 1.2907 - val_microF1Loss: 0.4797\n",
      "Query no. 7\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 556us/step - loss: 2.1507 - microF1Loss: 0.2265 - val_loss: 1.3121 - val_microF1Loss: 0.4983\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 550us/step - loss: 2.0909 - microF1Loss: 0.2801 - val_loss: 1.3313 - val_microF1Loss: 0.5262\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 537us/step - loss: 2.0286 - microF1Loss: 0.3047 - val_loss: 1.3526 - val_microF1Loss: 0.5431\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 523us/step - loss: 1.9640 - microF1Loss: 0.4270 - val_loss: 1.3771 - val_microF1Loss: 0.5553\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 516us/step - loss: 1.9228 - microF1Loss: 0.5208 - val_loss: 1.4046 - val_microF1Loss: 0.5568\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 536us/step - loss: 1.8652 - microF1Loss: 0.5919 - val_loss: 1.4333 - val_microF1Loss: 0.5326\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 529us/step - loss: 1.8435 - microF1Loss: 0.6400 - val_loss: 1.4618 - val_microF1Loss: 0.4892\n",
      "Query no. 8\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 539us/step - loss: 2.0148 - microF1Loss: 0.3580 - val_loss: 1.4972 - val_microF1Loss: 0.4659\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 505us/step - loss: 2.0091 - microF1Loss: 0.3093 - val_loss: 1.5368 - val_microF1Loss: 0.4487\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 552us/step - loss: 2.0073 - microF1Loss: 0.3592 - val_loss: 1.5802 - val_microF1Loss: 0.4492\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 551us/step - loss: 1.9751 - microF1Loss: 0.3897 - val_loss: 1.6246 - val_microF1Loss: 0.4497\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 512us/step - loss: 1.9210 - microF1Loss: 0.4866 - val_loss: 1.6693 - val_microF1Loss: 0.4407\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 545us/step - loss: 1.8885 - microF1Loss: 0.5677 - val_loss: 1.7118 - val_microF1Loss: 0.4208\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 533us/step - loss: 1.8403 - microF1Loss: 0.6317 - val_loss: 1.7501 - val_microF1Loss: 0.3850\n",
      "Query no. 9\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 509us/step - loss: 2.0483 - microF1Loss: 0.1741 - val_loss: 1.7383 - val_microF1Loss: 0.3770\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 554us/step - loss: 2.0320 - microF1Loss: 0.1598 - val_loss: 1.6939 - val_microF1Loss: 0.3959\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 546us/step - loss: 1.9474 - microF1Loss: 0.2285 - val_loss: 1.6393 - val_microF1Loss: 0.4173\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 543us/step - loss: 1.8618 - microF1Loss: 0.2935 - val_loss: 1.5892 - val_microF1Loss: 0.4495\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 517us/step - loss: 1.7878 - microF1Loss: 0.4140 - val_loss: 1.5468 - val_microF1Loss: 0.4488\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 0s 497us/step - loss: 1.7301 - microF1Loss: 0.4673 - val_loss: 1.5095 - val_microF1Loss: 0.4352\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 525us/step - loss: 1.6856 - microF1Loss: 0.4716 - val_loss: 1.4683 - val_microF1Loss: 0.4351\n",
      "Query no. 10\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 0s 497us/step - loss: 1.9837 - microF1Loss: 0.3685 - val_loss: 1.4267 - val_microF1Loss: 0.4408\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 538us/step - loss: 1.9866 - microF1Loss: 0.3531 - val_loss: 1.3812 - val_microF1Loss: 0.4524\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 547us/step - loss: 2.0239 - microF1Loss: 0.2936 - val_loss: 1.3339 - val_microF1Loss: 0.4723\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 521us/step - loss: 2.0182 - microF1Loss: 0.2982 - val_loss: 1.2939 - val_microF1Loss: 0.5040\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 547us/step - loss: 1.9579 - microF1Loss: 0.3641 - val_loss: 1.2646 - val_microF1Loss: 0.5284\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 0s 498us/step - loss: 1.9209 - microF1Loss: 0.3819 - val_loss: 1.2481 - val_microF1Loss: 0.5616\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 550us/step - loss: 1.8531 - microF1Loss: 0.4814 - val_loss: 1.2439 - val_microF1Loss: 0.5919\n",
      "Query no. 11\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 528us/step - loss: 1.7733 - microF1Loss: 0.4533 - val_loss: 1.2533 - val_microF1Loss: 0.5716\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 507us/step - loss: 1.7521 - microF1Loss: 0.4801 - val_loss: 1.2740 - val_microF1Loss: 0.5578\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 534us/step - loss: 1.7390 - microF1Loss: 0.5325 - val_loss: 1.3044 - val_microF1Loss: 0.5450\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 527us/step - loss: 1.7127 - microF1Loss: 0.5841 - val_loss: 1.3430 - val_microF1Loss: 0.5239\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 525us/step - loss: 1.6655 - microF1Loss: 0.6401 - val_loss: 1.3873 - val_microF1Loss: 0.4999\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 601us/step - loss: 1.6932 - microF1Loss: 0.6430 - val_loss: 1.4339 - val_microF1Loss: 0.4597\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 561us/step - loss: 1.6861 - microF1Loss: 0.6482 - val_loss: 1.4791 - val_microF1Loss: 0.4309\n",
      "Query no. 12\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 542us/step - loss: 1.8426 - microF1Loss: 0.2850 - val_loss: 1.4952 - val_microF1Loss: 0.4173\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 561us/step - loss: 1.8797 - microF1Loss: 0.2651 - val_loss: 1.4816 - val_microF1Loss: 0.4213\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 531us/step - loss: 1.8469 - microF1Loss: 0.2967 - val_loss: 1.4492 - val_microF1Loss: 0.4336\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 559us/step - loss: 1.7808 - microF1Loss: 0.3431 - val_loss: 1.4100 - val_microF1Loss: 0.4434\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 521us/step - loss: 1.7073 - microF1Loss: 0.3664 - val_loss: 1.3724 - val_microF1Loss: 0.4654\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 540us/step - loss: 1.6535 - microF1Loss: 0.3833 - val_loss: 1.3411 - val_microF1Loss: 0.4752\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 533us/step - loss: 1.6252 - microF1Loss: 0.4301 - val_loss: 1.3166 - val_microF1Loss: 0.4776\n",
      "Query no. 13\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 547us/step - loss: 1.7173 - microF1Loss: 0.3910 - val_loss: 1.2762 - val_microF1Loss: 0.5004\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 535us/step - loss: 1.6958 - microF1Loss: 0.3550 - val_loss: 1.2247 - val_microF1Loss: 0.5224\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 521us/step - loss: 1.6909 - microF1Loss: 0.3321 - val_loss: 1.1739 - val_microF1Loss: 0.5574\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 547us/step - loss: 1.6739 - microF1Loss: 0.3324 - val_loss: 1.1342 - val_microF1Loss: 0.5913\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 530us/step - loss: 1.6590 - microF1Loss: 0.3233 - val_loss: 1.1087 - val_microF1Loss: 0.6003\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 514us/step - loss: 1.6601 - microF1Loss: 0.2819 - val_loss: 1.0950 - val_microF1Loss: 0.5921\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 541us/step - loss: 1.6538 - microF1Loss: 0.2746 - val_loss: 1.0903 - val_microF1Loss: 0.5990\n",
      "Query no. 14\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 535us/step - loss: 1.7940 - microF1Loss: 0.4647 - val_loss: 1.0978 - val_microF1Loss: 0.5980\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 541us/step - loss: 1.7694 - microF1Loss: 0.4471 - val_loss: 1.1175 - val_microF1Loss: 0.6064\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 542us/step - loss: 1.6805 - microF1Loss: 0.5506 - val_loss: 1.1500 - val_microF1Loss: 0.6038\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 525us/step - loss: 1.5946 - microF1Loss: 0.6600 - val_loss: 1.1948 - val_microF1Loss: 0.5924\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 520us/step - loss: 1.5458 - microF1Loss: 0.7278 - val_loss: 1.2497 - val_microF1Loss: 0.5647\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 565us/step - loss: 1.5070 - microF1Loss: 0.7707 - val_loss: 1.3124 - val_microF1Loss: 0.5221\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 556us/step - loss: 1.4495 - microF1Loss: 0.8073 - val_loss: 1.3804 - val_microF1Loss: 0.4789\n",
      "Query no. 15\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 533us/step - loss: 1.6028 - microF1Loss: 0.2329 - val_loss: 1.4262 - val_microF1Loss: 0.4585\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 539us/step - loss: 1.6285 - microF1Loss: 0.3086 - val_loss: 1.4487 - val_microF1Loss: 0.4485\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 553us/step - loss: 1.6304 - microF1Loss: 0.2762 - val_loss: 1.4533 - val_microF1Loss: 0.4420\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 537us/step - loss: 1.6102 - microF1Loss: 0.3155 - val_loss: 1.4465 - val_microF1Loss: 0.4461\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 541us/step - loss: 1.6027 - microF1Loss: 0.2900 - val_loss: 1.4315 - val_microF1Loss: 0.4477\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 554us/step - loss: 1.5784 - microF1Loss: 0.3006 - val_loss: 1.4091 - val_microF1Loss: 0.4609\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 535us/step - loss: 1.5573 - microF1Loss: 0.2287 - val_loss: 1.3802 - val_microF1Loss: 0.4724\n",
      "Query no. 16\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 513us/step - loss: 1.6018 - microF1Loss: 0.2950 - val_loss: 1.3410 - val_microF1Loss: 0.4814\n",
      "Epoch 2/7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 1s 598us/step - loss: 1.6036 - microF1Loss: 0.2820 - val_loss: 1.2960 - val_microF1Loss: 0.5056\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 528us/step - loss: 1.5774 - microF1Loss: 0.2481 - val_loss: 1.2517 - val_microF1Loss: 0.5261\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 557us/step - loss: 1.5042 - microF1Loss: 0.2787 - val_loss: 1.2128 - val_microF1Loss: 0.5414\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 571us/step - loss: 1.4732 - microF1Loss: 0.2270 - val_loss: 1.1827 - val_microF1Loss: 0.5546\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 584us/step - loss: 1.4323 - microF1Loss: 0.2749 - val_loss: 1.1631 - val_microF1Loss: 0.5594\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 569us/step - loss: 1.3939 - microF1Loss: 0.3595 - val_loss: 1.1531 - val_microF1Loss: 0.5718\n",
      "Query no. 17\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 0s 492us/step - loss: 1.5004 - microF1Loss: 0.5798 - val_loss: 1.1576 - val_microF1Loss: 0.5673\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 512us/step - loss: 1.4598 - microF1Loss: 0.6327 - val_loss: 1.1747 - val_microF1Loss: 0.5618\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 564us/step - loss: 1.4375 - microF1Loss: 0.7013 - val_loss: 1.2021 - val_microF1Loss: 0.5402\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 0s 498us/step - loss: 1.4262 - microF1Loss: 0.6973 - val_loss: 1.2365 - val_microF1Loss: 0.5138\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 540us/step - loss: 1.4461 - microF1Loss: 0.7127 - val_loss: 1.2700 - val_microF1Loss: 0.4980\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 566us/step - loss: 1.4571 - microF1Loss: 0.7210 - val_loss: 1.2976 - val_microF1Loss: 0.4818\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 557us/step - loss: 1.4640 - microF1Loss: 0.7202 - val_loss: 1.3124 - val_microF1Loss: 0.4747\n",
      "Query no. 18\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 548us/step - loss: 1.4821 - microF1Loss: 0.3141 - val_loss: 1.2826 - val_microF1Loss: 0.4942\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 510us/step - loss: 1.4408 - microF1Loss: 0.2874 - val_loss: 1.2282 - val_microF1Loss: 0.5352\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 0s 493us/step - loss: 1.3582 - microF1Loss: 0.3375 - val_loss: 1.1696 - val_microF1Loss: 0.5698\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 523us/step - loss: 1.2956 - microF1Loss: 0.3853 - val_loss: 1.1198 - val_microF1Loss: 0.5962\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 516us/step - loss: 1.2464 - microF1Loss: 0.4421 - val_loss: 1.0847 - val_microF1Loss: 0.5859\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 545us/step - loss: 1.2146 - microF1Loss: 0.4155 - val_loss: 1.0630 - val_microF1Loss: 0.5722\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 569us/step - loss: 1.1883 - microF1Loss: 0.4555 - val_loss: 1.0523 - val_microF1Loss: 0.5433\n",
      "Query no. 19\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 581us/step - loss: 1.6677 - microF1Loss: 0.4497 - val_loss: 1.0535 - val_microF1Loss: 0.5451\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 544us/step - loss: 1.6499 - microF1Loss: 0.4579 - val_loss: 1.0628 - val_microF1Loss: 0.5574\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 573us/step - loss: 1.5040 - microF1Loss: 0.6023 - val_loss: 1.0825 - val_microF1Loss: 0.5722\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 513us/step - loss: 1.3001 - microF1Loss: 0.8141 - val_loss: 1.1138 - val_microF1Loss: 0.5662\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 583us/step - loss: 1.1924 - microF1Loss: 0.8986 - val_loss: 1.1576 - val_microF1Loss: 0.5434\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 503us/step - loss: 1.0948 - microF1Loss: 0.9450 - val_loss: 1.2151 - val_microF1Loss: 0.5118\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 508us/step - loss: 1.0256 - microF1Loss: 0.9556 - val_loss: 1.2878 - val_microF1Loss: 0.4773\n",
      "Query no. 20\n",
      "Train on 1000 samples, validate on 2755 samples\n",
      "Epoch 1/7\n",
      "1000/1000 [==============================] - 1s 564us/step - loss: 1.3960 - microF1Loss: 0.5506 - val_loss: 1.3486 - val_microF1Loss: 0.4542\n",
      "Epoch 2/7\n",
      "1000/1000 [==============================] - 1s 538us/step - loss: 1.4483 - microF1Loss: 0.5258 - val_loss: 1.3841 - val_microF1Loss: 0.4369\n",
      "Epoch 3/7\n",
      "1000/1000 [==============================] - 1s 525us/step - loss: 1.4745 - microF1Loss: 0.5300 - val_loss: 1.3875 - val_microF1Loss: 0.4349\n",
      "Epoch 4/7\n",
      "1000/1000 [==============================] - 1s 569us/step - loss: 1.4857 - microF1Loss: 0.5311 - val_loss: 1.3611 - val_microF1Loss: 0.4463\n",
      "Epoch 5/7\n",
      "1000/1000 [==============================] - 1s 556us/step - loss: 1.4477 - microF1Loss: 0.5856 - val_loss: 1.3130 - val_microF1Loss: 0.4619\n",
      "Epoch 6/7\n",
      "1000/1000 [==============================] - 1s 563us/step - loss: 1.3843 - microF1Loss: 0.6268 - val_loss: 1.2562 - val_microF1Loss: 0.4790\n",
      "Epoch 7/7\n",
      "1000/1000 [==============================] - 1s 550us/step - loss: 1.3034 - microF1Loss: 0.6601 - val_loss: 1.2007 - val_microF1Loss: 0.4985\n"
     ]
    }
   ],
   "source": [
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from modAL.models import ActiveLearner\n",
    "\n",
    "real_model = buildSingleModel(embeddingMatrix_sswe, hidDim=32,\n",
    "                         dropout=0.25, multitask=False,\n",
    "                        first_type='lstm', second_type='gru')\n",
    "\n",
    "def get_my_model():\n",
    "    return real_model\n",
    "\n",
    "classifier = KerasClassifier(get_my_model)\n",
    "\n",
    "n_initial = 10000\n",
    "initial_idx = np.random.choice(range(len(train_all)), size=n_initial, replace=False)\n",
    "X_initial = train_all[initial_idx]\n",
    "y_initial = labels[initial_idx]\n",
    "\n",
    "X_pool = train_all[:]\n",
    "y_pool = labels[:]\n",
    "\n",
    "# initialize ActiveLearner\n",
    "learner = ActiveLearner(\n",
    "    estimator=classifier,\n",
    "    X_training=X_initial, y_training=y_initial,\n",
    "    verbose=1,\n",
    "    validation_data=(test_all, testLabels),\n",
    "    batch_size=2048,\n",
    "    epochs=50\n",
    ")\n",
    "\n",
    "# the active learning loop\n",
    "n_queries = 20\n",
    "for idx in range(n_queries):\n",
    "    print('Query no. %d' % (idx + 1))\n",
    "    query_idx, query_instance = learner.query(X_pool, n_instances=1000, verbose=0, batch_size=1024)\n",
    "    learner.teach(\n",
    "        X=X_pool[query_idx], y=y_pool[query_idx], only_new=True,\n",
    "        verbose=1,\n",
    "        validation_data=(test_all, testLabels),\n",
    "        batch_size=1024,\n",
    "        epochs=7\n",
    "    )\n",
    "    # remove queried instance from pool\n",
    "    X_pool = np.delete(X_pool, query_idx, axis=0)\n",
    "    y_pool = np.delete(y_pool, query_idx, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def construct_meta_features(data):\n",
    "    # Compute emoji-based features\n",
    "    emojis = ['😂', '😭', '😞', '😢', '😁', '😅', '😍',\n",
    "              '😀', '😃', '😡', '😄', '😆', '😒', '😊',\n",
    "              '😌', '😠', '😤', '🙂', '😺', '😫', '😩',\n",
    "              '😹', '😜', '👍', '😘', '😸', '😉', '😽',\n",
    "              '😻', '😏', '💔', '😝', '😑', '🙁', '😾',\n",
    "              '😿', '😬', '❤', '😋', '🙄', '😔', '🙀',\n",
    "              '😎', '👎', '😦', '😧', '❤️', '😛', '😶',\n",
    "              '😐', '👌', '🤔','😇', '😨', '😯', '😳',\n",
    "              '☹️', '💋', '👋', '😪', '😥', '💕', '😱',\n",
    "              '🙈', '😟', '🙏', '✌', '😖', '😣', '😮',\n",
    "              '🤗', '😓', '😷', '☹', '💞', '🏻', '🙌',\n",
    "              '💐', '🙊', '😰', '☺', '😴', '🖕', '♥', '😕',\n",
    "              '😈', '💗', '♡', '👀', '👊', '‑c', ' 8‑d', ' ‑d',\n",
    "              '👻', '：）', '.', '?', '!', ',', '-', '・', \"'-'\",\n",
    "              '\\U0001f923','・ω・', '\\U000fe339', ' ‑c']\n",
    "    happy_words = ['happy', 'lol', 'haha', 'enjoy', 'cool', 'glad',\n",
    "                   'smile', 'nice', 'funny', 'wow', 'good', 'best',\n",
    "                   'party', 'baby', 'sweet', 'joke', 'glad', 'perfect',\n",
    "                   'fantastic', 'excite', 'cute', 'enjoy', 'omg']\n",
    "    angry_words = ['angry', 'fuck', 'hell', 'shut up', 'bad', 'rude',\n",
    "                  'block', 'stupid', 'piss', 'lame', \"don't\", 'mean',\n",
    "                  'irritat', 'hate', 'ignore', 'get lost', 'reply',\n",
    "                  'fool', 'regret', 'dumb', 'cheat', 'whore', 'disgust']\n",
    "    sad_words   = ['sad', 'sorry', 'miss', 'alone', 'lonely', 'cry',\n",
    "                   'disappointed', 'not', 'no', 'not happy', 'crazy',\n",
    "                   'stress', 'depress', 'poor', 'care', 'health', 'break up',\n",
    "                   'breaking up', 'upset', 'forgive', 'left me', 'dump']\n",
    "    others_words = ['thank you', 'favorite', 'favourite']\n",
    "    indicator_words = emojis + happy_words + angry_words + sad_words + others_words\n",
    "    \n",
    "    word_features = np.zeros((len(data), len(indicator_words)))\n",
    "    for i, text in enumerate(data):\n",
    "        for j, word in enumerate(indicator_words):\n",
    "            useful_text = text.lower()\n",
    "#             useful_text = \" \".join([text.split(' <eos> ')[0], text.split(' <eos> ')[-1]]).lower()\n",
    "            word_features[i][j] += useful_text.count(word)\n",
    "    \n",
    "    # Compute CAPS-based features\n",
    "    capital_features = np.zeros((len(data), 3))\n",
    "    for i, text in enumerate(data):\n",
    "        for word in text.split(' '):\n",
    "            if word.isupper():\n",
    "                capital_features[i][0] += 1\n",
    "        capital_features[i][1] = capital_features[i][0] / (len(text.split(' ')) + 1)\n",
    "        capital_features[i][2] = sum([len(x) for x in text.split(' ')]) / len(text.split(' '))\n",
    "    \n",
    "    # Combine metadata-based features\n",
    "    metadata_features = np.concatenate((word_features, capital_features), axis=1)\n",
    "    return metadata_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 506,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30160, 180)\n"
     ]
    }
   ],
   "source": [
    "metadata_features = construct_meta_features(rawtrainTexts)\n",
    "print(metadata_features.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 510,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Positives per class :  [14930.  4063.  5074.  5246.]\n",
      "False Positives per class :  [787.  12.  44.   4.]\n",
      "False Negatives per class :  [ 18. 180. 389. 260.]\n",
      "Class happy : Precision : 0.997, Recall : 0.958, F1 : 0.977\n",
      "Class sad : Precision : 0.991, Recall : 0.929, F1 : 0.959\n",
      "Class angry : Precision : 0.999, Recall : 0.953, F1 : 0.975\n",
      "Ignoring the Others class, Macro Precision : 0.9959, Macro Recall : 0.9464, Macro F1 : 0.9705\n",
      "Ignoring the Others class, Micro TP : 14383, FP : 60, FN : 829\n",
      "Accuracy : 0.9719, Micro Precision : 0.9958, Micro Recall : 0.9455, Micro F1 : 0.9700\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.9719164456233422, 0.99584574, 0.94550353, 0.9700219082279653)"
      ]
     },
     "execution_count": 510,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train classifier for metadata-based classification\n",
    "from sklearn import tree\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "meta_clf = tree.DecisionTreeClassifier()\n",
    "meta_clf.fit(metadata_features, np.argmax(labels, axis=1))\n",
    "utils.getMetrics(meta_clf.predict_proba(metadata_features), labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 590,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict(test_all, batch_size=1024)\n",
    "eval_predictions = model.predict(eval_all, batch_size=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 512,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_meta = construct_meta_features(rawtestTexts)\n",
    "predictions_meta = meta_clf.predict_proba(test_meta)\n",
    "\n",
    "eval_meta = construct_meta_features(rawevalTexts)\n",
    "eval_predictions_meta = meta_clf.predict_proba(eval_meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 591,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Positives per class :  [2202.   75.   92.  124.]\n",
      "False Positives per class :  [110.  41.  35.  76.]\n",
      "False Negatives per class :  [136.  67.  33.  26.]\n",
      "Class happy : Precision : 0.647, Recall : 0.528, F1 : 0.581\n",
      "Class sad : Precision : 0.724, Recall : 0.736, F1 : 0.730\n",
      "Class angry : Precision : 0.620, Recall : 0.827, F1 : 0.709\n",
      "Ignoring the Others class, Macro Precision : 0.6637, Macro Recall : 0.6969, Macro F1 : 0.6799\n",
      "Ignoring the Others class, Micro TP : 291, FP : 152, FN : 126\n",
      "Accuracy : 0.9049, Micro Precision : 0.6569, Micro Recall : 0.6978, Micro F1 : 0.6767\n",
      "\n",
      "True Positives per class :  [2138.  102.   94.  127.]\n",
      "False Positives per class :  [ 76. 114.  35.  69.]\n",
      "False Negatives per class :  [200.  40.  31.  23.]\n",
      "Class happy : Precision : 0.472, Recall : 0.718, F1 : 0.570\n",
      "Class sad : Precision : 0.729, Recall : 0.752, F1 : 0.740\n",
      "Class angry : Precision : 0.648, Recall : 0.847, F1 : 0.734\n",
      "Ignoring the Others class, Macro Precision : 0.6163, Macro Recall : 0.7723, Macro F1 : 0.6855\n",
      "Ignoring the Others class, Micro TP : 323, FP : 218, FN : 94\n",
      "Accuracy : 0.8933, Micro Precision : 0.5970, Micro Recall : 0.7746, Micro F1 : 0.6743\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.8932849364791289, 0.5970425, 0.77458036, 0.6743215332041502)"
      ]
     },
     "execution_count": 591,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "utils.getMetrics(predictions, testLabels)\n",
    "print()\n",
    "predictions_sly = np.copy(predictions)\n",
    "eval_predictions_sly = np.copy(eval_predictions)\n",
    "\n",
    "for i in range(len(predictions_sly)):\n",
    "    if predictions_sly[i][1] >= 0.2:\n",
    "        predictions_sly[i] = [0, 1, 0, 0]\n",
    "#     if predictions_sly[i][0] >= 0.5:\n",
    "#         predictions_sly[i] = [1, 0, 0, 0]\n",
    "#     elif predictions_sly[i][1] >= 0.4:\n",
    "#         predictions_sly[i] = [0, 1, 0, 0]\n",
    "#     elif predictions_sly[i][2] >= 0.5:\n",
    "#         predictions_sly[i] = [0, 0, 1, 0]\n",
    "  \n",
    "alpha = 0.8\n",
    "# utils.getMetrics(predictions_sly, testLabels)\n",
    "# print()\n",
    "utils.getMetrics(alpha * predictions_sly + (1-alpha) * predictions_meta, testLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 552,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed\n"
     ]
    }
   ],
   "source": [
    "# predictions_towrite = (predictions).argmax(axis=1)\n",
    "predictions_towrite = (alpha * predictions_sly + (1-alpha) * predictions_meta).argmax(axis=1)\n",
    "\n",
    "with io.open(solutionPath, \"w\", encoding=\"utf8\") as fout:\n",
    "    fout.write('\\t'.join([\"id\", \"turn1\", \"turn2\", \"turn3\", \"label\"]) + '\\n')        \n",
    "    with io.open(testDataPath, encoding=\"utf8\") as fin:\n",
    "        fin.readline()\n",
    "        for lineNum, line in enumerate(fin):\n",
    "            fout.write('\\t'.join(line.strip().split('\\t')[:4]) + '\\t')\n",
    "            fout.write(label2emotion[predictions_towrite[lineNum]] + '\\n')\n",
    "print(\"Completed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 592,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Completed\n"
     ]
    }
   ],
   "source": [
    "# predictions_towrite = (predictions).argmax(axis=1)\n",
    "# evals_towrite = (alpha * eval_predictions_sly + (1-alpha) * eval_predictions_meta).argmax(axis=1)\n",
    "evals_towrite = eval_predictions_sly.argmax(axis=1)\n",
    "\n",
    "with io.open(solutionPath, \"w\", encoding=\"utf8\") as fout:\n",
    "    fout.write('\\t'.join([\"id\", \"turn1\", \"turn2\", \"turn3\", \"label\"]) + '\\n')        \n",
    "    with io.open(evalDataPath, encoding=\"utf8\") as fin:\n",
    "        fin.readline()\n",
    "        for lineNum, line in enumerate(fin):\n",
    "            fout.write('\\t'.join(line.strip().split('\\t')[:4]) + '\\t')\n",
    "            fout.write(label2emotion[evals_towrite[lineNum]] + '\\n')\n",
    "print(\"Completed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "persona",
   "language": "python",
   "name": "persona"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}