TC_LSTM_ATTENTION-AT3-MTC-Viz-Sentencelevel-Cleanup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline  \n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.layers import Bidirectional, Input, LSTM, Dense, Activation, Conv1D, Flatten, Embedding, MaxPooling1D, Dropout\n",
    "from keras.layers import Add, Concatenate, Lambda, Reshape, Permute, Average, Layer, TimeDistributed, Multiply, GRU, BatchNormalization\n",
    "#from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras import optimizers\n",
    "from gensim.models import Word2Vec\n",
    "from keras.models import Sequential, Model\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from sklearn.utils import shuffle\n",
    "import pickle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from autocorrect import spell\n",
    "import spacy\n",
    "from spacy.gold import GoldParse\n",
    "nlp = spacy.load('en')\n",
    "import re\n",
    "from sklearn.utils import shuffle\n",
    "import keras\n",
    "import joblib\n",
    "from keras.utils.vis_utils import plot_model\n",
    "import keras.backend as K\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from collections import defaultdict\n",
    "import tokenizer_util as tu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_cols = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['total_classes'] = df['toxic']+df['severe_toxic']+df['obscene']+df['threat']+df['insult']+df['identity_hate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comment_col = 'comment_text'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df[comment_col] = df[comment_col].astype(str).apply(lambda x : x.replace(\"'\", \"\").replace('\"',''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df[comment_col] = df[comment_col].apply(lambda x: re.sub('[0-9]','',x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comment_list = df[comment_col].tolist()\n",
    "n_classes = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total vocabulary is 233222\n"
     ]
    }
   ],
   "source": [
    "tokenizer = joblib.load('tokenizer_100')\n",
    "final_emb_matrix = joblib.load('embedding_100')\n",
    "print('Total vocabulary is {0}'.format(final_emb_matrix.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replacing all the unknown words with UNK. This will have no impact on training as all the words are known"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The vocabulary size is: 233221\n",
      "[[65247, 65247, 36, 65247, 65247]]\n"
     ]
    }
   ],
   "source": [
    "print (\"The vocabulary size is: {0}\".format(len(tokenizer.word_index)))\n",
    "print (tokenizer.texts_to_sequences([tu.replace_unknown_words_with_UNK(\"DFLSDKJFLS ADFSDF was Infosys CEO\", tokenizer)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('toxic', 15294), ('severe_toxic', 1595), ('obscene', 8449), ('threat', 478), ('insult', 7877), ('identity_hate', 1405)]\n"
     ]
    }
   ],
   "source": [
    "class_count = []\n",
    "for col in pred_cols:\n",
    "    class_count.append((col,len(df[df[col]==1])))\n",
    "print (class_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "      <th>total_classes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>34117</th>\n",
       "      <td>5b02208daa29a40f</td>\n",
       "      <td>Outrageous!!!!! \\n\\nThis block is outrageous a...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6579</th>\n",
       "      <td>1190ddc487465bd2</td>\n",
       "      <td>Except that you would never dare say something...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59858</th>\n",
       "      <td>a0473abe447e04e3</td>\n",
       "      <td>Thanks for your reply and your explanation; yo...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86152</th>\n",
       "      <td>e6763dac9d770096</td>\n",
       "      <td>or attempted generalization</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7620</th>\n",
       "      <td>1446437fe8605add</td>\n",
       "      <td>You seem to be vandalising the article.  Why a...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     id                                       comment_text  \\\n",
       "34117  5b02208daa29a40f  Outrageous!!!!! \\n\\nThis block is outrageous a...   \n",
       "6579   1190ddc487465bd2  Except that you would never dare say something...   \n",
       "59858  a0473abe447e04e3  Thanks for your reply and your explanation; yo...   \n",
       "86152  e6763dac9d770096                        or attempted generalization   \n",
       "7620   1446437fe8605add  You seem to be vandalising the article.  Why a...   \n",
       "\n",
       "       toxic  severe_toxic  obscene  threat  insult  identity_hate  \\\n",
       "34117      0             0        0       0       0              0   \n",
       "6579       0             0        0       0       0              0   \n",
       "59858      0             0        0       0       0              0   \n",
       "86152      0             0        0       0       0              0   \n",
       "7620       1             0        0       0       0              0   \n",
       "\n",
       "       total_classes  \n",
       "34117              0  \n",
       "6579               0  \n",
       "59858              0  \n",
       "86152              0  \n",
       "7620               1  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train, test = train_test_split(df, test_size=0.10, random_state=42)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "XVal = tokenizer.texts_to_sequences(test.astype(str)[comment_col].tolist())\n",
    "#YTrain = np.array(train[['toxic','severe_toxic','obscene','threat','insult','identity_hate']])\n",
    "#YVal = np.array(test[['toxic','severe_toxic','obscene','threat','insult','identity_hate']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ys(dftox, predcols):\n",
    "    ys = []\n",
    "    for col in predcols:\n",
    "        ys.append(np.array(dftox[col].tolist()))\n",
    "    return ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ys_unified(dftox, predcols):\n",
    "    ys = dftox[predcols].values\n",
    "    return ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "YTrain = ys(train, pred_cols)\n",
    "YVal = ys(test, pred_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Attention Layer with works follows the math from https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf\n",
    "This layer only computes the weights, does not multiply the RNN output with the weights. This layer has to be\n",
    "followed by a Multiply layer, followed by Reshape, followed by a Lambda for summing.\n",
    "\"\"\"\n",
    "class ATTNWORD(Layer):\n",
    "    def __init__(self,output_dim, **kwargs):\n",
    "        self.output_dim = output_dim\n",
    "        #self.supports_masking = True\n",
    "        super(ATTNWORD, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        # Create a trainable weight variable for this layer.\n",
    "        print('The input shape is: {}'.format(input_shape))\n",
    "        self.kernel = self.add_weight(name='kernel', \n",
    "                                      shape=(input_shape[-1], self.output_dim),\n",
    "                                      initializer='uniform',\n",
    "                                      trainable=True)\n",
    "        self.input_shape_bk = input_shape\n",
    "        super(ATTNWORD, self).build(input_shape)  \n",
    "\n",
    "    def call(self, x,mask=None):\n",
    "        print ('kernel shape', self.kernel.shape)\n",
    "        print ('Input shape', x.shape)\n",
    "        product = K.dot(x, self.kernel)\n",
    "        product = K.reshape(product, (-1, self.output_dim, self.input_shape_bk[1]))\n",
    "\n",
    "        x_norm  = K.softmax(product)\n",
    "        print ('Norm shape', x_norm.shape)\n",
    "        x_norm = K.reshape(x_norm, (-1, self.input_shape_bk[1],self.output_dim))\n",
    "\n",
    "        return x_norm\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0], input_shape[1], self.output_dim)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "A attenion layer, built on the basis of https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf.\n",
    "Takes care of all the atention compute, Takes array of input - Bidirectional RNN output and the TanH layer output.\n",
    "Usage ATTNWORD_COMPLETE(1)([tanh_output, rnn_output])\n",
    "\"\"\"\n",
    "class ATTNWORD_COMPLETE(Layer):\n",
    "    def __init__(self,output_dim, **kwargs):\n",
    "        self.output_dim = output_dim\n",
    "        #self.supports_masking = True\n",
    "        super(ATTNWORD, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        # Create a trainable weight variable for this layer.\n",
    "        print('The input shape is: {}'.format(input_shape))\n",
    "        self.kernel = self.add_weight(name='kernel', \n",
    "                                      shape=(input_shape[0][-1], self.output_dim),\n",
    "                                      initializer='uniform',\n",
    "                                      trainable=True)\n",
    "        self.input_shape_bk = input_shape\n",
    "        super(ATTNWORD, self).build(input_shape)  \n",
    "\n",
    "    def call(self, x,mask=None):\n",
    "        print ('kernel shape', self.kernel.shape)\n",
    "        print ('Input shape', x[0].shape)\n",
    "        product = K.dot(x[0], self.kernel)\n",
    "        product = K.reshape(product, (-1, self.output_dim, self.input_shape_bk[0][1]))\n",
    "        x_norm  = K.softmax(product)\n",
    "        print ('Norm shape', x_norm.shape)\n",
    "        x_norm = K.reshape(x_norm, (-1, self.input_shape_bk[0][1],self.output_dim))\n",
    "        print ('reshaped Norm shape: {0} and hit shape is {1}'.format( x_norm.shape, x[1].shape))\n",
    "        attn_final = x[1]*x_norm\n",
    "        print ('Attn final shape', attn_final.shape)\n",
    "        attn_final = K.reshape(attn_final, (-1, self.input_shape_bk[1][-1], self.input_shape_bk[0][1]))\n",
    "\n",
    "        attn_final = K.sum(attn_final, axis=2)\n",
    "        print ('Attn final shape sum', attn_final.shape)\n",
    "        return attn_final\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return (input_shape[0][0], input_shape[1][-1])        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This method creates a model with an input of word length, followed by embedding layer and finally GRU, \n",
    "with output dim as passed in the argument.\n",
    "\"\"\"\n",
    "def get_word_attention(emb_matrix, word_length, optimizer, nclasses, gru_output_dim=50):\n",
    "    input = Input(shape=(word_length, ), dtype='int32')\n",
    "    embedding = Embedding( input_dim=emb_matrix.shape[0], output_dim=emb_matrix.shape[1], weights=[emb_matrix],input_length=word_length,trainable=True, mask_zero=True)\n",
    "    sequence_input = embedding(input)\n",
    "    print('embedding: ',sequence_input.shape)\n",
    "    x = Bidirectional(GRU(gru_output_dim,return_sequences=True, dropout=0.1))(sequence_input)\n",
    "    #x = BatchNormalization()(x)\n",
    "    print('Shape after BD LSTM',x.shape)\n",
    "    model = Model(input, x) \n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This method applies attention only at the word level. The last layer is a sigmoid layer with output of 1. \n",
    "The output is going to be an array, the number of output is determined by n_classes.\n",
    "Here the labels are assumed to be independent of each other and probability for each label is independently calculated\n",
    "using dedicated Attention layer for each.\n",
    "\"\"\"\n",
    "def attention_words_only(emb_matrix, word_length, n_classes, trainable=True):\n",
    "    nclasses = n_classes\n",
    "    preds = []\n",
    "    attentions_pred = []\n",
    "    input = Input(shape=(word_length, ), dtype='int32')\n",
    "    embedding = Embedding( input_dim=emb_matrix.shape[0], output_dim=emb_matrix.shape[1], weights=[emb_matrix],input_length=word_length,trainable=True)\n",
    "    sequence_input = embedding(input)\n",
    "    print('embedding: ',sequence_input.shape)\n",
    "    x = Bidirectional(GRU(50,return_sequences=True))(sequence_input)\n",
    "    word_vectors = TimeDistributed(Dense(100, activation='tanh'))(x) #TanH layer as required by the paper, is external to the Attn layer.\n",
    "    print('Shape after word vector',word_vectors.shape)\n",
    "    h_it = x\n",
    "    print('Shape after reshape word vector',h_it.shape)\n",
    "\n",
    "    attn_final_word = [ATTNWORD_COMPLETE(1)([word_vectors, h_it]) for i in range(nclasses)]\n",
    "    print('ATTN Shape', attn_final_word[0].shape)\n",
    "    \n",
    "    for i in range(nclasses):\n",
    "        #x = Dense(128, activation='relu',trainable=trainable)(attn_final_word[i])\n",
    "        #x = Dropout(0.2)(x)\n",
    "        #x = Dense(128, activation='relu',trainable=trainable)(x)\n",
    "        #x = Dropout(0.2)(x)\n",
    "        #x = Dense(64, activation='relu',trainable=trainable)(x)\n",
    "        #x = Dropout(0.2)(x)\n",
    "        #x = Dense(64, activation='relu',trainable=trainable)(x)\n",
    "        p = Dense(1, activation='sigmoid')(attn_final_word[i])\n",
    "        preds.append(p)\n",
    "    model = Model(input, preds)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Method to return model with hierarchical attention. The output is an array of output of the size n_classes, each with \n",
    "its own sigmoid. \n",
    "\"\"\"\n",
    "def get_sentence_attention(word_model , word_length, sent_length, n_classes):\n",
    "    #x = Permute((2,1))(si_vects)\n",
    "    nclasses = n_classes\n",
    "    input = Input(shape=(sent_length, word_length ), dtype='int32')\n",
    "    print(' input to sentence attn network',word_model)\n",
    "    preds = []\n",
    "    attentions_pred = []\n",
    "\n",
    "    si_vects = TimeDistributed(word_model)(input)\n",
    "    print('Shape after si_vects', si_vects.shape)\n",
    "    u_it = TimeDistributed(TimeDistributed(Dense(100, activation='tanh')))(si_vects)\n",
    "    print('Shape after word vector',u_it.shape)\n",
    "    u_it = TimeDistributed(TimeDistributed(BatchNormalization()))(u_it)\n",
    "    \n",
    "    attn_final_word = [TimeDistributed(ATTNWORD(1))(u_it) for i in range(nclasses)]\n",
    "\n",
    "    print('ATTN Shape', attn_final_word[0].shape)\n",
    "    attn_final_word = [Multiply()([si_vects, attn_final_word[i]]) for i in range(nclasses)]#Multiply()([h_it,a_it])\n",
    "    print('Multi word Shape', attn_final_word[0].shape)\n",
    "    attn_final_word = [Reshape((sent_length, 100,word_length))(attn_final_word[i]) for i in range(nclasses)]\n",
    "    print ('Shape of the att1 is {}'.format(attn_final_word[0].shape))\n",
    "    attn_final_word = [Lambda(lambda x: K.sum(x, axis=3))(attn_final_word[i]) for i in range(nclasses)]\n",
    "    print ('Shape of the lambda word is {}'.format(attn_final_word[0].shape))\n",
    "    for i in range(nclasses):\n",
    "        x = Bidirectional(GRU(50,return_sequences=True, dropout=0.1))(attn_final_word[i])\n",
    "        #x = BatchNormalization()(x)\n",
    "\n",
    "        print('Shape after BD LSTM',x.shape)\n",
    "\n",
    "        u_it = TimeDistributed(Dense(100, activation='tanh'))(x)\n",
    "        #u_it = BatchNormalization()(u_it)\n",
    "        print('Shape after word vector',u_it.shape)\n",
    "\n",
    "        attn_final_sent = ATTNWORD(1)(u_it)\n",
    "        print ('Shape of the sent att is {}'.format(attn_final_sent.shape))\n",
    "\n",
    "        attn_final_sent = Multiply()([x, attn_final_sent])\n",
    "        print ('Shape of the multi sent att is {}'.format(attn_final_sent.shape))\n",
    "        attn_final_sent = Reshape((100,sent_length))(attn_final_sent)\n",
    "        attn_final_sent = Lambda(lambda x: K.sum(x, axis=2))(attn_final_sent)\n",
    "        print ('Shape of the lambda sent att is {}'.format(attn_final_sent.shape))\n",
    "        #p = Dense(100, activation='relu')(attn_final_sent)\n",
    "        #p = BatchNormalization()(p)\n",
    "        p = Dense(1, activation='sigmoid')(attn_final_sent)\n",
    "        preds.append(p)\n",
    "    model = Model(input, preds)\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_sentence_attention_combined_output(word_model , word_length, sent_length, n_classes):\n",
    "    #x = Permute((2,1))(si_vects)\n",
    "    nclasses = n_classes\n",
    "    input = Input(shape=(sent_length, word_length ), dtype='int32')\n",
    "    print(' input to sentence attn network',word_model)\n",
    "    attentions_pred = []\n",
    "    #print(output.summary())\n",
    "    si_vects = TimeDistributed(word_model)(input)\n",
    "    print('Shape after si_vects', si_vects.shape)\n",
    "    u_it = TimeDistributed(TimeDistributed(Dense(100, activation='tanh')))(si_vects)\n",
    "    print('Shape after word vector',u_it.shape)\n",
    "    #h_it = TimeDistributed(Reshape((100,word_length)))(si_vects)\n",
    "    #print('Shape after reshape word vector',h_it.shape)\n",
    "\n",
    "    attn_final_word = [TimeDistributed(ATTNWORD(1))(u_it) for i in range(nclasses)]\n",
    "    #a_it = Reshape(( word_length, 1))(a_it)\n",
    "    #h_it = Reshape((word_length, 512))(h_it)\n",
    "    print('ATTN Shape', attn_final_word[0].shape)\n",
    "    attn_final_word = [Multiply()([si_vects, attn_final_word[i]]) for i in range(nclasses)]#Multiply()([h_it,a_it])\n",
    "    print('Multi word Shape', attn_final_word[0].shape)\n",
    "    attn_final_word = [Reshape((sent_length, 100,word_length))(attn_final_word[i]) for i in range(nclasses)]\n",
    "    print ('Shape of the att1 is {}'.format(attn_final_word[0].shape))\n",
    "    attn_final_word = [Lambda(lambda x: K.sum(x, axis=3))(attn_final_word[i]) for i in range(nclasses)]\n",
    "    print ('Shape of the lambda word is {}'.format(attn_final_word[0].shape))\n",
    "    attn_sents_for_all_classes = []\n",
    "    for i in range(nclasses):\n",
    "        x = Bidirectional(GRU(50,return_sequences=True))(attn_final_word[i])\n",
    "        #x = Bidirectional(LSTM(256,return_sequences=True))(x)\n",
    "        print('Shape after BD LSTM',x.shape)\n",
    "        #x1 = Permute((2,1))(x)\n",
    "        #print('Shape after permute',x1.shape)\n",
    "        u_it = TimeDistributed(Dense(100, activation='tanh'))(x)\n",
    "        print('Shape after word vector',u_it.shape)\n",
    "        #h_it = Reshape((100,sent_length))(x)\n",
    "        attn_final_sent = ATTNWORD(1)(u_it)\n",
    "        print ('Shape of the sent att is {}'.format(attn_final_sent.shape))\n",
    "        #attentions_pred.append(attn_final)\n",
    "        attn_final_sent = Multiply()([x, attn_final_sent])\n",
    "        print ('Shape of the multi sent att is {}'.format(attn_final_sent.shape))\n",
    "        attn_final_sent = Reshape((100,sent_length))(attn_final_sent)\n",
    "        attn_final_sent = Lambda(lambda x: K.sum(x, axis=2))(attn_final_sent)\n",
    "        print ('Shape of the lambda sent att is {}'.format(attn_final_sent.shape))\n",
    "        attn_sents_for_all_classes.append(attn_final_sent)\n",
    "    x = Concatenate()(attn_sents_for_all_classes)\n",
    "    x = Dense(256, activation='relu')(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    #x = Dense(128, activation='relu')(x)\n",
    "    #x = Dropout(0.2)(x)\n",
    "    #x = Dense(64, activation='relu')(x)\n",
    "    #x = Dropout(0.2)(x)\n",
    "    x = Dense(64, activation='relu')(x)\n",
    "    preds = Dense(nclasses, activation='sigmoid')(x)\n",
    "    \n",
    "    model = Model(input, preds)\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This returns LSTM based model. There are 6 output classes, all soft sharing the parameters of a common network.\n",
    "\"\"\"\n",
    "def get_model_soft_sharing_lstm_singleoutput(emb_matrix, sentence_length, word_length, learning_rate=0.001, n_classes=1, decay=0.1, combined_model=False):\n",
    "    \n",
    "    rmsprop = optimizers.Adam(lr=learning_rate, clipnorm=0.1, clipvalue=0.05,decay=0.1)# \n",
    "    word_model = get_word_attention(emb_matrix, word_length, rmsprop, n_classes)\n",
    "    if not combined_model:\n",
    "        model = get_sentence_attention(word_model, word_length, sentence_length, n_classes)\n",
    "    else:\n",
    "        model = get_sentence_attention_combined_output(word_model, word_length, sentence_length, n_classes)\n",
    "    #model = attention_words_only(emb_matrix, word_length, 1)#sent_model\n",
    "    #model.add(Activation('softmax'))\n",
    "    #adam = optimizers.Adam(clipnorm=0.1,lr=learning_rate, clipvalue=0.05, decay=0.1)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=rmsprop,metrics=['accuracy'])\n",
    "    #model.compile(loss='mse', optimizer=adam,metrics=['accuracy'])\n",
    "\n",
    "    print (model.summary())\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Callbacks are passed to the model fit the `callbacks` argument in `fit`,\n",
    "# which takes a list of callbacks. You can pass any number of callbacks.\n",
    "callbacks_list = [\n",
    "    # This callback will interrupt training when we have stopped improving\n",
    "    keras.callbacks.EarlyStopping(\n",
    "        # This callback will monitor the validation accuracy of the model\n",
    "        monitor='val_loss',\n",
    "        # Training will be interrupted when the accuracy\n",
    "        # has stopped improving for *more* than 1 epochs (i.e. 2 epochs)\n",
    "        patience=10,\n",
    "    ),\n",
    "    # This callback will save the current weights after every epoch\n",
    "    keras.callbacks.ModelCheckpoint(\n",
    "        filepath='/Users/mayoor/dev/kaggle/tc/models/resnet_best_model_han_split.h5',  # Path to the destination model file\n",
    "        # The two arguments below mean that we will not overwrite the\n",
    "        # model file unless `val_loss` has improved, which\n",
    "        # allows us to keep the best model every seen during training.\n",
    "        monitor='val_loss',\n",
    "        save_best_only=True,\n",
    "    ),\n",
    "    \n",
    "       keras.callbacks.ReduceLROnPlateau(\n",
    "           # This callback will monitor the validation loss of the model\n",
    "           monitor='val_loss',\n",
    "           # It will divide the learning by 10 when it gets triggered\n",
    "           factor=0.1,\n",
    "           # It will get triggered after the validation loss has stopped improving\n",
    "           # for at least 10 epochs\n",
    "           patience=3,\n",
    ") ,\n",
    "\n",
    "    keras.callbacks.TensorBoard(\n",
    "        # Log files will be written at this location\n",
    "        log_dir='/Users/mayoor/dev/kaggle/tc/logs',\n",
    "        # We will record activation histograms every 1 epoch\n",
    "        histogram_freq=1\n",
    "        \n",
    ") \n",
    "\n",
    "\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "      <th>total_classes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>119879</th>\n",
       "      <td>811ed72c51830f42</td>\n",
       "      <td>REDIRECT Talk:John Loveday (experimental physi...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103694</th>\n",
       "      <td>2acc7c7d0386401f</td>\n",
       "      <td>Back it up. Post the line here with the refere...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131932</th>\n",
       "      <td>c1f95b89050a9ee4</td>\n",
       "      <td>I won't stop that. Sometimes Germanic equals G...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146867</th>\n",
       "      <td>32e8bdecfe1d66f0</td>\n",
       "      <td>\"\\n\\n British Bands?  \\n\\nI think you've mista...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121958</th>\n",
       "      <td>8c6c5e4228fb6ba8</td>\n",
       "      <td>You are WRONG. \\n\\nJustin Thompson is mentione...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      id                                       comment_text  \\\n",
       "119879  811ed72c51830f42  REDIRECT Talk:John Loveday (experimental physi...   \n",
       "103694  2acc7c7d0386401f  Back it up. Post the line here with the refere...   \n",
       "131932  c1f95b89050a9ee4  I won't stop that. Sometimes Germanic equals G...   \n",
       "146867  32e8bdecfe1d66f0  \"\\n\\n British Bands?  \\n\\nI think you've mista...   \n",
       "121958  8c6c5e4228fb6ba8  You are WRONG. \\n\\nJustin Thompson is mentione...   \n",
       "\n",
       "        toxic  severe_toxic  obscene  threat  insult  identity_hate  \\\n",
       "119879      0             0        0       0       0              0   \n",
       "103694      0             0        0       0       0              0   \n",
       "131932      1             0        0       0       0              0   \n",
       "146867      0             0        0       0       0              0   \n",
       "121958      0             0        0       0       0              0   \n",
       "\n",
       "        total_classes  \n",
       "119879              0  \n",
       "103694              0  \n",
       "131932              1  \n",
       "146867              0  \n",
       "121958              0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Use the X_train_* to create XTrains and YTrains."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(233222, 100)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_emb_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embedding:  (?, 10, 100)\n",
      "Shape after BD LSTM (?, ?, 100)\n",
      " input to sentence attn network <keras.engine.training.Model object at 0x122f90320>\n",
      "Shape after si_vects (?, 30, 10, 100)\n",
      "Shape after word vector (?, 30, 10, 100)\n",
      "The input shape is: (None, 10, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 10, 100)\n",
      "Norm shape (?, 1, 10)\n",
      "The input shape is: (None, 10, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 10, 100)\n",
      "Norm shape (?, 1, 10)\n",
      "The input shape is: (None, 10, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 10, 100)\n",
      "Norm shape (?, 1, 10)\n",
      "The input shape is: (None, 10, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 10, 100)\n",
      "Norm shape (?, 1, 10)\n",
      "The input shape is: (None, 10, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 10, 100)\n",
      "Norm shape (?, 1, 10)\n",
      "The input shape is: (None, 10, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 10, 100)\n",
      "Norm shape (?, 1, 10)\n",
      "ATTN Shape (?, 30, 10, 1)\n",
      "Multi word Shape (?, 30, 10, 100)\n",
      "Shape of the att1 is (?, 30, 100, 10)\n",
      "Shape of the lambda word is (?, 30, 100)\n",
      "Shape after BD LSTM (?, ?, 100)\n",
      "Shape after word vector (?, 30, 100)\n",
      "The input shape is: (None, 30, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 30, 100)\n",
      "Norm shape (?, 1, 30)\n",
      "Shape of the sent att is (?, 30, 1)\n",
      "Shape of the multi sent att is (?, 30, 100)\n",
      "Shape of the lambda sent att is (?, 100)\n",
      "Shape after BD LSTM (?, ?, 100)\n",
      "Shape after word vector (?, 30, 100)\n",
      "The input shape is: (None, 30, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 30, 100)\n",
      "Norm shape (?, 1, 30)\n",
      "Shape of the sent att is (?, 30, 1)\n",
      "Shape of the multi sent att is (?, 30, 100)\n",
      "Shape of the lambda sent att is (?, 100)\n",
      "Shape after BD LSTM (?, ?, 100)\n",
      "Shape after word vector (?, 30, 100)\n",
      "The input shape is: (None, 30, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 30, 100)\n",
      "Norm shape (?, 1, 30)\n",
      "Shape of the sent att is (?, 30, 1)\n",
      "Shape of the multi sent att is (?, 30, 100)\n",
      "Shape of the lambda sent att is (?, 100)\n",
      "Shape after BD LSTM (?, ?, 100)\n",
      "Shape after word vector (?, 30, 100)\n",
      "The input shape is: (None, 30, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 30, 100)\n",
      "Norm shape (?, 1, 30)\n",
      "Shape of the sent att is (?, 30, 1)\n",
      "Shape of the multi sent att is (?, 30, 100)\n",
      "Shape of the lambda sent att is (?, 100)\n",
      "Shape after BD LSTM (?, ?, 100)\n",
      "Shape after word vector (?, 30, 100)\n",
      "The input shape is: (None, 30, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 30, 100)\n",
      "Norm shape (?, 1, 30)\n",
      "Shape of the sent att is (?, 30, 1)\n",
      "Shape of the multi sent att is (?, 30, 100)\n",
      "Shape of the lambda sent att is (?, 100)\n",
      "Shape after BD LSTM (?, ?, 100)\n",
      "Shape after word vector (?, 30, 100)\n",
      "The input shape is: (None, 30, 100)\n",
      "kernel shape (100, 1)\n",
      "Input shape (?, 30, 100)\n",
      "Norm shape (?, 1, 30)\n",
      "Shape of the sent att is (?, 30, 1)\n",
      "Shape of the multi sent att is (?, 30, 100)\n",
      "Shape of the lambda sent att is (?, 100)\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 30, 10)       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 30, 10, 100)  23367500    input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_3 (TimeDistrib (None, 30, 10, 100)  10100       time_distributed_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_5 (TimeDistrib (None, 30, 10, 100)  400         time_distributed_3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_6 (TimeDistrib (None, 30, 10, 1)    100         time_distributed_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_7 (TimeDistrib (None, 30, 10, 1)    100         time_distributed_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_8 (TimeDistrib (None, 30, 10, 1)    100         time_distributed_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_9 (TimeDistrib (None, 30, 10, 1)    100         time_distributed_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_10 (TimeDistri (None, 30, 10, 1)    100         time_distributed_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_11 (TimeDistri (None, 30, 10, 1)    100         time_distributed_5[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "multiply_1 (Multiply)           (None, 30, 10, 100)  0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "multiply_2 (Multiply)           (None, 30, 10, 100)  0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_7[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "multiply_3 (Multiply)           (None, 30, 10, 100)  0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_8[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "multiply_4 (Multiply)           (None, 30, 10, 100)  0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_9[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "multiply_5 (Multiply)           (None, 30, 10, 100)  0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_10[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_6 (Multiply)           (None, 30, 10, 100)  0           time_distributed_1[0][0]         \n",
      "                                                                 time_distributed_11[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, 30, 100, 10)  0           multiply_1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_2 (Reshape)             (None, 30, 100, 10)  0           multiply_2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_3 (Reshape)             (None, 30, 100, 10)  0           multiply_3[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_4 (Reshape)             (None, 30, 100, 10)  0           multiply_4[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_5 (Reshape)             (None, 30, 100, 10)  0           multiply_5[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_6 (Reshape)             (None, 30, 100, 10)  0           multiply_6[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 30, 100)      0           reshape_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 30, 100)      0           reshape_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 30, 100)      0           reshape_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_4 (Lambda)               (None, 30, 100)      0           reshape_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_5 (Lambda)               (None, 30, 100)      0           reshape_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_6 (Lambda)               (None, 30, 100)      0           reshape_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) (None, 30, 100)      45300       lambda_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_3 (Bidirectional) (None, 30, 100)      45300       lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_4 (Bidirectional) (None, 30, 100)      45300       lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_5 (Bidirectional) (None, 30, 100)      45300       lambda_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_6 (Bidirectional) (None, 30, 100)      45300       lambda_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_7 (Bidirectional) (None, 30, 100)      45300       lambda_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_12 (TimeDistri (None, 30, 100)      10100       bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_13 (TimeDistri (None, 30, 100)      10100       bidirectional_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_14 (TimeDistri (None, 30, 100)      10100       bidirectional_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_15 (TimeDistri (None, 30, 100)      10100       bidirectional_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_16 (TimeDistri (None, 30, 100)      10100       bidirectional_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_17 (TimeDistri (None, 30, 100)      10100       bidirectional_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "attnword_7 (ATTNWORD)           (None, 30, 1)        100         time_distributed_12[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "attnword_8 (ATTNWORD)           (None, 30, 1)        100         time_distributed_13[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "attnword_9 (ATTNWORD)           (None, 30, 1)        100         time_distributed_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "attnword_10 (ATTNWORD)          (None, 30, 1)        100         time_distributed_15[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "attnword_11 (ATTNWORD)          (None, 30, 1)        100         time_distributed_16[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "attnword_12 (ATTNWORD)          (None, 30, 1)        100         time_distributed_17[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "multiply_7 (Multiply)           (None, 30, 100)      0           bidirectional_2[0][0]            \n",
      "                                                                 attnword_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_8 (Multiply)           (None, 30, 100)      0           bidirectional_3[0][0]            \n",
      "                                                                 attnword_8[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_9 (Multiply)           (None, 30, 100)      0           bidirectional_4[0][0]            \n",
      "                                                                 attnword_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "multiply_10 (Multiply)          (None, 30, 100)      0           bidirectional_5[0][0]            \n",
      "                                                                 attnword_10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multiply_11 (Multiply)          (None, 30, 100)      0           bidirectional_6[0][0]            \n",
      "                                                                 attnword_11[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "multiply_12 (Multiply)          (None, 30, 100)      0           bidirectional_7[0][0]            \n",
      "                                                                 attnword_12[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_7 (Reshape)             (None, 100, 30)      0           multiply_7[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_8 (Reshape)             (None, 100, 30)      0           multiply_8[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_9 (Reshape)             (None, 100, 30)      0           multiply_9[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "reshape_10 (Reshape)            (None, 100, 30)      0           multiply_10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_11 (Reshape)            (None, 100, 30)      0           multiply_11[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "reshape_12 (Reshape)            (None, 100, 30)      0           multiply_12[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lambda_7 (Lambda)               (None, 100)          0           reshape_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_8 (Lambda)               (None, 100)          0           reshape_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_9 (Lambda)               (None, 100)          0           reshape_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lambda_10 (Lambda)              (None, 100)          0           reshape_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_11 (Lambda)              (None, 100)          0           reshape_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "lambda_12 (Lambda)              (None, 100)          0           reshape_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1)            101         lambda_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            101         lambda_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 1)            101         lambda_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 1)            101         lambda_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 1)            101         lambda_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 1)            101         lambda_12[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 23,712,206\n",
      "Trainable params: 23,712,006\n",
      "Non-trainable params: 200\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = get_model_soft_sharing_lstm_singleoutput(final_emb_matrix, 30, 10, learning_rate=0.01, n_classes=6, decay=0.1, combined_model=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_model(model,to_file='attn_model_multi_rework_sent_allclasses_bn.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is a great day !']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tu.sentence_tokenizer(\"This is a great day!\", 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lookup_words = tu.get_word_reverse_lookup(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "comment_list = train.astype(str)[comment_col].tolist()\n",
    "xtrain = tu.pad_sentences_sent(comment_list,30,10, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#ct = tu.sent_counter(comment_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#pd.Series(ct).describe(percentiles=[.10,.20,.30,.40,.50,.60,.70,.8,.90,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[65247, 86, 65247, 28, 139, 518, 11, 411, 65247]]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#plt.hist(pd.Series(ct))\n",
    "tokenizer.word_index['unk']\n",
    "tokenizer.texts_to_sequences([tu.replace_unknown_words_with_UNK(\"I am hosbnahf? where everyone is considered asdkfjsla\", tokenizer)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[    0     0     0     0  5284    16    16    16    16    16]\n",
      " [    0     0     0    18   190    11  5284     8 47785    16]\n",
      " [15195 32761    11    48    10 13330   399    71   375     3]\n",
      " [ 7029     8  2370   106    45   312   236     7    55     2]\n",
      " [    0    35  1504    22    18  1250     7    10   307     1]\n",
      " [   90     3   387    90     3    27    41    29   830    49]\n",
      " [ 7076 75701     8    26   121    42 13465  2129     8 20245]\n",
      " [    0     0     0     0     0     0 11530    28    28    28]\n",
      " [   31    55     7     9    26     4  4915     2  1112     8]\n",
      " [10688     7     2   818     7   230   725    64    82     9]\n",
      " [  134     4   227  2863     8   375    49  7407    28    28]\n",
      " [   27    55     7     9   471 75702 10127  1251    15 11374]\n",
      " [41466    32  6413 10883     3  1659     7     2   343     7]\n",
      " [    0     0     0     0     0     0  8132     8  1112    28]\n",
      " [    0     0     0     0     0     0     0     0 12745     1]\n",
      " [   18 15195 32761 33436  1885    39   219    24  2934  1607]\n",
      " [    0     0     0     0     0     8   780    17   158     1]\n",
      " [ 1017   244    34   938    69   750    17     2    41   587]\n",
      " [  610  1041     8 18295    73  4493    41  9137     4   110]\n",
      " [    0     0     0     0     0     0     0    10   320     1]\n",
      " [   31    34     9    85    26  3955     3    37    31     9]\n",
      " [   64   814    32  4446    75   225     9   136  1544    32]\n",
      " [    0     0     8 47786     2   818     7   230  1235    28]\n",
      " [    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0]\n",
      " [    0     0     0     0     0     0     0     0     0     0]]\n"
     ]
    }
   ],
   "source": [
    "print(xtrain[0][:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15958, 30, 10)\n"
     ]
    }
   ],
   "source": [
    "test_comment_list = test.astype(str)[comment_col].tolist()\n",
    "xval = tu.pad_sentences_sent(test_comment_list,30,10, tokenizer)\n",
    "print (xval.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_label_stat(y):\n",
    "    #y = y.tolist()\n",
    "    total_count = pd.Series(y).count()\n",
    "    y1 = (pd.Series(y).sum()/total_count)*100\n",
    "    y0 = 100-y1\n",
    "    return total_count, y1, y0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'> <class 'numpy.ndarray'> <class 'list'> <class 'list'>\n"
     ]
    }
   ],
   "source": [
    "print (type(xtrain), type(xval), type(YTrain), type(YVal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training State - Total Records: 143613, Toxic percent: 9.618906366415295, Normal percent: 90.3810936335847\n",
      "Validation State - Total Records: 15958, Toxic percent: 9.274345156034592, Normal percent: 90.7256548439654\n"
     ]
    }
   ],
   "source": [
    "total_count_train, y1, y0 = get_label_stat(YTrain[0])\n",
    "print ('Training State - Total Records: {0}, Toxic percent: {1}, Normal percent: {2}'.format(total_count_train, y1, y0))\n",
    "total_count_val, y1, y0 = get_label_stat(YVal[0])\n",
    "print ('Validation State - Total Records: {0}, Toxic percent: {1}, Normal percent: {2}'.format(total_count_val, y1, y0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 143613 samples, validate on 15958 samples\n",
      "Epoch 1/5\n",
      "143613/143613 [==============================] - 3488s 24ms/step - loss: 0.3988 - dense_3_loss: 0.1378 - dense_5_loss: 0.0341 - dense_7_loss: 0.0783 - dense_9_loss: 0.0200 - dense_11_loss: 0.0892 - dense_13_loss: 0.0394 - dense_3_acc: 0.9509 - dense_5_acc: 0.9898 - dense_7_acc: 0.9742 - dense_9_acc: 0.9966 - dense_11_acc: 0.9669 - dense_13_acc: 0.9909 - val_loss: 0.3299 - val_dense_3_loss: 0.1105 - val_dense_5_loss: 0.0289 - val_dense_7_loss: 0.0635 - val_dense_9_loss: 0.0139 - val_dense_11_loss: 0.0761 - val_dense_13_loss: 0.0369 - val_dense_3_acc: 0.9596 - val_dense_5_acc: 0.9905 - val_dense_7_acc: 0.9774 - val_dense_9_acc: 0.9976 - val_dense_11_acc: 0.9703 - val_dense_13_acc: 0.9905\n",
      "Epoch 2/5\n",
      "143613/143613 [==============================] - 6521s 45ms/step - loss: 0.3349 - dense_3_loss: 0.1156 - dense_5_loss: 0.0270 - dense_7_loss: 0.0649 - dense_9_loss: 0.0175 - dense_11_loss: 0.0770 - dense_13_loss: 0.0328 - dense_3_acc: 0.9577 - dense_5_acc: 0.9901 - dense_7_acc: 0.9781 - dense_9_acc: 0.9969 - dense_11_acc: 0.9700 - dense_13_acc: 0.9911 - val_loss: 0.3194 - val_dense_3_loss: 0.1061 - val_dense_5_loss: 0.0276 - val_dense_7_loss: 0.0620 - val_dense_9_loss: 0.0133 - val_dense_11_loss: 0.0740 - val_dense_13_loss: 0.0364 - val_dense_3_acc: 0.9615 - val_dense_5_acc: 0.9905 - val_dense_7_acc: 0.9785 - val_dense_9_acc: 0.9977 - val_dense_11_acc: 0.9705 - val_dense_13_acc: 0.9905\n",
      "Epoch 3/5\n",
      "143613/143613 [==============================] - 3638s 25ms/step - loss: 0.3268 - dense_3_loss: 0.1120 - dense_5_loss: 0.0262 - dense_7_loss: 0.0636 - dense_9_loss: 0.0170 - dense_11_loss: 0.0758 - dense_13_loss: 0.0322 - dense_3_acc: 0.9586 - dense_5_acc: 0.9901 - dense_7_acc: 0.9787 - dense_9_acc: 0.9969 - dense_11_acc: 0.9703 - dense_13_acc: 0.9911 - val_loss: 0.3141 - val_dense_3_loss: 0.1044 - val_dense_5_loss: 0.0270 - val_dense_7_loss: 0.0609 - val_dense_9_loss: 0.0130 - val_dense_11_loss: 0.0730 - val_dense_13_loss: 0.0359 - val_dense_3_acc: 0.9617 - val_dense_5_acc: 0.9905 - val_dense_7_acc: 0.9787 - val_dense_9_acc: 0.9977 - val_dense_11_acc: 0.9706 - val_dense_13_acc: 0.9905\n",
      "Epoch 4/5\n",
      " 95808/143613 [===================>..........] - ETA: 16:26 - loss: 0.3230 - dense_3_loss: 0.1102 - dense_5_loss: 0.0260 - dense_7_loss: 0.0626 - dense_9_loss: 0.0174 - dense_11_loss: 0.0751 - dense_13_loss: 0.0317 - dense_3_acc: 0.9591 - dense_5_acc: 0.9901 - dense_7_acc: 0.9786 - dense_9_acc: 0.9968 - dense_11_acc: 0.9707 - dense_13_acc: 0.9913"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-a56590a254e1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxtrain\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mYTrain\u001b[0m \u001b[0;34m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mYVal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#, callbacks=callbacks_list\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1667\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1668\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1669\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1670\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1671\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1204\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1206\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1207\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1208\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2473\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2474\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2475\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2476\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    790\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    995\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    998\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1133\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1137\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1140\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1121\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(xtrain,YTrain ,batch_size=64, epochs=5, verbose=1, validation_data=(xval, YVal), shuffle=True, callbacks=callbacks_list)#, callbacks=callbacks_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#model.load_weights('/Users/mayoor/dev/kaggle/tc/models/resnet_best_model_han_split.h5')\n",
    "xt =  tu.pad_sentences_sent(df[comment_col].astype(str).tolist(),30,10, tokenizer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "yt = ys(df, pred_cols)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "159571/159571 [==============================] - 258s 2ms/step\n",
      "\n",
      "Test score: 0.10400655906169301, accuracy: 0.9608826165196411\n"
     ]
    }
   ],
   "source": [
    "v_score, v_acc = model.evaluate(xt, yt, batch_size=128)\n",
    "print(\"\\nTest score: {0}, accuracy: {1}\".format(v_score, v_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                      id                                       comment_text  \\\n",
      "25255   42d923c91158e044  no worries \\n\\nWe will use another account, an...   \n",
      "12791   21ec8eeb0890003d  hi kannadiga shame on you \\n\\nYou look like a ...   \n",
      "138234  e3b8bc8940f2a818  Oaks and Homophobia\\nI provided a full citatio...   \n",
      "201     007f1839ada915e6  Your blatant POV pushing \\n\\nNeither of you gu...   \n",
      "55184   936476a4c9b51863  \\nThe warning is such bullshit. I dont give a ...   \n",
      "\n",
      "        toxic  severe_toxic  obscene  threat  insult  identity_hate  \\\n",
      "25255       1             0        0       0       0              1   \n",
      "12791       1             0        0       0       1              0   \n",
      "138234      1             0        1       0       0              0   \n",
      "201         1             0        1       0       0              0   \n",
      "55184       1             0        1       0       0              0   \n",
      "\n",
      "        total_classes  \n",
      "25255               2  \n",
      "12791               2  \n",
      "138234              2  \n",
      "201                 2  \n",
      "55184               2  \n",
      "['\\nThe warning is such bullshit. I dont give a fuck if you log my IP, all I did was go to a website, which is NOT a crime. They can kiss my ass. ...  ']\n",
      "                     id                                       comment_text  \\\n",
      "55184  936476a4c9b51863  \\nThe warning is such bullshit. I dont give a ...   \n",
      "\n",
      "       toxic  severe_toxic  obscene  threat  insult  identity_hate  \\\n",
      "55184      1             0        1       0       0              0   \n",
      "\n",
      "       total_classes  \n",
      "55184              2  \n"
     ]
    }
   ],
   "source": [
    "print(test[test['total_classes']==2].head())\n",
    "print(test[test[\"id\"]==\"936476a4c9b51863\"][comment_col].tolist())\n",
    "#print(test[test[\"id\"]==\"b0c9e9304f37c9d3\"]['comment_text'].tolist())\n",
    "#print( pad_sentences(test[test[\"id\"]==\"b0c9e9304f37c9d3\"]['comment_text'], 25, 25).shape)\n",
    "#print(model.predict( pad_sentences(test[test[\"id\"]==\"936476a4c9b51863\"]['comment_text'], 12, 25)))\n",
    "print(test[test[\"id\"]==\"936476a4c9b51863\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 43, 10)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sentence = comment_list[0]\n",
    "#print (sample_sentence)\n",
    "imagined_sample = \"fuck you.  I will come there. what a mess. you son a of bitch. This is a test \"\n",
    "out_of_sample = df[df[\"id\"]==\"936476a4c9b51863\"][comment_col].tolist()[0]\n",
    "#model.predict( tu.pad_sentences_sent([sample_sentence, imagined_sample, out_of_sample], 30,10, tokenizer))\n",
    "tu.pad_sentences_sent([sample_sentence], 30,10, tokenizer).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nclasses = 6\n",
    "word_output = [model.layers[i+1].layer.layers[-1] for i in range(nclasses)]\n",
    "sent_output = [model.layers[-(nclasses*2)+i] for i in range(nclasses)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calculate_attention(sentence, sent, words):\n",
    "    input_sent = tu.get_padded_words([sentence], words)\n",
    "    print (input_sent.shape)\n",
    "    model_word_hit_model = Model(model.inputs[0],model.layers[-3].output)\n",
    "    model_word_hit = model_word_hit_model.predict(input_sent)\n",
    "    attn_kernel_weight = model.layers[-2].get_weights()[0]\n",
    "    print (attn_kernel_weight.shape)\n",
    "    product = np.dot(model_word_hit, attn_kernel_weight)\n",
    "    product = np.reshape(product, (-1, 1, 300))\n",
    "    x_norm = np.exp(product)/np.sum(np.exp(product))\n",
    "    return x_norm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'calculate_attention' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-7422c9baf5c0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mattn_ws\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalculate_attention\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"fuck you.  I will come there. what a mess. you son a of bitch. This is a test \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m300\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'calculate_attention' is not defined"
     ]
    }
   ],
   "source": [
    "attn_ws = calculate_attention(\"fuck you.  I will come there. what a mess. you son a of bitch. This is a test \", 1,300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 1, 300)\n",
      "1.0000002\n"
     ]
    }
   ],
   "source": [
    "print (attn_ws.shape)\n",
    "print (np.sum(attn_ws))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.engine.topology.InputLayer at 0x1231183c8>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x123046358>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x122f8a7f0>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x1239b9b38>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x1239b9668>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x11f42b2b0>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x1239a6828>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x123934b38>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x123917b00>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x1238dc4e0>,\n",
       " <keras.layers.merge.Multiply at 0x1238b8ac8>,\n",
       " <keras.layers.merge.Multiply at 0x123919ac8>,\n",
       " <keras.layers.merge.Multiply at 0x1238d7400>,\n",
       " <keras.layers.merge.Multiply at 0x1238cf630>,\n",
       " <keras.layers.merge.Multiply at 0x1238af160>,\n",
       " <keras.layers.merge.Multiply at 0x1238af0f0>,\n",
       " <keras.layers.core.Reshape at 0x1238ae0b8>,\n",
       " <keras.layers.core.Reshape at 0x1238ae588>,\n",
       " <keras.layers.core.Reshape at 0x1238ae320>,\n",
       " <keras.layers.core.Reshape at 0x1238aa860>,\n",
       " <keras.layers.core.Reshape at 0x1238aa208>,\n",
       " <keras.layers.core.Reshape at 0x1238aa4a8>,\n",
       " <keras.layers.core.Lambda at 0x1238aa7f0>,\n",
       " <keras.layers.core.Lambda at 0x1238a9358>,\n",
       " <keras.layers.core.Lambda at 0x1238ac7b8>,\n",
       " <keras.layers.core.Lambda at 0x1238a6518>,\n",
       " <keras.layers.core.Lambda at 0x1238a9470>,\n",
       " <keras.layers.core.Lambda at 0x1238a3ac8>,\n",
       " <keras.layers.wrappers.Bidirectional at 0x123896080>,\n",
       " <keras.layers.wrappers.Bidirectional at 0x11ff57f28>,\n",
       " <keras.layers.wrappers.Bidirectional at 0x11fc35128>,\n",
       " <keras.layers.wrappers.Bidirectional at 0x122e0e550>,\n",
       " <keras.layers.wrappers.Bidirectional at 0x19ae07128>,\n",
       " <keras.layers.wrappers.Bidirectional at 0x19bbc5a20>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x12388f828>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x11ff43240>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x11fc2fda0>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x122e03da0>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19ae1e2e8>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19bbe8ac8>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x123689d68>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x11fd99438>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x11fac20f0>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x19a5cbac8>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x19b3cccc0>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x19c296c18>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x1236896d8>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x11fd99048>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x11fa860b8>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19a6991d0>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19b495f60>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19c41deb8>,\n",
       " <__main__.ATTNWORD at 0x12367c278>,\n",
       " <__main__.ATTNWORD at 0x11fd8f7f0>,\n",
       " <__main__.ATTNWORD at 0x11fa6f208>,\n",
       " <__main__.ATTNWORD at 0x19a61bfd0>,\n",
       " <__main__.ATTNWORD at 0x19b4d54e0>,\n",
       " <__main__.ATTNWORD at 0x19c45b6d8>,\n",
       " <keras.layers.merge.Multiply at 0x12366f2e8>,\n",
       " <keras.layers.merge.Multiply at 0x11fd622e8>,\n",
       " <keras.layers.merge.Multiply at 0x11fa694a8>,\n",
       " <keras.layers.merge.Multiply at 0x19a733e48>,\n",
       " <keras.layers.merge.Multiply at 0x19b4c94e0>,\n",
       " <keras.layers.merge.Multiply at 0x19c58be48>,\n",
       " <keras.layers.core.Reshape at 0x12362d240>,\n",
       " <keras.layers.core.Reshape at 0x11fd3b0b8>,\n",
       " <keras.layers.core.Reshape at 0x11fa51358>,\n",
       " <keras.layers.core.Reshape at 0x19aa1f630>,\n",
       " <keras.layers.core.Reshape at 0x19b4ed470>,\n",
       " <keras.layers.core.Reshape at 0x19c57ee48>,\n",
       " <keras.layers.core.Lambda at 0x12361d278>,\n",
       " <keras.layers.core.Lambda at 0x11fd42cc0>,\n",
       " <keras.layers.core.Lambda at 0x11fabf128>,\n",
       " <keras.layers.core.Lambda at 0x19a6b2630>,\n",
       " <keras.layers.core.Lambda at 0x19b6dfdd8>,\n",
       " <keras.layers.core.Lambda at 0x19c57e7b8>,\n",
       " <keras.layers.core.Dense at 0x1200032e8>,\n",
       " <keras.layers.core.Dense at 0x11fd32668>,\n",
       " <keras.layers.core.Dense at 0x122ea86d8>,\n",
       " <keras.layers.core.Dense at 0x19a7af550>,\n",
       " <keras.layers.core.Dense at 0x19b93ec50>,\n",
       " <keras.layers.core.Dense at 0x19c42b1d0>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x120002400>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x11fd30ac8>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x122ea85c0>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19aa73c88>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19b93efd0>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19c5d04a8>,\n",
       " <keras.layers.core.Dropout at 0x1200020f0>,\n",
       " <keras.layers.core.Dropout at 0x11fd28fd0>,\n",
       " <keras.layers.core.Dropout at 0x122eab6a0>,\n",
       " <keras.layers.core.Dropout at 0x19aa73710>,\n",
       " <keras.layers.core.Dropout at 0x19b955dd8>,\n",
       " <keras.layers.core.Dropout at 0x19c5f63c8>,\n",
       " <keras.layers.core.Dense at 0x11fffa748>,\n",
       " <keras.layers.core.Dense at 0x11fd27cf8>,\n",
       " <keras.layers.core.Dense at 0x122e8df60>,\n",
       " <keras.layers.core.Dense at 0x19aa8ad30>,\n",
       " <keras.layers.core.Dense at 0x19b96def0>,\n",
       " <keras.layers.core.Dense at 0x19c601208>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x11ffaa320>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x11fd16b00>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x122e768d0>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19aad0e10>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19b99cf60>,\n",
       " <keras.layers.normalization.BatchNormalization at 0x19c623e10>,\n",
       " <keras.layers.core.Dense at 0x11ff91080>,\n",
       " <keras.layers.core.Dense at 0x11fcc5b00>,\n",
       " <keras.layers.core.Dense at 0x122e3fa20>,\n",
       " <keras.layers.core.Dense at 0x19abacb70>,\n",
       " <keras.layers.core.Dense at 0x19ba854a8>,\n",
       " <keras.layers.core.Dense at 0x19c80e748>]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#word_attns, sent_attns\n",
    "def get_sentence_rank(sentence, sent, words, tokenizer, class_label=0):\n",
    "    #test[test[\"id\"]==\"936476a4c9b51863\"][comment_col]\n",
    "    input_sent = tu.pad_sentences_sent([sentence], sent, words, tokenizer) #get_padded_words([sentence], words)\n",
    "    model_word_attn = Model(model.inputs[0],model.layers[-5].output)#[wa.output for wa in word_output])\n",
    "    weights = model_word_attn.predict(input_sent)\n",
    "    print (np.sum(weights))\n",
    "    sentences_rank = np.argsort(weights,axis=1).flatten()[::-1]\n",
    "    print (sentences_rank)\n",
    "    return sentences_rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4231643\n",
      "[0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_sentence_rank(\"fuck you.  I will come there. what a mess. you son a of bitch. This is a test \", 6,50, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_rank(sentence, sent, words):\n",
    "    attn_ws = calculate_attention(sentence, sent, words)\n",
    "    return np.argsort(attn_ws,axis=2).flatten()[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word_rank(sentence, sent_len, word_len):\n",
    "    input_sent = pad_sentences_sent([sentence], sent_len, word_len)\n",
    "    model_word_attn = Model(model.inputs[0],model.layers[-11].output)#[wa.output for wa in word_output])\n",
    "    attention_output = model_word_attn.predict(input_sent)\n",
    "    output = attention_output.reshape(sent_len,word_len)\n",
    "    return np.argsort(output,axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def top_ranked_words_for_sentence(sentence, sent_len, word_len, lookup_words, nwords=-1, nsents=-1):\n",
    "    sent_rank = get_sentence_rank(sentence, sent_len, word_len)\n",
    "    processed_sent = tu.readable_pad_sent(sentence, sent_len, word_len, lookup_words)\n",
    "    word_level_rank = word_rank(sentence, sent_len, word_len)\n",
    "    print (\"word_level_rank>>\",word_level_rank.shape)\n",
    "    if nwords == -1:\n",
    "        nwords = word_len\n",
    "    if nsents == -1:\n",
    "        nsents = sent_len\n",
    "    \n",
    "    ranked_data = []\n",
    "    for i in range(nsents):\n",
    "        s = processed_sent[sent_rank[i]]\n",
    "        wrank = word_level_rank[sent_rank[i]]\n",
    "        wrank = wrank.flatten()[::-1]\n",
    "        ordered_words = [s[w] for w in wrank]\n",
    "        ordered_words = ordered_words[:nwords]\n",
    "        ranked_data.append(ordered_words)\n",
    "    return ranked_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def top_ranked_words(sentence, sent_len, word_len, lookup_words, nwords=-1, nsents=-1):\n",
    "    w_rank = get_rank(sentence, sent_len, word_len)\n",
    "    print (\"$$\",w_rank)\n",
    "    processed_sent = tu.readable_pad_sent(sentence, sent_len, word_len, lookup_words)\n",
    "    #word_level_rank = word_rank(sentence, sent_len, word_len)\n",
    "    if nwords == -1:\n",
    "        nwords = word_len\n",
    "    \n",
    "    ranked_data = []\n",
    "    s = processed_sent[0]\n",
    "    print (\">>\", s)\n",
    "\n",
    "    ordered_words = [s[w] for w in w_rank]\n",
    "    ordered_words = ordered_words[:nwords]\n",
    "    ranked_data.append(ordered_words)\n",
    "    return ranked_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.99999994\n",
      "[0 1 2 3 4 5]\n",
      "[[124, 6, 7, 44, 271, 41, 39, 5, 1651, 6, 1387, 5, 3, 636, 13, 8, 5, 706], [], [], [], [], []]\n",
      "word_level_rank>> (6, 50)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['bitch', 'fuck', 'you', 'a', 'of', 'son', 'mess', 'a', 'you', 'what'],\n",
       " [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '],\n",
       " [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '],\n",
       " [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '],\n",
       " [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '],\n",
       " [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ']]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sentence = comment_list[0]\n",
    "imagined_sample = \"fuck you.  I will come there. what a mess. you son a of bitch. This is a test \"\n",
    "#top_ranked_words(imagined_sample, 1, 300, nwords=50, nsents=1)\n",
    "top_ranked_words_for_sentence(imagined_sample, 6, 50, nwords=10, nsents=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The warning is such bullshit. I dont give a fuck if you log my IP, all I did was go to a website, which is NOT a crime. They can kiss my ass. ...  \n",
      "1.0\n",
      "[3 2 1 0 4 5]\n",
      "[[5367, 13, 173, 8, 5367, 4, 1485, 157, 14888, 32097, 8, 36, 5, 13136, 393, 59, 369, 7098, 4, 2398, 93, 34, 309, 3, 42, 1, 23, 879, 15, 13, 1253, 3, 5, 292, 77, 375, 77, 19, 453, 796], [37, 7140, 5, 13495, 4, 18, 107, 30, 13277, 2061, 4, 18547, 10449, 33, 42, 3, 6, 18, 2, 5048, 1, 1127, 4, 10498, 3, 1, 838, 3, 196, 718, 51, 70, 6, 121, 2, 212, 2790, 4, 369, 37, 7515, 19, 42, 3, 6, 474], [70466, 10281, 1284, 10, 10329, 40251, 21, 6278, 10838, 1668, 3, 1, 338, 3, 7699, 4, 1127, 12603, 13, 14888, 32097, 32748, 1769, 27, 202, 16, 2871, 1626, 4, 798, 12, 141, 1072, 207, 969, 56, 767, 12, 1, 28, 606, 617, 1054, 4, 18304, 62], [4280, 28, 8460, 2, 96, 5, 299, 54, 6, 76, 18, 3850, 25, 33, 6, 51, 813, 21, 4477, 64, 211, 168, 1532, 21, 4, 46053, 1, 838, 3, 196, 1162], [], []]\n",
      "word_level_rank>> (6, 50)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['a',\n",
       "  'get',\n",
       "  'you',\n",
       "  'life',\n",
       "  'freak',\n",
       "  'people',\n",
       "  'yourselves',\n",
       "  'you',\n",
       "  'dont',\n",
       "  'jobs',\n",
       "  'youre',\n",
       "  'do',\n",
       "  'to',\n",
       "  'unions',\n",
       "  'wikipedia',\n",
       "  'have',\n",
       "  'or',\n",
       "  'just',\n",
       "  'and',\n",
       "  'with',\n",
       "  'us',\n",
       "  'tormenting',\n",
       "  'contributors',\n",
       "  'while',\n",
       "  'play',\n",
       "  'here',\n",
       "  'of',\n",
       "  'with',\n",
       "  'playing',\n",
       "  'rest',\n",
       "  'the',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' '],\n",
       " ['moron',\n",
       "  'afraid',\n",
       "  'wikipedia',\n",
       "  'administrator',\n",
       "  'of',\n",
       "  'the',\n",
       "  'glasses',\n",
       "  'other',\n",
       "  'sheesh',\n",
       "  'and',\n",
       "  'fun',\n",
       "  'thought',\n",
       "  'thick',\n",
       "  'assorted',\n",
       "  'of',\n",
       "  'romance',\n",
       "  'with',\n",
       "  'the',\n",
       "  'longer',\n",
       "  'for',\n",
       "  'cant',\n",
       "  'in',\n",
       "  'and',\n",
       "  'any',\n",
       "  'wait',\n",
       "  'this',\n",
       "  'america',\n",
       "  'club',\n",
       "  'be',\n",
       "  'librarian',\n",
       "  'work',\n",
       "  'types',\n",
       "  'wool',\n",
       "  'sweaters',\n",
       "  'better',\n",
       "  'approved',\n",
       "  'and',\n",
       "  'for',\n",
       "  'lauren',\n",
       "  'quickly',\n",
       "  'marm',\n",
       "  'available',\n",
       "  'page',\n",
       "  'caitlin',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  'upton',\n",
       "  'bio'],\n",
       " ['freaking',\n",
       "  'a',\n",
       "  'so',\n",
       "  'you',\n",
       "  'retentive',\n",
       "  'you',\n",
       "  'ruin',\n",
       "  'to',\n",
       "  'inferiority',\n",
       "  'you',\n",
       "  'do',\n",
       "  'an',\n",
       "  'egos',\n",
       "  'of',\n",
       "  'all',\n",
       "  'school',\n",
       "  'have',\n",
       "  'so',\n",
       "  'fragile',\n",
       "  'such',\n",
       "  'of',\n",
       "  'all',\n",
       "  'have',\n",
       "  'desperately',\n",
       "  'are',\n",
       "  'and',\n",
       "  'because',\n",
       "  'just',\n",
       "  'feel',\n",
       "  'and',\n",
       "  'us',\n",
       "  'to',\n",
       "  'members',\n",
       "  'need',\n",
       "  'fun',\n",
       "  'powerful',\n",
       "  'of',\n",
       "  'important',\n",
       "  'of',\n",
       "  'rest',\n",
       "  'the',\n",
       "  'and',\n",
       "  'and',\n",
       "  'enthusiasm',\n",
       "  'the',\n",
       "  'complex',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ',\n",
       "  ' ']]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_train = df[df[\"id\"]==\"936476a4c9b51863\"][comment_col].values[0]\n",
    "print (sample_train)\n",
    "top_ranked_words_for_sentence(sample_sentence, 6, 50, nwords=50, nsents=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentence_readable = tu.readable_pad_sent(test[test[\"id\"]==\"936476a4c9b51863\"][comment_col].tolist()[0], 50, 30)\n",
    "print(sentences_rank)\n",
    "#[sentence_readable[i] for i in sentences_rank[:10]]\n",
    "print (sentence_readable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00001cee341fdb12</td>\n",
       "      <td>yo bitch ja rule is more succesful then youll ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000247867823ef7</td>\n",
       "      <td>from rfc the title is fine as it is imo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00013b17ad220c46</td>\n",
       "      <td>sources unk ashton on lapland —</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00017563c3f7919a</td>\n",
       "      <td>if you have a look back at the source the info...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00017695ad8997eb</td>\n",
       "      <td>i dont anonymously edit articles at all</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text\n",
       "0  00001cee341fdb12  yo bitch ja rule is more succesful then youll ...\n",
       "1  0000247867823ef7            from rfc the title is fine as it is imo\n",
       "2  00013b17ad220c46                    sources unk ashton on lapland —\n",
       "3  00017563c3f7919a  if you have a look back at the source the info...\n",
       "4  00017695ad8997eb            i dont anonymously edit articles at all"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('test.csv')\n",
    "test_df = tu.clean_up(test_df)\n",
    "test_df['comment_text'] = test_df['comment_text'].apply(lambda x: tu.replace_unknown_words_with_UNK(x, tokenizer))\n",
    "test_comments = test_df.astype(str)['comment_text'].tolist()\n",
    "xtrain = tu.pad_sentences_sent(test_comments,6,50, tokenizer)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "178415"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index['unk']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(xtrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00001cee341fdb12</td>\n",
       "      <td>0.939705</td>\n",
       "      <td>0.939705</td>\n",
       "      <td>0.939705</td>\n",
       "      <td>0.939705</td>\n",
       "      <td>0.939705</td>\n",
       "      <td>0.939705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000247867823ef7</td>\n",
       "      <td>0.001127</td>\n",
       "      <td>0.001127</td>\n",
       "      <td>0.001127</td>\n",
       "      <td>0.001127</td>\n",
       "      <td>0.001127</td>\n",
       "      <td>0.001127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00013b17ad220c46</td>\n",
       "      <td>0.002598</td>\n",
       "      <td>0.002598</td>\n",
       "      <td>0.002598</td>\n",
       "      <td>0.002598</td>\n",
       "      <td>0.002598</td>\n",
       "      <td>0.002598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00017563c3f7919a</td>\n",
       "      <td>0.004476</td>\n",
       "      <td>0.004476</td>\n",
       "      <td>0.004476</td>\n",
       "      <td>0.004476</td>\n",
       "      <td>0.004476</td>\n",
       "      <td>0.004476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00017695ad8997eb</td>\n",
       "      <td>0.002830</td>\n",
       "      <td>0.002830</td>\n",
       "      <td>0.002830</td>\n",
       "      <td>0.002830</td>\n",
       "      <td>0.002830</td>\n",
       "      <td>0.002830</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id     toxic  severe_toxic   obscene    threat    insult  \\\n",
       "0  00001cee341fdb12  0.939705      0.939705  0.939705  0.939705  0.939705   \n",
       "1  0000247867823ef7  0.001127      0.001127  0.001127  0.001127  0.001127   \n",
       "2  00013b17ad220c46  0.002598      0.002598  0.002598  0.002598  0.002598   \n",
       "3  00017563c3f7919a  0.004476      0.004476  0.004476  0.004476  0.004476   \n",
       "4  00017695ad8997eb  0.002830      0.002830  0.002830  0.002830  0.002830   \n",
       "\n",
       "   identity_hate  \n",
       "0       0.939705  \n",
       "1       0.001127  \n",
       "2       0.002598  \n",
       "3       0.004476  \n",
       "4       0.002830  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_df = pd.DataFrame(columns=['id','toxic','severe_toxic','obscene','threat','insult','identity_hate'])\n",
    "predicted_df['id'] = test_df['id']\n",
    "for i, k in enumerate(pred_cols):\n",
    "    predicted_df[k] = predictions[:]\n",
    "predicted_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predicted_df.to_csv('first_submission_h_n_consolidated.csv',index=False, header=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}