TC-MultiNetwork.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline  \n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras.layers import Bidirectional, Input, LSTM, Dense, Activation, Conv1D, Flatten, Embedding, MaxPooling1D, Dropout\n",
    "#Concatenate keras.layers.embeddings import Embedding\n",
    "from keras.layers import Concatenate\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras import optimizers\n",
    "from gensim.models import Word2Vec\n",
    "from keras.models import Sequential, Model\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from sklearn.utils import shuffle\n",
    "import pickle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from autocorrect import spell\n",
    "import spacy\n",
    "from spacy.gold import GoldParse\n",
    "nlp = spacy.load('en')\n",
    "import re\n",
    "from sklearn.utils import shuffle\n",
    "import keras\n",
    "from keras.utils.vis_utils import plot_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_cols = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['total_classes'] = df['toxic']+df['severe_toxic']+df['obscene']+df['threat']+df['insult']+df['identity_hate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['comment_text'] = df['comment_text'].apply(lambda x : x.replace(\"'\", \"\").replace('\"',''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def correct_spelling(text):\n",
    "    words = text_to_word_sequence(text)\n",
    "    #print (words)\n",
    "    words = [spell(w) for w in words]\n",
    "    return \" \".join(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#df['comment_text'] = df['comment_text'].apply(lambda x : correct_spelling(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['comment_text'] = df['comment_text'].apply(lambda x: re.sub('[0-9]','',x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def replace_unknown_words_with_UNK(sentence):\n",
    "    words = text_to_word_sequence(sentence)\n",
    "    words = [get_word(w) for w in words]\n",
    "    return \" \".join(words)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_word(w):\n",
    "    if w in tokenizer.word_index:\n",
    "        return w\n",
    "    else:\n",
    "        return \"unk\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_tokenizer(texts):\n",
    "    tokenizer = Tokenizer()\n",
    "    sent_list = texts\n",
    "    tokenizer.fit_on_texts(sent_list)\n",
    "    return tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_glove_embedding(glove_path):\n",
    "    word2emb = {}\n",
    "    with open(glove_path, \"rb\") as fglove:\n",
    "        for line in fglove:\n",
    "            cols = line.strip().split()\n",
    "            word = cols[0]\n",
    "            embedding = np.array(cols[1:], dtype=\"float32\")\n",
    "            word2emb[word] = embedding\n",
    "    return word2emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_word2vec(comments):\n",
    "    sents = [text_to_word_sequence(s) for s in comments]\n",
    "    vector = Word2Vec(sents, size=100, iter=50, min_count=1)\n",
    "    return vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "comment_list = df['comment_text'].tolist()\n",
    "glove_file = 'glove.840B.300d.txt'\n",
    "#glove_file = 'glove.6B.100d.txt'\n",
    "emb_matrix = load_glove_embedding(glove_file)\n",
    "#emb_matrix = generate_word2vec(comment_list)\n",
    "max_len = 300\n",
    "comment_list.append(\"unk\")\n",
    "tokenizer = train_tokenizer(comment_list)\n",
    "n_classes = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Replacing all the unknown words with UNK. This will have no impact on training as all the words are known"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['comment_text'] = df['comment_text'].apply(lambda x : replace_unknown_words_with_UNK(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The vocabulary size is: 136352\n",
      "[[136352, 136352, 24, 80719, 6595]]\n"
     ]
    }
   ],
   "source": [
    "print (\"The vocabulary size is: {0}\".format(len(tokenizer.word_index)))\n",
    "print (tokenizer.texts_to_sequences([replace_unknown_words_with_UNK(\"DFLSDKJFLS ADFSDF was Infosys CEO\")]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_up(dfin):\n",
    "    dfin['comment_text'] = dfin['comment_text'].apply(lambda x : str(x).replace(\"'\", \"\").replace('\"',''))\n",
    "    dfin['comment_text'] = dfin['comment_text'].apply(lambda x: re.sub('[0-9]','',x))\n",
    "    #dfin['comment_text'] = dfin['comment_text'].apply(lambda x : replace_unknown_words_with_UNK(x))\n",
    "    return dfin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('toxic', 9237), ('severe_toxic', 965), ('obscene', 5109), ('threat', 305), ('insult', 4765), ('identity_hate', 814)]\n"
     ]
    }
   ],
   "source": [
    "class_count = []\n",
    "for col in pred_cols:\n",
    "    class_count.append((col,len(df[df[col]==1])))\n",
    "print (class_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_stratified_train(df, oversample=None):\n",
    "    df_all_toxic = df[np.logical_and(df['toxic'] ==1 , df['total_classes'] ==1)]\n",
    "    df_all_severe_toxic = df[np.logical_and(df['severe_toxic'] ==1 , df['total_classes'] <=6)]\n",
    "    df_all_obscene = df[np.logical_and(df['obscene'] ==1 , df['total_classes'] <=6)]\n",
    "    df_all_threat = df[np.logical_and(df['threat'] ==1 , df['total_classes'] <=6)]\n",
    "    df_all_insult = df[np.logical_and(df['insult'] ==1 , df['total_classes'] <=6)]\n",
    "    df_all_identity_hate = df[np.logical_and(df['identity_hate'] ==1 , df['total_classes'] <=6)]\n",
    "    df_all_rest =df[df['total_classes'] ==0]\n",
    "    \n",
    "    print(\"Counts:- toxic:{0}, severe_toxic:{1}, obscene:{2}, threat:{3}, insult:{4}, identity_hate:{5}, rest:{6}\".format(len(df_all_toxic),len(df_all_severe_toxic),len(df_all_obscene),len(df_all_threat),len(df_all_insult),len(df_all_identity_hate), len(df_all_rest)))\n",
    "    \n",
    "    X_train_toxic, X_test_toxic = train_test_split(df_all_toxic, test_size=0.10, random_state=42)\n",
    "    X_train_severe_toxic, X_test_severe_toxic = train_test_split(df_all_severe_toxic, test_size=0.1, random_state=42)\n",
    "    X_train_obscene, X_test_obscene = train_test_split(df_all_obscene, test_size=0.05, random_state=42)\n",
    "    X_train_threat, X_test_threat = train_test_split(df_all_threat, test_size=0.05, random_state=42)\n",
    "    X_train_insult, X_test_insult = train_test_split(df_all_insult, test_size=0.10, random_state=42)\n",
    "    X_train_identity_hate, X_test_identity_hate = train_test_split(df_all_identity_hate, test_size=0.1, random_state=42)\n",
    "    X_train_rest, X_test_rest = train_test_split(df_all_rest, test_size=0.10, random_state=42)\n",
    "    print(\"Train Counts:- toxic:{0}, severe_toxic:{1}, obscene:{2}, threat:{3}, insult:{4}, identity_hate:{5}, rest:{6}\".format(len(X_train_toxic),len(X_train_severe_toxic),len(X_train_obscene),len(X_train_threat),len(X_train_insult),len(X_train_identity_hate), len(X_train_rest)))\n",
    "    print(\"Test Counts:- toxic:{0}, severe_toxic:{1}, obscene:{2}, threat:{3}, insult:{4}, identity_hate:{5}, rest:{6}\".format(len(X_test_toxic),len(X_test_severe_toxic),len(X_test_obscene),len(X_test_threat),len(X_test_insult),len(X_test_identity_hate), len(X_test_rest)))\n",
    "    X_train = pd.concat([X_train_toxic, X_train_severe_toxic, X_train_obscene, X_train_threat, X_train_insult, X_train_identity_hate, X_train_rest])\n",
    "    X_test = pd.concat([X_test_toxic, X_test_severe_toxic, X_test_obscene, df_all_threat, X_test_insult, X_test_identity_hate, X_test_rest[:500]])\n",
    "    X_train = X_train.fillna(0)\n",
    "    X_test = X_test.fillna(0)\n",
    "    print (X_train.count(), X_test.count())\n",
    "    X_train.head()\n",
    "    print(X_test.head())\n",
    "    if oversample:\n",
    "        X_train_toxic_samp = rand_over_sample(40000, X_train_toxic)\n",
    "        X_train_severe_toxic_samp = rand_over_sample(40000, X_train_severe_toxic)\n",
    "        X_train_obscene_samp = rand_over_sample(40000, X_train_obscene)\n",
    "        X_train_threat_samp = rand_over_sample(40000, X_train_threat)\n",
    "        X_train_insult_samp = rand_over_sample(40000, X_train_insult)\n",
    "        X_train_identity_hate_samp = rand_over_sample(40000, X_train_identity_hate)\n",
    "        X_train = pd.concat([X_train_toxic_samp, X_train_severe_toxic_samp, X_train_obscene_samp, X_train_threat_samp, X_train_insult_samp, X_train_identity_hate_samp, X_train_rest])\n",
    "\n",
    "    return X_train, X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rand_over_sample(number_of_records, records):\n",
    "    sample = records.sample(n=number_of_records, replace=True)\n",
    "    return sample\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_split(df):\n",
    "    train, test = train_test_split(df, test_size=0.10, random_state=42)\n",
    "    train.head()\n",
    "    XTrain = tokenizer.texts_to_sequences(train.astype(str)['comment_text'].tolist())\n",
    "    XVal = tokenizer.texts_to_sequences(test.astype(str)['comment_text'].tolist())\n",
    "    YTrain = np.array(train[['toxic','severe_toxic','obscene','threat','insult','identity_hate']])\n",
    "    YVal = np.array(test[['toxic','severe_toxic','obscene','threat','insult','identity_hate']])\n",
    "    ytemp = train['toxic'].astype(str)+train['severe_toxic'].astype(str)+train['obscene'].astype(str)+train['threat'].astype(str)+train['insult'].astype(str)+train['identity_hate'].astype(str)\n",
    "    YTrainNum = ytemp.apply(lambda x : int(x,2))\n",
    "    ytemp = test['toxic'].astype(str)+test['severe_toxic'].astype(str)+test['obscene'].astype(str)+test['threat'].astype(str)+test['insult'].astype(str)+test['identity_hate'].astype(str)\n",
    "    YValNum = ytemp.apply(lambda x : int(x,2))\n",
    "    return XTrain, XVal, YTrain, YVal, YTrainNum.values, YValNum.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_weight_matrix_glove(w2vec, tokenizer, emb_dim=100):\n",
    "    matrix = np.zeros((len(tokenizer.word_index)+1,emb_dim))\n",
    "    count = 0\n",
    "    absent_words = []\n",
    "    for key in tokenizer.word_index:\n",
    "        if str.encode(key.replace(\"'\", \"\").replace('\"','')) in w2vec:\n",
    "            matrix[tokenizer.word_index[key]] = w2vec[str.encode(key.replace(\"'\", \"\").replace('\"',''))]\n",
    "        else:\n",
    "            count+=1\n",
    "            absent_words.append(key)\n",
    "    print (count)\n",
    "    #print (absent_words)\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_weight_matrix_local(w2vec, tokenizer, emb_dim=100):\n",
    "    matrix = np.zeros((len(tokenizer.word_index)+1,emb_dim))\n",
    "    count = 0\n",
    "    absent_words = []\n",
    "    for key in tokenizer.word_index:\n",
    "        if key.replace(\"'\", \"\").replace('\"','') in w2vec:\n",
    "            matrix[tokenizer.word_index[key]] = w2vec[key.replace(\"'\", \"\").replace('\"','')]\n",
    "        else:\n",
    "            count+=1\n",
    "            absent_words.append(key)\n",
    "    print (count)\n",
    "    #print (absent_words)\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This returns CNN based model. There are 6 output classes, all sharing the parameters of a common network.\n",
    "\"\"\"\n",
    "def get_model(emb_matrix, learning_rate=0.001):\n",
    "    input = Input(shape=(maxlen,), dtype='int32')\n",
    "    embedding = Embedding( input_dim=emb_matrix.shape[0], output_dim=emb_matrix.shape[1], weights=[emb_matrix],input_length=maxlen,trainable=True)\n",
    "\n",
    "    sequence_input = embedding(input)\n",
    "    x = Conv1D(64, 3, activation='relu')(sequence_input)\n",
    "    x = MaxPooling1D(2)(x)\n",
    "    x = Conv1D(128, 3, activation='relu')(x)\n",
    "    x = MaxPooling1D(2)(x)\n",
    "    x = Conv1D(256, 3, activation='relu')(x)\n",
    "    x = MaxPooling1D(2)(x)\n",
    "\n",
    "    x = Flatten()(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(64, activation='relu')(x)\n",
    "\n",
    "    #toxic\tsevere_toxic\tobscene\tthreat\tinsult\tidentity_hate\n",
    "    preds_toxic = Dense(n_classes, activation='sigmoid')(x)\n",
    "    preds_servere_toxic = Dense(n_classes, activation='sigmoid')(x)\n",
    "    preds_obscene = Dense(n_classes, activation='sigmoid')(x)\n",
    "    preds_threat = Dense(n_classes, activation='sigmoid')(x)\n",
    "    preds_insult = Dense(n_classes, activation='sigmoid')(x)\n",
    "    preds_identity_hate = Dense(n_classes, activation='sigmoid')(x)\n",
    "    \n",
    "    model = Model(input,[preds_toxic, preds_servere_toxic, preds_obscene, preds_threat, preds_insult, preds_identity_hate])\n",
    "    #model.add(Activation('softmax'))\n",
    "    sgd = optimizers.SGD(lr=learning_rate, clipvalue=0.5)\n",
    "    model.compile(loss='mse', optimizer=sgd,metrics=['accuracy'])\n",
    "\n",
    "    print (model.summary())\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This returns LSTM based model. There are 6 output classes, all soft sharing the parameters of a common network.\n",
    "\"\"\"\n",
    "def get_model_soft_sharing_lstm(emb_matrix, learning_rate=0.001):\n",
    "    input = Input(shape=(maxlen,), dtype='int32')\n",
    "    embedding = Embedding( input_dim=emb_matrix.shape[0], output_dim=emb_matrix.shape[1], weights=[emb_matrix],input_length=maxlen,trainable=True)\n",
    "\n",
    "    sequence_input = embedding(input)\n",
    "    x = Bidirectional(LSTM(128,return_sequences=True))(sequence_input)\n",
    "    x = Bidirectional(LSTM(128,return_sequences=False))(x)\n",
    "\n",
    "    x = Dense(256, activation='relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x1 = Dense(64, activation='relu')(x)\n",
    "    x2 = Dense(64, activation='relu')(x)\n",
    "    x3 = Dense(64, activation='relu')(x)\n",
    "    x4 = Dense(64, activation='relu')(x)\n",
    "    x5 = Dense(64, activation='relu')(x)\n",
    "    x6 = Dense(64, activation='relu')(x)\n",
    "\n",
    "    #toxic\tsevere_toxic\tobscene\tthreat\tinsult\tidentity_hate\n",
    "    preds_toxic = Dense(n_classes, activation='sigmoid')(x1)\n",
    "    preds_servere_toxic = Dense(n_classes, activation='sigmoid')(x2)\n",
    "    preds_obscene = Dense(n_classes, activation='sigmoid')(x3)\n",
    "    preds_threat = Dense(n_classes, activation='sigmoid')(x4)\n",
    "    preds_insult = Dense(n_classes, activation='sigmoid')(x5)\n",
    "    preds_identity_hate = Dense(n_classes, activation='sigmoid')(x6)\n",
    "    \n",
    "    model = Model(input,[preds_toxic, preds_servere_toxic, preds_obscene, preds_threat, preds_insult, preds_identity_hate])\n",
    "    #model.add(Activation('softmax'))\n",
    "    adam = optimizers.Adam(lr=learning_rate)\n",
    "    model.compile(loss='mse', optimizer=adam,metrics=['accuracy'])\n",
    "\n",
    "    print (model.summary())\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This returns LSTM based model. There are 6 output classes, all soft sharing the parameters of a common network.\n",
    "\"\"\"\n",
    "def get_model_soft_sharing_lstm_singleoutput(emb_matrix, learning_rate=0.001, n_classes=1, loss='binary_crossentropy',trainable=True, embtrainable=True):\n",
    "    input = Input(shape=(maxlen,), dtype='int32')\n",
    "    embedding = Embedding( input_dim=emb_matrix.shape[0], output_dim=emb_matrix.shape[1], weights=[emb_matrix],input_length=maxlen,trainable=embtrainable)\n",
    "\n",
    "    sequence_input = embedding(input)\n",
    "    x = Bidirectional(LSTM(128,return_sequences=True,trainable=trainable))(sequence_input)\n",
    "    x = Bidirectional(LSTM(128,return_sequences=False,trainable=trainable))(x)\n",
    "\n",
    "    x = Dense(256, activation='relu',trainable=trainable)(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    x = Dense(128, activation='relu',trainable=trainable)(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    x = Dense(128, activation='relu',trainable=trainable)(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    x = Dense(64, activation='relu',trainable=trainable)(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    x = Dense(64, activation='relu',trainable=trainable)(x)\n",
    "    preds = Dense(n_classes, activation='sigmoid',trainable=trainable)(x)\n",
    "\n",
    "    #toxic\tsevere_toxic\tobscene\tthreat\tinsult\tidentity_hate\n",
    "   \n",
    "    \n",
    "    model = Model(input,preds)\n",
    "    #model.add(Activation('softmax'))\n",
    "    adam = optimizers.Adam(lr=learning_rate)\n",
    "    model.compile(loss=loss, optimizer=adam,metrics=['accuracy'])\n",
    "    #model.compile(loss='mse', optimizer=adam,metrics=['accuracy'])\n",
    "\n",
    "    print (model.summary())\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_model(model,to_file='model.png')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_combined_model(emb_matrix, learning_rate=0.001, n_classes=1, loss='binary_crossentropy', weights_path=''):\n",
    "    pretrained_model = get_model_soft_sharing_lstm_singleoutput(emb_matrix, learning_rate=learning_rate, n_classes=64, loss='categorical_crossentropy',trainable=False, embtrainable=False)\n",
    "    pretrained_model.load_weights(weights_path)\n",
    "    print('weights loaded into the network')\n",
    "    pretrained_model.layers.pop()\n",
    "    tox_model = get_model_soft_sharing_lstm_singleoutput(emb_matrix, learning_rate=learning_rate, n_classes=n_classes, loss=loss, embtrainable=True)\n",
    "    tox_model.layers.pop()\n",
    "    pretrained_out = pretrained_model.layers[-1].output\n",
    "    tox_model_out = tox_model.layers[-1].output\n",
    "    conc_output = Concatenate()([pretrained_out,tox_model_out])\n",
    "    preds = Dense(n_classes, activation='sigmoid',trainable=True)(conc_output)\n",
    "    model = Model([pretrained_model.input, tox_model.input], preds)\n",
    "    adam = optimizers.Adam(lr=learning_rate)\n",
    "    model.compile(loss=loss, optimizer=adam,metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Callbacks are passed to the model fit the `callbacks` argument in `fit`,\n",
    "# which takes a list of callbacks. You can pass any number of callbacks.\n",
    "callbacks_list = [\n",
    "    # This callback will interrupt training when we have stopped improving\n",
    "    keras.callbacks.EarlyStopping(\n",
    "        # This callback will monitor the validation accuracy of the model\n",
    "        monitor='val_loss',\n",
    "        # Training will be interrupted when the accuracy\n",
    "        # has stopped improving for *more* than 1 epochs (i.e. 2 epochs)\n",
    "        patience=10,\n",
    "    ),\n",
    "    # This callback will save the current weights after every epoch\n",
    "    keras.callbacks.ModelCheckpoint(\n",
    "        filepath='/Users/mayoor/dev/kaggle/tc/models/tc.h5',  # Path to the destination model file\n",
    "        # The two arguments below mean that we will not overwrite the\n",
    "        # model file unless `val_loss` has improved, which\n",
    "        # allows us to keep the best model every seen during training.\n",
    "        monitor='val_loss',\n",
    "        save_best_only=True,\n",
    "    ),\n",
    "    \n",
    "       keras.callbacks.ReduceLROnPlateau(\n",
    "           # This callback will monitor the validation loss of the model\n",
    "           monitor='val_loss',\n",
    "           # It will divide the learning by 10 when it gets triggered\n",
    "           factor=0.1,\n",
    "           # It will get triggered after the validation loss has stopped improving\n",
    "           # for at least 10 epochs\n",
    "           patience=3,\n",
    ") ,\n",
    "\n",
    "    keras.callbacks.TensorBoard(\n",
    "        # Log files will be written at this location\n",
    "        log_dir='/Users/mayoor/dev/kaggle/tc/logs',\n",
    "        # We will record activation histograms every 1 epoch\n",
    "        histogram_freq=1,\n",
    "        # We will record embedding data every 1 epoch\n",
    "        embeddings_freq=1,\n",
    ") \n",
    "\n",
    "]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Use the X_train_* to create XTrains and YTrains."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Call this method if you are not directly using the df to split into test and train\n",
    "\"\"\"\n",
    "def get_xtrain_Ytrain(X_train):\n",
    "    X_train = shuffle(X_train)\n",
    "    XTrain = tokenizer.texts_to_sequences(X_train.astype(str)['comment_text'].tolist())\n",
    "    YTrain_toxic = np.array(X_train['toxic'].tolist())\n",
    "    YTrain_severe_toxic = np.array(X_train['severe_toxic'].tolist())\n",
    "    YTrain_obscene = np.array(X_train['obscene'].tolist())\n",
    "    YTrain_threat = np.array(X_train['threat'].tolist())\n",
    "    YTrain_insult = np.array(X_train['insult'].tolist())\n",
    "    YTrain_identity_hate = np.array(X_train['identity_hate'].tolist())\n",
    "    #YTrain = [YTrain_toxic, YTrain_severe_toxic, YTrain_obscene, YTrain_threat, YTrain_insult, YTrain_identity_hate]\n",
    "    YTrain = np.array(X_train[['toxic','severe_toxic','obscene','threat','insult','identity_hate']])\n",
    "    #print(YTrain.shape)\n",
    "    X_train.head()\n",
    "    return XTrain, YTrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_xval_Yval(X_test):\n",
    "    XVal = tokenizer.texts_to_sequences(X_test.astype(str)['comment_text'].tolist())\n",
    "    #print(XTrain[0:10],(X_test.astype(str)['comment_text'][0:10]))\n",
    "    YVal_toxic = np.array(X_test['toxic'].tolist())\n",
    "    YVal_severe_toxic = np.array(X_test['severe_toxic'].tolist())\n",
    "    YVal_obscene = np.array(X_test['obscene'].tolist())\n",
    "    YVal_threat = np.array(X_test['threat'].tolist())\n",
    "    YVal_insult = np.array(X_test['insult'].tolist())\n",
    "    YVal_identity_hate = np.array(X_test['identity_hate'].tolist())\n",
    "    #YVal = [YVal_toxic, YVal_severe_toxic, YVal_obscene, YVal_threat, YVal_insult, YVal_identity_hate]\n",
    "    YVal = np.array(X_test[['toxic','severe_toxic','obscene','threat','insult','identity_hate']])\n",
    "    return XVal, YVal\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "XTrain, XVal, YTrain, YVal, YTrainNum, YValNum = get_train_split(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54301\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_9 (InputLayer)         (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "embedding_9 (Embedding)      (None, 300, 300)          40905900  \n",
      "_________________________________________________________________\n",
      "bidirectional_17 (Bidirectio (None, 300, 256)          439296    \n",
      "_________________________________________________________________\n",
      "bidirectional_18 (Bidirectio (None, 256)               394240    \n",
      "_________________________________________________________________\n",
      "dense_52 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_33 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_53 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_34 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_54 (Dense)             (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dropout_35 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_55 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_36 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_56 (Dense)             (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "dense_57 (Dense)             (None, 64)                4160      \n",
      "=================================================================\n",
      "Total params: 41,871,212\n",
      "Trainable params: 0\n",
      "Non-trainable params: 41,871,212\n",
      "_________________________________________________________________\n",
      "None\n",
      "weights loaded into the network\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_10 (InputLayer)        (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "embedding_10 (Embedding)     (None, 300, 300)          40905900  \n",
      "_________________________________________________________________\n",
      "bidirectional_19 (Bidirectio (None, 300, 256)          439296    \n",
      "_________________________________________________________________\n",
      "bidirectional_20 (Bidirectio (None, 256)               394240    \n",
      "_________________________________________________________________\n",
      "dense_58 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_37 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_59 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_38 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_60 (Dense)             (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dropout_39 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_61 (Dense)             (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_40 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_62 (Dense)             (None, 64)                4160      \n",
      "_________________________________________________________________\n",
      "dense_63 (Dense)             (None, 6)                 390       \n",
      "=================================================================\n",
      "Total params: 41,867,442\n",
      "Trainable params: 41,867,442\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_9 (InputLayer)             (None, 300)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_10 (InputLayer)            (None, 300)           0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_9 (Embedding)          (None, 300, 300)      40905900    input_9[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "embedding_10 (Embedding)         (None, 300, 300)      40905900    input_10[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_17 (Bidirectional) (None, 300, 256)      439296      embedding_9[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_19 (Bidirectional) (None, 300, 256)      439296      embedding_10[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_18 (Bidirectional) (None, 256)           394240      bidirectional_17[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "bidirectional_20 (Bidirectional) (None, 256)           394240      bidirectional_19[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "dense_52 (Dense)                 (None, 256)           65792       bidirectional_18[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "dense_58 (Dense)                 (None, 256)           65792       bidirectional_20[0][0]           \n",
      "____________________________________________________________________________________________________\n",
      "dropout_33 (Dropout)             (None, 256)           0           dense_52[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dropout_37 (Dropout)             (None, 256)           0           dense_58[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_53 (Dense)                 (None, 128)           32896       dropout_33[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dense_59 (Dense)                 (None, 128)           32896       dropout_37[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_34 (Dropout)             (None, 128)           0           dense_53[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dropout_38 (Dropout)             (None, 128)           0           dense_59[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_54 (Dense)                 (None, 128)           16512       dropout_34[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dense_60 (Dense)                 (None, 128)           16512       dropout_38[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_35 (Dropout)             (None, 128)           0           dense_54[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dropout_39 (Dropout)             (None, 128)           0           dense_60[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_55 (Dense)                 (None, 64)            8256        dropout_35[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dense_61 (Dense)                 (None, 64)            8256        dropout_39[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_36 (Dropout)             (None, 64)            0           dense_55[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dropout_40 (Dropout)             (None, 64)            0           dense_61[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_56 (Dense)                 (None, 64)            4160        dropout_36[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dense_62 (Dense)                 (None, 64)            4160        dropout_40[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)      (None, 128)           0           dense_56[0][0]                   \n",
      "                                                                   dense_62[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_64 (Dense)                 (None, 6)             774         concatenate_5[0][0]              \n",
      "====================================================================================================\n",
      "Total params: 83,734,878\n",
      "Trainable params: 41,867,826\n",
      "Non-trainable params: 41,867,052\n",
      "____________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "maxlen = 300\n",
    "#final_emb_matrix = get_weight_matrix_local(emb_matrix, tokenizer, 100)\n",
    "final_emb_matrix = get_weight_matrix_glove(emb_matrix, tokenizer, 300)\n",
    "#model = get_model(final_emb_matrix, learning_rate=0.001)\n",
    "#model = get_model_soft_sharing_lstm_singleoutput(final_emb_matrix, learning_rate=0.01, n_classes=64, loss='categorical_crossentropy')\n",
    "#model = get_model_soft_sharing_lstm(final_emb_matrix, learning_rate=0.001)\n",
    "model = get_combined_model(final_emb_matrix, learning_rate=0.001, n_classes=6, loss='binary_crossentropy', weights_path='models/64.h5')\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "ytrainbi = np.zeros((len(YTrainNum),64))\n",
    "ytestbi = np.zeros((len(YValNum),64))\n",
    "for i in range(len(YTrainNum)):\n",
    "    ytrainbi[i][YTrainNum[i]] = 1\n",
    "for i in range(len(YValNum)):\n",
    "    ytestbi[i][YValNum[i]] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 [0 0 0 0 1 0]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(YTrainNum[9000], YTrain[9000])\n",
    "ytrainbi[9000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 86265 samples, validate on 9586 samples\n",
      "Epoch 1/10\n",
      "86265/86265 [==============================] - 5520s - loss: 0.2119 - acc: 0.9529 - val_loss: 0.0473 - val_acc: 0.9824\n",
      "Epoch 2/10\n",
      "86265/86265 [==============================] - 5530s - loss: 0.0470 - acc: 0.9823 - val_loss: 0.0442 - val_acc: 0.9831\n",
      "Epoch 3/10\n",
      "86265/86265 [==============================] - 7109s - loss: 0.0420 - acc: 0.9836 - val_loss: 0.0448 - val_acc: 0.9833\n",
      "Epoch 4/10\n",
      "14848/86265 [====>.........................] - ETA: 5228s - loss: 0.0383 - acc: 0.9847"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-59-68ffebcccfd2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpad_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXTrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpad_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXTrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mYTrain\u001b[0m \u001b[0;34m,\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpad_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXVal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpad_sequences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXVal\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mYVal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1596\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1597\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1598\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1599\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1600\u001b[0m     def evaluate(self, x, y,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1181\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m                     \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1183\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1184\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1185\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2271\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2272\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2273\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2274\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    790\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    995\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    998\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1133\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1137\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1140\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1121\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit([pad_sequences(XTrain, maxlen),pad_sequences(XTrain, maxlen)],YTrain ,batch_size=256, epochs=10, verbose=1, validation_data=([pad_sequences(XVal, maxlen),pad_sequences(XVal, maxlen)], YVal), callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('/Users/mayoor/dev/kaggle/tc/models/tc.h5')\n",
    "model.evaluate(pad_sequences(XVal, maxlen), YVal, batch_size=128)\n",
    "#print(\"\\nTest score: %.3f, accuracy: %.3f\" % (v_score, v_acc))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_df = pd.read_csv('test.csv')\n",
    "test_df = clean_up(test_df)\n",
    "test_comments = test_df['comment_text'].astype(str).tolist()\n",
    "XTest = tokenizer.texts_to_sequences(test_comments)\n",
    "print (test_df.columns)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = model.predict(pad_sequences(XTest, maxlen))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predicted_df = pd.DataFrame(columns=['id','toxic','severe_toxic','obscene','threat','insult','identity_hate'])\n",
    "predicted_df['id'] = test_df['id']\n",
    "for i, k in enumerate(pred_cols):\n",
    "    predicted_df[k] = predictions[:,i]\n",
    "predicted_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predicted_df.to_csv('first_submission.csv',index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(test_df[test_df['id']==361592343415])\n",
    "predicted_df[predicted_df['id']==361592343415]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(test_df[test_df['id']==361543686278])\n",
    "predicted_df[predicted_df['id']==361543686278]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.options.display.max_colwidth = 600\n",
    "print(test_df[test_df['id']==361544361532]['comment_text'])\n",
    "predicted_df[predicted_df['id']==361544361532]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}