AnalyzeModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from keras.layers import LSTM, Bidirectional, Embedding, Input, Flatten, Dense, BatchNormalization, Dropout, Conv1D, Concatenate, MaxPool1D, AveragePooling1D, GlobalAveragePooling1D, GlobalMaxPool1D, TimeDistributed, Lambda, Add\n",
    "\n",
    "from keras.models import Model\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from keras.preprocessing.sequence import pad_sequences\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import transformer as tr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import toxicity_metrics as tm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import misclass_report as mr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns; sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = joblib.load('./tokenizer')\n",
    "embedding = joblib.load('./embedding')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_text_for_prediction(text, tokenizer=tokenizer, maxlen=100):\n",
    "    return pad_sequences(tokenizer.texts_to_sequences([text]),maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_predictions(testcases, model, tokenizer,maxlen=200):\n",
    "    result = []\n",
    "    for testcase in testcases:\n",
    "        result.append(model.predict(convert_text_for_prediction(testcase,tokenizer,maxlen)))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "testcases = [\"women are good people but men are trash\",\"you are a mother fucking asshole\",\n",
    "             \"You are gay\",\"I am gay\",\"You are dead if i see you ever again\",'It\\'s ridiculous that these guys are being called \"protesters\". Being armed is a threat of violence, which makes them terrorists.']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformer Model Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/mayoor/dev/kaggle/tc2/tc2/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "PE shape is (200, 300)\n",
      "The PE output shape is Tensor(\"positional_encoder_1/add:0\", shape=(?, 200, 300), dtype=float32)\n",
      "WARNING:tensorflow:From /Users/mayoor/dev/kaggle/tc2/tc2/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "transformer_model = tr.get_transformer_model(3,embedding,200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformer_model.load_weights('models/best_model_transformer.h5')\n",
    "transformer_model.load_weights('models/best_model_transformer_new.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pepare_train_test(file, tokenizer,sample=None):\n",
    "    import pandas as pd\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    train_df = pd.read_csv(file)\n",
    "    train_binary = np.zeros((len(train_df),))\n",
    "    train_binary[train_df['target']>0.5] = 1\n",
    "    train_df['target_binary'] = train_binary\n",
    "    train, test  = train_test_split(train_df, test_size=0.002, random_state=42,stratify=train_df['target_binary'])\n",
    "    if sample:\n",
    "        train = train.sample(sample)\n",
    "        test = test.sample(int(sample/10))\n",
    "#     return tokenizer.texts_to_sequences(train['comment_text'].values.astype(str).tolist()), train['target_binary'].values, tokenizer.texts_to_sequences(test['comment_text'].values.astype(str).tolist()), test['target_binary'].values\n",
    "    return train, test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = pepare_train_test('train_clean.csv', tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert taget and identity columns to booleans\n",
    "def convert_to_bool(df, col_name):\n",
    "    df[col_name] = np.where(df[col_name] >= 0.5, True, False)\n",
    "    \n",
    "def convert_dataframe_to_bool(df):\n",
    "    bool_df = df.copy()\n",
    "    for col in ['binary_target']+identity_columns:\n",
    "        convert_to_bool(bool_df, col)\n",
    "    return bool_df\n",
    "\n",
    "validate_df = convert_dataframe_to_bool(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = get_predictions(testcases,transformer_model,tokenizer,200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[0.65063244]], dtype=float32),\n",
       " array([[0.9917435]], dtype=float32),\n",
       " array([[0.5660384]], dtype=float32),\n",
       " array([[0.47629952]], dtype=float32),\n",
       " array([[0.05181587]], dtype=float32),\n",
       " array([[0.32145128]], dtype=float32)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.engine.input_layer.InputLayer at 0x12f3570b8>,\n",
       " <keras.layers.embeddings.Embedding at 0x1320f5f28>,\n",
       " <transformer.PositionalEncoder at 0x1320f5d30>,\n",
       " <transformer.SDPA at 0x102913908>,\n",
       " <transformer.SDPA at 0x1320f5b00>,\n",
       " <transformer.SDPA at 0x1320f5d68>,\n",
       " <keras.layers.merge.Concatenate at 0x131438fd0>,\n",
       " <keras.layers.core.Dense at 0x15f907c18>,\n",
       " <keras.layers.merge.Add at 0x15fb01a90>,\n",
       " <layer_normalization.LayerNormalization at 0x15fb01da0>,\n",
       " <keras.layers.wrappers.TimeDistributed at 0x15fb74978>,\n",
       " <keras.layers.merge.Add at 0x15fb74a20>,\n",
       " <layer_normalization.LayerNormalization at 0x15fb74a58>,\n",
       " <keras.layers.pooling.GlobalMaxPooling1D at 0x15fb86518>,\n",
       " <keras.layers.pooling.GlobalAveragePooling1D at 0x15fb9fac8>,\n",
       " <keras.layers.merge.Concatenate at 0x15fb9fa58>,\n",
       " <keras.layers.core.Dense at 0x15fbb2e80>,\n",
       " <keras.layers.core.Dropout at 0x15fbde550>,\n",
       " <keras.layers.core.Dense at 0x15fbde1d0>,\n",
       " <keras.layers.core.Dropout at 0x15fbf0438>,\n",
       " <keras.layers.core.Dense at 0x15fc20ba8>,\n",
       " <keras.layers.core.Dropout at 0x15fc3c940>,\n",
       " <keras.layers.core.Dense at 0x15fc3cb38>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformer_model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = mr.prepare_for_report(transformer_model, tokenizer, 200, './train_clean.csv', 'comment_text', 'transformer',sample_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = mr.get_groupwise_auc(train_df,'binary_target', 'transformer' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bnsp_auc</th>\n",
       "      <th>bpsn_auc</th>\n",
       "      <th>subgroup</th>\n",
       "      <th>subgroup_auc</th>\n",
       "      <th>subgroup_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.540618</td>\n",
       "      <td>0.638733</td>\n",
       "      <td>muslim</td>\n",
       "      <td>0.540418</td>\n",
       "      <td>4232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.545603</td>\n",
       "      <td>0.635706</td>\n",
       "      <td>homosexual_gay_or_lesbian</td>\n",
       "      <td>0.543685</td>\n",
       "      <td>2237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.543601</td>\n",
       "      <td>0.638202</td>\n",
       "      <td>christian</td>\n",
       "      <td>0.543721</td>\n",
       "      <td>8061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.553867</td>\n",
       "      <td>0.636358</td>\n",
       "      <td>jewish</td>\n",
       "      <td>0.553801</td>\n",
       "      <td>1562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.559881</td>\n",
       "      <td>0.636641</td>\n",
       "      <td>black</td>\n",
       "      <td>0.558341</td>\n",
       "      <td>2919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.563988</td>\n",
       "      <td>0.638240</td>\n",
       "      <td>white</td>\n",
       "      <td>0.562808</td>\n",
       "      <td>5019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.568935</td>\n",
       "      <td>0.637628</td>\n",
       "      <td>psychiatric_or_mental_illness</td>\n",
       "      <td>0.570423</td>\n",
       "      <td>984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.578221</td>\n",
       "      <td>0.638265</td>\n",
       "      <td>female</td>\n",
       "      <td>0.577698</td>\n",
       "      <td>10770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.581935</td>\n",
       "      <td>0.637268</td>\n",
       "      <td>male</td>\n",
       "      <td>0.580942</td>\n",
       "      <td>8863</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bnsp_auc  bpsn_auc                       subgroup  subgroup_auc  \\\n",
       "5  0.540618  0.638733                         muslim      0.540418   \n",
       "2  0.545603  0.635706      homosexual_gay_or_lesbian      0.543685   \n",
       "3  0.543601  0.638202                      christian      0.543721   \n",
       "4  0.553867  0.636358                         jewish      0.553801   \n",
       "6  0.559881  0.636641                          black      0.558341   \n",
       "7  0.563988  0.638240                          white      0.562808   \n",
       "8  0.568935  0.637628  psychiatric_or_mental_illness      0.570423   \n",
       "1  0.578221  0.638265                         female      0.577698   \n",
       "0  0.581935  0.637268                           male      0.580942   \n",
       "\n",
       "   subgroup_size  \n",
       "5           4232  \n",
       "2           2237  \n",
       "3           8061  \n",
       "4           1562  \n",
       "6           2919  \n",
       "7           5019  \n",
       "8            984  \n",
       "1          10770  \n",
       "0           8863  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sdpa1 = transformer_model.layers[3]\n",
    "sdpa2 = transformer_model.layers[4]\n",
    "sdpa3 = transformer_model.layers[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(transformer_model.input,sdpa1.output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "values = model.predict(convert_text_for_prediction(\"you are a mother fucking asshole\",tokenizer,200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 200, 100)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "sdpa_weights = sdpa1.get_weights()\n",
    "sdpa_weights_2 = sdpa2.get_weights()\n",
    "sdpa_weights_3 = sdpa3.get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_sdpa(sent, kernel_Q, kernel_K, kernel_V, verbose=False):\n",
    "    Q = sent.dot(kernel_Q)\n",
    "    if verbose:\n",
    "        print (\"Q is {}\".format(Q))\n",
    "\n",
    "    K = sent.dot(kernel_K)\n",
    "    if verbose:\n",
    "        print (\"K is {}\".format(K))\n",
    "\n",
    "    V = sent.dot(kernel_V)\n",
    "    if verbose:\n",
    "        print (\"V is {}\".format(V))\n",
    "    numerator = Q.dot(K.T)\n",
    "    if verbose:\n",
    "        print (\"Numerator is {}\".format(numerator))\n",
    "    \n",
    "    numerator_div_sqrt = numerator/np.sqrt(Q.shape[1])\n",
    "    if verbose:\n",
    "        print (\"Numerator after div by sqrt is {}\".format(numerator_div_sqrt))\n",
    "    \n",
    "    softmax = np.exp(numerator_div_sqrt)/np.sum(np.exp(numerator_div_sqrt), axis=1)\n",
    "    if verbose:\n",
    "        print (\"Softmax numerator is {}\".format(softmax))\n",
    "        print (\"Softmax shape is {}\".format(softmax.shape))\n",
    "    \n",
    "    final = softmax.dot(V)\n",
    "    if verbose:\n",
    "        print (\"Final dot product is {}\".format(final))\n",
    "    return final, softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_output = transformer_model.layers[1].output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_model = Model(transformer_model.input, embedding_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_sent_association(sent, emb_model, sdpa_weights, tokenizer, maxlen):\n",
    "    emb_val = emb_model.predict(convert_text_for_prediction(sent,tokenizer,maxlen))\n",
    "    final, softmax = compute_sdpa(emb_val[0], sdpa_weights[0],sdpa_weights[1],sdpa_weights[2])\n",
    "    words = sent.split()\n",
    "    indexes = [i for i in range(maxlen-len(words),200)]\n",
    "    values = {indexes[ind]:w for ind,w in enumerate(words)}\n",
    "    assoc_words = []\n",
    "    attn_report = softmax.argsort(axis=1)[-len(words):][:,-len(words):]\n",
    "    for ind, prob in enumerate(attn_report):\n",
    "        assoc_words.append ({words[ind]:[values[k] if k in values else 'NA' for k in prob][::-1]})\n",
    "    return assoc_words\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_association(association):\n",
    "    for item in association:\n",
    "        print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "association = check_sent_association(\"women are good men are trash\", emb_model, sdpa_weights_3, tokenizer, 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'women': ['good', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'good': ['good', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'men': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'trash': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    }
   ],
   "source": [
    "print_association(association)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'whenever': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'i': ['I', 'get', 'see', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'see': ['I', 'get', 'see', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'that': ['I', 'get', 'see', 'think', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'mother': ['see', 'I', 'get', 'think', 'angry.', 'that', 'i', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'fucker': ['I', 'get', 'see', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'i': ['fucker', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'get': ['I', 'get', 'see', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'angry.': ['I', 'get', 'see', 'think', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'I': ['I', 'get', 'see', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'think': ['I', 'get', 'see', 'think', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'is': ['I', 'get', 'see', 'think', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'a': ['I', 'get', 'see', 'think', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'gone': ['I', 'get', 'see', 'think', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'mad': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    }
   ],
   "source": [
    "association = check_sent_association(\"whenever i see that mother fucker i get angry. I think is a gone mad\", emb_model, sdpa_weights_3, tokenizer, 200)\n",
    "print_association(association)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'whenever': ['fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'i': ['that', 'mother', 'a', 'is', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'see': ['angry.', 'i', 'that', 'a', 'is', 'mother', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'that': ['angry.', 'i', 'that', 'a', 'mother', 'is', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'mother': ['a', 'is', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'fucker': ['fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'i': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'get': ['angry.', 'i', 'that', 'a', 'is', 'mother', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'angry.': ['angry.', 'i', 'that', 'a', 'mother', 'is', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'I': ['angry.', 'i', 'that', 'a', 'is', 'mother', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'think': ['a', 'is', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'is': ['fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'a': ['a', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'gone': ['a', 'is', 'fucker', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'mad': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mayoor/dev/kaggle/tc2/tc2/lib/python3.6/site-packages/ipykernel_launcher.py:21: RuntimeWarning: overflow encountered in exp\n",
      "/Users/mayoor/dev/kaggle/tc2/tc2/lib/python3.6/site-packages/ipykernel_launcher.py:21: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    }
   ],
   "source": [
    "association = check_sent_association(\"whenever i see that mother fucker i get angry. I think is a gone mad\", emb_model, sdpa_weights_2, tokenizer, 200)\n",
    "print_association(association)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'women': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['trash', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'good': ['trash', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'men': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['trash', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'trash': ['trash', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mayoor/dev/kaggle/tc2/tc2/lib/python3.6/site-packages/ipykernel_launcher.py:21: RuntimeWarning: overflow encountered in exp\n",
      "/Users/mayoor/dev/kaggle/tc2/tc2/lib/python3.6/site-packages/ipykernel_launcher.py:21: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    }
   ],
   "source": [
    "association = check_sent_association(\"women are good men are trash\", emb_model, sdpa_weights_2, tokenizer, 200)\n",
    "print_association(association)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'It': ['a', 'is', 'ridiculous', 'that', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'is': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'ridiculous': ['are', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'that': ['are', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'these': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'guys': ['are', 'is', 'that', 'makes', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['are', 'is', 'ridiculous', 'a', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'being': ['are', 'is', 'makes', 'that', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'called': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'protesters.': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'Being': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'armed': ['are', 'that', 'is', 'makes', 'them', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'is': ['are', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'a': ['are', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'threat': ['are', 'is', 'makes', 'that', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'of': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'violence': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{',': ['a', 'is', 'ridiculous', 'that', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'which': ['are', 'is', 'makes', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'makes': ['are', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'them': ['are', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'terrorists': ['are', 'that', 'makes', 'is', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    }
   ],
   "source": [
    "association = check_sent_association(\"It is ridiculous that these guys are being called protesters. Being armed is a threat of violence , which makes them terrorists\", emb_model, sdpa_weights_3, tokenizer, 200)\n",
    "print_association(association)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'haha': ['a', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'you': ['you', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'guys': ['you', 'guys', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['guys', 'you', 'haha', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'a': ['you', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'bunch': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'of': ['haha', 'you', 'guys', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'losers': ['a', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    }
   ],
   "source": [
    "association = check_sent_association(\"haha you guys are a bunch of losers\", emb_model, sdpa_weights_3, tokenizer, 200)\n",
    "print_association(association)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transfomer trained along with YK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_multi_head_attn(input, head_size, embedding_size,name=\"multi_head\"):\n",
    "    transformers = []\n",
    "    for i in range(head_size):\n",
    "        x1 = tr.SDPA(150)(input)\n",
    "#         xh = Dense(100, activation='relu')(x1)\n",
    "        transformers.append(x1)\n",
    "    x = Concatenate()(transformers)\n",
    "    x = Dense(embedding_size, name=name)(x)\n",
    "    return x\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_transformer(input, head_size=5, embedding_size=100,index=0):\n",
    "    multi_head = get_multi_head_attn(input, head_size, embedding_size, \"multi_head_{}\".format(index))\n",
    "    add_out = Add()([multi_head, input])\n",
    "    norm_out = tr.LayerNormalization(name=\"multi_head_norm_{}\".format(index))(add_out)\n",
    "\n",
    "    ffout = TimeDistributed(Dense(embedding_size,activation='relu'))(norm_out)\n",
    "    add_out = Add()([norm_out, ffout])\n",
    "    norm_out = tr.LayerNormalization(name=\"transformer_{}\".format(index))(add_out)\n",
    "    return norm_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_transformer_yk_model(multi_head_size,embedding,maxlen,depth=1):\n",
    "    input = Input(shape=(maxlen,), name=\"input_sentence\")\n",
    "    embx = Embedding(input_dim=embedding.shape[0], output_dim=embedding.shape[1], input_length=maxlen, weights=[embedding], trainable=True, mask_zero=False)(input)\n",
    "    x = tr.PositionalEncoder()(embx)\n",
    "    for i in range(depth):\n",
    "        x = get_transformer(x,multi_head_size,embedding.shape[1],index=i)\n",
    "#     x = Flatten()(x)\n",
    "#     print(x.shape)\n",
    "    maxoutput = GlobalMaxPool1D()(x)\n",
    "#     maxoutput = Flatten()(maxoutput)\n",
    "    avgoutput = GlobalAveragePooling1D()(x)\n",
    "    transformer_output_concat = Concatenate()([maxoutput,avgoutput])\n",
    "  \n",
    "    cnn_x1 = Conv1D(kernel_size=2, filters=128)(embx)\n",
    "    cnn_x2 = Conv1D(kernel_size=3,filters=128)(embx)\n",
    "    cnn_x3 = Conv1D(kernel_size=4,filters=128)(embx)\n",
    "    cnn_x4 = Conv1D(kernel_size=5,filters=128)(embx)\n",
    "    cnn_x1_mp = GlobalMaxPool1D()(cnn_x1)\n",
    "    cnn_x2_mp = GlobalMaxPool1D()(cnn_x2)\n",
    "    cnn_x3_mp = GlobalMaxPool1D()(cnn_x3)\n",
    "    cnn_x4_mp = GlobalMaxPool1D()(cnn_x4)\n",
    "    cnn_x1_avg = GlobalAveragePooling1D()(cnn_x1)\n",
    "    cnn_x2_avg = GlobalAveragePooling1D()(cnn_x2)\n",
    "    cnn_x3_avg = GlobalAveragePooling1D()(cnn_x3)\n",
    "    cnn_x4_avg = GlobalAveragePooling1D()(cnn_x4)\n",
    "    concat_layer = Concatenate()([cnn_x1_mp,cnn_x2_mp,cnn_x3_mp,cnn_x4_mp,cnn_x1_avg,cnn_x2_avg,cnn_x3_avg,cnn_x4_avg])\n",
    "#     flatten_layer = Flatten()(concat_layer)\n",
    "#     concat_layer_drop = Dropout(0.2)(concat_layer)\n",
    "    yk_output_dense = Dense(128, activation='relu')(concat_layer)\n",
    "    transformer_output_dense = Dense(128, activation='relu')(transformer_output_concat)\n",
    "\n",
    "    add_out = Add()([transformer_output_dense,yk_output_dense])\n",
    "    norm_out = tr.LayerNormalization(name=\"yk_transformer_normalizer\")(add_out)\n",
    "\n",
    "#     concat_output = Concatenate()([])\n",
    "    output_dense = Dense(128, activation='relu')(norm_out)\n",
    "\n",
    "    output_dense = Dropout(0.2)(output_dense)\n",
    "#     output_dense = Dense(32, activation='relu')(output_dense)\n",
    "#     output_dense = Dropout(0.2)(output_dense)\n",
    "    output = Dense(1, activation='sigmoid')(output_dense)\n",
    "    return Model(input,output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PE shape is (200, 300)\n",
      "The PE output shape is Tensor(\"positional_encoder_6/add:0\", shape=(?, 200, 300), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "model = get_transformer_yk_model(2,embedding,200,depth=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_sentence (InputLayer)     (None, 200)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_7 (Embedding)         (None, 200, 300)     30450300    input_sentence[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "positional_encoder_6 (Positiona (None, 200, 300)     60000       embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "sdpa_12 (SDPA)                  (None, 200, 150)     135000      positional_encoder_6[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "sdpa_13 (SDPA)                  (None, 200, 150)     135000      positional_encoder_6[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_15 (Concatenate)    (None, 200, 300)     0           sdpa_12[0][0]                    \n",
      "                                                                 sdpa_13[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "multi_head_0 (Dense)            (None, 200, 300)     90300       concatenate_15[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 200, 300)     0           multi_head_0[0][0]               \n",
      "                                                                 positional_encoder_6[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "multi_head_norm_0 (LayerNormali (None, 200, 300)     600         add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_6 (TimeDistrib (None, 200, 300)     90300       multi_head_norm_0[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 200, 300)     0           multi_head_norm_0[0][0]          \n",
      "                                                                 time_distributed_6[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "transformer_0 (LayerNormalizati (None, 200, 300)     600         add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_17 (Conv1D)              (None, 199, 128)     76928       embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_18 (Conv1D)              (None, 198, 128)     115328      embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_19 (Conv1D)              (None, 197, 128)     153728      embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_20 (Conv1D)              (None, 196, 128)     192128      embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_22 (Global (None, 300)          0           transformer_0[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_22 (Gl (None, 300)          0           transformer_0[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_23 (Global (None, 128)          0           conv1d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_24 (Global (None, 128)          0           conv1d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_25 (Global (None, 128)          0           conv1d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_26 (Global (None, 128)          0           conv1d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_23 (Gl (None, 128)          0           conv1d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_24 (Gl (None, 128)          0           conv1d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_25 (Gl (None, 128)          0           conv1d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling1d_26 (Gl (None, 128)          0           conv1d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_16 (Concatenate)    (None, 600)          0           global_max_pooling1d_22[0][0]    \n",
      "                                                                 global_average_pooling1d_22[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_17 (Concatenate)    (None, 1024)         0           global_max_pooling1d_23[0][0]    \n",
      "                                                                 global_max_pooling1d_24[0][0]    \n",
      "                                                                 global_max_pooling1d_25[0][0]    \n",
      "                                                                 global_max_pooling1d_26[0][0]    \n",
      "                                                                 global_average_pooling1d_23[0][0]\n",
      "                                                                 global_average_pooling1d_24[0][0]\n",
      "                                                                 global_average_pooling1d_25[0][0]\n",
      "                                                                 global_average_pooling1d_26[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "dense_26 (Dense)                (None, 128)          76928       concatenate_16[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_25 (Dense)                (None, 128)          131200      concatenate_17[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 128)          0           dense_26[0][0]                   \n",
      "                                                                 dense_25[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "yk_transformer_normalizer (Laye (None, 128)          256         add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_27 (Dense)                (None, 128)          16512       yk_transformer_normalizer[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 128)          0           dense_27[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_28 (Dense)                (None, 1)            129         dropout_7[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 31,725,237\n",
      "Trainable params: 31,665,237\n",
      "Non-trainable params: 60,000\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('models/best_model_transformer_yk_embtrue_0001__3.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_model = Model(model.input, model.layers[1].output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "sdpa1 = transformer_model.layers[3]\n",
    "sdpa2 = transformer_model.layers[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "sdpa_weights = sdpa1.get_weights()\n",
    "sdpa_weights_2 = sdpa2.get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'haha': ['of', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'you': ['of', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'guys': ['of', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['of', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'a': ['of', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'bunch': ['of', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'of': ['of', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'losers': ['haha', 'guys', 'bunch', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    }
   ],
   "source": [
    "association = check_sent_association(\"haha you guys are a bunch of losers\", emb_model, sdpa_weights, tokenizer, 200)\n",
    "print_association(association)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'It': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'is': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'ridiculous': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'that': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'these': ['threat', 'violence', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'guys': ['threat', 'which', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'are': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'being': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'called': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'protesters.': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'Being': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'armed': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'is': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'a': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'threat': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'of': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'violence': [',', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{',': ['NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'which': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'makes': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'them': ['threat', 'which', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n",
      "{'terrorists': ['threat', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA', 'NA']}\n"
     ]
    }
   ],
   "source": [
    "association = check_sent_association(\"It is ridiculous that these guys are being called protesters. Being armed is a threat of violence , which makes them terrorists\", emb_model, sdpa_weights, tokenizer, 200)\n",
    "print_association(association)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}