self-attentive.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tflearn/data_utils.py:201: VocabularyProcessor.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.text) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/preprocessing/text.py:154: CategoricalVocabulary.__init__ (from tensorflow.contrib.learn.python.learn.preprocessing.categorical_vocabulary) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tensorflow/transform or tf.data.\n",
      "INFO:tensorflow:Restoring parameters from ./save/self-attentive.ckpt\n",
      "Test Accuracy: 0.85000\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tflearn\n",
    "import numpy as np\n",
    "import re\n",
    "import csv\n",
    "\n",
    "from tensorflow.python.platform import gfile\n",
    "from sklearn.utils.extmath import softmax\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "TOKENIZER_RE = re.compile(r\"[A-Z]{2,}(?![a-z])|[A-Z][a-z]+(?=[A-Z])|[\\'\\w\\-]+\", re.UNICODE)\n",
    "MAXLEN = 25\n",
    "\n",
    "SAVE_DIR = \"./save\"\n",
    "SAVE_FILE_PATH = SAVE_DIR + \"/self-attentive.ckpt\"\n",
    "\n",
    "class SelfAttenModel(object):\n",
    "    \n",
    "    def __init__(self,\n",
    "                 batch_size=40, \n",
    "                 vocab_size=200,\n",
    "                 hidden_size=2000,\n",
    "                 label_num=4,\n",
    "                 layer_num=1, \n",
    "                 embedding_size=100, \n",
    "                 keep_prob=0.8, \n",
    "                 max_sequence_length=10,\n",
    "                 num_units=128,\n",
    "                 d_a=350,\n",
    "                 r=30,\n",
    "                 learning_rate=0.01,\n",
    "                 p_coef=0.5,\n",
    "                 use_penalization=True):\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.vocab_size = vocab_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.label_num = label_num\n",
    "        self.layer_num = layer_num\n",
    "        self.embedding_size = embedding_size\n",
    "        self.keep_prob = keep_prob\n",
    "        self.n = self.max_sequence_length = max_sequence_length\n",
    "        self.u = self.num_units = num_units\n",
    "        self.d_a = d_a\n",
    "        self.r = r\n",
    "        self.learning_rate = learning_rate\n",
    "        self.p_coef = p_coef\n",
    "        self.use_penalization = use_penalization\n",
    "        \n",
    "        self._build_placeholder()\n",
    "        self._build_model()\n",
    "        self._build_optimizer()\n",
    "            \n",
    "    def _build_placeholder(self):\n",
    "        self.sources = tf.placeholder(name='sources', shape=[self.batch_size, self.max_sequence_length], dtype=tf.int64)\n",
    "        self.labels = tf.placeholder(name='labels', shape=[self.batch_size], dtype=tf.int64)\n",
    "        self.global_step = tf.Variable(0, name='global_step', trainable=False)\n",
    "\n",
    "    def _build_single_cell(self):\n",
    "        cell = tf.contrib.rnn.BasicLSTMCell(self.num_units)\n",
    "        cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=self.keep_prob)\n",
    "        return cell\n",
    "    \n",
    "    def _build_model(self):\n",
    "        # Word embedding #\n",
    "        with tf.variable_scope(\"embedding\"):\n",
    "            initializer = tf.contrib.layers.xavier_initializer()\n",
    "            embeddings = tf.get_variable(name=\"embedding_encoder\",\n",
    "                                                shape=[self.vocab_size, self.embedding_size], \n",
    "                                                dtype=tf.float32,\n",
    "                                                initializer=initializer,\n",
    "                                                trainable=True)\n",
    "\n",
    "            input_embeddings = tf.nn.embedding_lookup(params=embeddings,\n",
    "                                                      ids=self.sources)\n",
    "\n",
    "        # Bidirectional rnn #\n",
    "        with tf.variable_scope(\"bidirectional_rnn\"):\n",
    "            cell_forward = self._build_single_cell()\n",
    "            cell_backward = self._build_single_cell()\n",
    "            \n",
    "            # outputs is state 'H'\n",
    "            outputs, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw=cell_forward, \n",
    "                                                              cell_bw=cell_backward, \n",
    "                                                              inputs=input_embeddings,\n",
    "                                                              dtype=tf.float32)\n",
    "            \n",
    "            H = tf.concat(outputs, -1)\n",
    "            \n",
    "        # Self Attention #\n",
    "        with tf.variable_scope(\"self_attention\"):\n",
    "            initializer = tf.contrib.layers.xavier_initializer()\n",
    "            W_s1 = tf.get_variable(name=\"W_s1\", shape=[self.d_a, 2*self.u], initializer=initializer)\n",
    "            W_s2 = tf.get_variable(name='W_s2', shape=[self.r, self.d_a],initializer=initializer)\n",
    "            \n",
    "            a_prev = tf.map_fn(lambda x: tf.matmul(W_s1, tf.transpose(x)), H)\n",
    "            a_prev = tf.tanh(a_prev)\n",
    "            a_prev = tf.map_fn(lambda x: tf.matmul(W_s2, x), a_prev)\n",
    "            \n",
    "            self.A = tf.nn.softmax(a_prev)\n",
    "            self.M = tf.matmul(self.A, H)\n",
    "        \n",
    "        # Fully connected layer #\n",
    "        with tf.variable_scope(\"fully_connected_layer\"):\n",
    "            input_fc = tf.layers.flatten(self.M)\n",
    "            layer_fc = tf.contrib.layers.fully_connected(inputs=input_fc, \n",
    "                                                         num_outputs=self.hidden_size,\n",
    "                                                         activation_fn=tf.nn.relu)\n",
    "            \n",
    "            self.logits = tf.contrib.layers.fully_connected(inputs=layer_fc, \n",
    "                                                            num_outputs=self.label_num,\n",
    "                                                            activation_fn=None)\n",
    "            \n",
    "            \n",
    "            \n",
    "    def _build_optimizer(self):\n",
    "        with tf.variable_scope(\"optimizer\"):\n",
    "            cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits,\n",
    "                                                                           labels=self.labels)\n",
    "            self.loss = tf.reduce_mean(cross_entropy)\n",
    "            \n",
    "            if self.use_penalization:\n",
    "                A_T = tf.transpose(self.A, perm=[0, 2, 1])\n",
    "                tile_eye = tf.tile(tf.eye(self.r), [self.batch_size, 1])\n",
    "                tile_eye = tf.reshape(tile_eye, [-1, self.r, self.r])\n",
    "                AA_T = tf.matmul(self.A, A_T) - tile_eye\n",
    "                P = tf.square(tf.norm(AA_T, axis=[-2, -1], ord='fro'))\n",
    "                p_loss = self.p_coef * P\n",
    "                self.loss = self.loss + p_loss\n",
    "            \n",
    "            self.loss = tf.reduce_mean(self.loss)\n",
    "            \n",
    "            params = tf.trainable_variables()\n",
    "            optimizer = tf.train.AdamOptimizer(self.learning_rate)\n",
    "            grad_and_vars = tf.gradients(self.loss, params)\n",
    "            clipped_gradients, _ = tf.clip_by_global_norm(grad_and_vars, 0.5)\n",
    "            self.optimizer = optimizer.apply_gradients(zip(clipped_gradients, params), global_step=self.global_step)\n",
    "            \n",
    "            self.predict = tf.argmax(self.logits, -1)\n",
    "            self.correct_pred = tf.equal(self.predict, self.labels)\n",
    "            self.accuracy = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))\n",
    "            \n",
    "\n",
    "def load_csv(filepath, target_column=-1, data_column=None, has_header=False):\n",
    "    with gfile.Open(filepath) as csv_file:\n",
    "        data_file = csv.reader(csv_file)\n",
    "        if has_header:\n",
    "            header = next(data_file)\n",
    "\n",
    "        data, target = [], []\n",
    "        for i, row in enumerate(data_file):\n",
    "            data.append([_d for _i, _d in enumerate(row) if _i == data_column])\n",
    "            target.append([int(_d) for _i, _d in enumerate(row) if _i == target_column])\n",
    "\n",
    "        return data, target\n",
    "\n",
    "def token_parse(iterator):\n",
    "    for value in iterator:\n",
    "        return TOKENIZER_RE.findall(value)\n",
    "\n",
    "\n",
    "def string_parser(arr, tokenizer, fit):\n",
    "    if fit == False:\n",
    "        return list(tokenizer.transform(arr)), tokenizer\n",
    "    else:\n",
    "        return list(tokenizer.fit_transform(arr)), tokenizer\n",
    "\n",
    "def main():\n",
    "    # Set mode\n",
    "    is_training = False\n",
    "    \n",
    "    # Preparing data\n",
    "    tokenizer = tflearn.data_utils.VocabularyProcessor(MAXLEN, tokenizer_fn=lambda tokens: [token_parse(x) for x in tokens])\n",
    "    sources_raw, labels = load_csv('./data/train.csv', target_column=0, data_column=2)\n",
    "    sources, vocab_processor = string_parser(sources_raw, tokenizer, fit=True)\n",
    "    sources = tflearn.data_utils.pad_sequences(sources, maxlen=MAXLEN)\n",
    "    labels = np.squeeze(labels)\n",
    "        \n",
    "    sources, labels = shuffle(sources, labels)\n",
    "    vocab_size = len(vocab_processor.vocabulary_._mapping)\n",
    "    label_num = int(np.max(labels) + 1)\n",
    "    \n",
    "    # Training options \n",
    "    epoch_nums = 1\n",
    "    batch_size = 80\n",
    "    total = len(sources)\n",
    "    step_nums = int(total/batch_size)\n",
    "    display_step = int(step_nums / 100)\n",
    "    r = 30\n",
    "    \n",
    "    if is_training == True:\n",
    "        keep_prob = 0.8\n",
    "    else:\n",
    "        keep_prob = 1.0\n",
    "        \n",
    "    model = SelfAttenModel(batch_size=batch_size,\n",
    "                           vocab_size=vocab_size,\n",
    "                           label_num=label_num,\n",
    "                           keep_prob=keep_prob,\n",
    "                           learning_rate=0.01,\n",
    "                           p_coef=0.25,\n",
    "                           r = r,\n",
    "                           max_sequence_length=MAXLEN)\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        \n",
    "        # Saver\n",
    "        saver = tf.train.Saver()\n",
    "        ckpt_path = tf.train.latest_checkpoint(checkpoint_dir=SAVE_DIR)\n",
    "        if ckpt_path: \n",
    "            saver.restore(sess, ckpt_path)\n",
    "        else:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            \n",
    "        # train mode\n",
    "        if is_training == True:\n",
    "            for epoch in range(epoch_nums):\n",
    "                print(\"%d Epoch Start\" % epoch)\n",
    "                display_loss = []\n",
    "                display_accuracy = []\n",
    "                for step in range(step_nums):\n",
    "\n",
    "                    batch_start = step * batch_size\n",
    "                    batch_end = batch_start + batch_size\n",
    "                    batch_sources, batch_labels = (sources[batch_start:batch_end], labels[batch_start:batch_end])\n",
    "\n",
    "                    loss, accuracy, predict, _= sess.run([model.loss, model.accuracy, model.predict, model.optimizer], \n",
    "                                                feed_dict={model.sources: batch_sources, \n",
    "                                                           model.labels: batch_labels})\n",
    "                    display_loss.append(loss)\n",
    "                    display_accuracy.append(accuracy)\n",
    "                    \n",
    "                    # display\n",
    "                    if (step % display_step) == 0:\n",
    "                        \n",
    "                        # display training accuracy\n",
    "                        print(\"Step \" + str(step * batch_size) + \", Minibatch Loss= \" + \\\n",
    "                              \"{:.6f}\".format(np.mean(display_loss)) + \", Training Accuracy= \" + \\\n",
    "                              \"{:.5f}\".format(np.mean(display_accuracy)))\n",
    "                        \n",
    "                        display_loss.clear()\n",
    "                        display_accuracy.clear()\n",
    "\n",
    "                        # save weights\n",
    "                        saver.save(sess, SAVE_FILE_PATH)\n",
    "                        \n",
    "                        # display test set accuracy\n",
    "                        test_sources_raw, test_labels = load_csv('./data/test.csv', target_column=0, data_column=2)\n",
    "                        test_sources, vocab_processor = string_parser(test_sources_raw, tokenizer, fit=True)\n",
    "                        test_sources = tflearn.data_utils.pad_sequences(test_sources, maxlen=MAXLEN)\n",
    "                        test_labels = np.squeeze(test_labels)\n",
    "\n",
    "                        test_total = len(test_sources)\n",
    "                        test_step_nums = int(test_total/batch_size)\n",
    "\n",
    "                        accuracy_list = []\n",
    "                        for step in range(test_step_nums):\n",
    "\n",
    "                            batch_start = step * batch_size\n",
    "                            batch_end = batch_start + batch_size\n",
    "                            batch_sources, batch_labels = (test_sources[batch_start:batch_end], test_labels[batch_start:batch_end])\n",
    "\n",
    "                            A, accuracy = sess.run([model.A, model.accuracy], \n",
    "                                                                feed_dict={model.sources: batch_sources,\n",
    "                                                                           model.labels: batch_labels})\n",
    "                            accuracy_list.append(accuracy)\n",
    "\n",
    "                        print(\"Test Accuracy: {:.5f}\".format(np.mean(accuracy_list)))\n",
    "\n",
    "            print(\"Optimization Finished!\")\n",
    "        \n",
    "        # test mode\n",
    "        else:\n",
    "            \n",
    "            test_sources_raw, test_labels = load_csv('./data/test.csv', target_column=0, data_column=2)\n",
    "            test_sources, vocab_processor = string_parser(test_sources_raw, tokenizer, fit=True)\n",
    "            test_sources = tflearn.data_utils.pad_sequences(test_sources, maxlen=MAXLEN)\n",
    "            test_labels = np.squeeze(test_labels)\n",
    "            \n",
    "            test_total = len(test_sources)\n",
    "            test_step_nums = int(test_total/batch_size)\n",
    "    \n",
    "            accuracy_list = []\n",
    "            \n",
    "            step = 3 #for step in range(test_step_nums):\n",
    "            batch_start = step * batch_size\n",
    "            batch_end = batch_start + batch_size\n",
    "            batch_sources, batch_labels = (test_sources[batch_start:batch_end], test_labels[batch_start:batch_end])\n",
    "\n",
    "            A, predicts, accuracy = sess.run([model.A, model.predict, model.accuracy], \n",
    "                                                feed_dict={model.sources: batch_sources,\n",
    "                                                           model.labels: batch_labels})\n",
    "            accuracy_list.append(accuracy)\n",
    "            a_sum = np.sum(A, axis=1)\n",
    "            a_sum_softmax = softmax(a_sum)\n",
    "            print(\"Test Accuracy: {:.5f}\".format(np.mean(accuracy_list)))\n",
    "            \n",
    "            # Visualize\n",
    "            with open(\"visualize.html\", \"w\") as f:\n",
    "                label_dic = [\"None\", \"World\", \"Sports\", \"Business\", \"Sci/Tech\"]\n",
    "                for b_idx in range(batch_start, batch_end):\n",
    "                    sentence = test_sources_raw[b_idx]\n",
    "                    label = test_labels[b_idx]\n",
    "                    tokens = token_parse(sentence)\n",
    "                    \n",
    "                    predict = predicts[b_idx-batch_start]\n",
    "                    aa = A[b_idx-batch_start]\n",
    "                    for r_idx in range(r): # 1 or r\n",
    "                        a = aa[r_idx]\n",
    "\n",
    "                        f.write('<div>')\n",
    "                        f.write('<p>')\n",
    "                        f.write('<span style=\"margin-left: 5px;background-color:rgba(0,255,0,0.4)\">{}</span>\\n'.format(label_dic[label]))\n",
    "                        f.write('<span style=\"margin-left: 5px;background-color:rgba(0,0,255,0.4)\">{}</span>\\n'.format(label_dic[predict]))\n",
    "                        for idx, prob in enumerate(a):\n",
    "                            if idx < len(tokens):\n",
    "                                token = tokens[idx]\n",
    "                            else:\n",
    "                                token = \" \"\n",
    "                            f.write('<span style=\"margin-left: 5px;background-color:rgba(255,0,0,{})\">{}</span> \\n'.format(prob, token))\n",
    "                        f.write('</p>\\n')\n",
    "                        f.write('</div>\\n')\n",
    "                \n",
    "            \n",
    "if __name__ == '__main__':\n",
    "    main()\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}