13b_sentence-embeds_gluon-only.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adaptation of Gluon's Self-attentive Sentence Embedding\n",
    "\n",
    "#### Eryk Wdowiak and Eric Adsetts\n",
    "\n",
    "we are adapting [Gluon tutorial](https://gluon-nlp.mxnet.io/examples/sentence_embedding/self_attentive_sentence_embedding.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import time\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "# import re\n",
    "import string\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "import mxnet as mx\n",
    "import multiprocessing as mp\n",
    "import gluonnlp as nlp\n",
    "\n",
    "from mxnet import gluon, nd, init\n",
    "from mxnet.gluon import nn, rnn\n",
    "from mxnet import autograd, gluon, nd\n",
    "\n",
    "# fixed random number seed\n",
    "np.random.seed(2018)\n",
    "mx.random.seed(2018)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### preliminary data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  load data\n",
    "data = pd.read_csv('dataset/judge-1377884607_tweet_product_company_v2-clean.csv')\n",
    "data.columns = ['tweet','direction','emotion']\n",
    "# data.shape  # (9093, 3)\n",
    "\n",
    "##  remove rows without tweet\n",
    "data = data.dropna(subset=['tweet','emotion'],axis='index')\n",
    "# data.shape  # (9092, 3)\n",
    "\n",
    "##  clean emotions\n",
    "emo_dict = {'Negative emotion':'negative', \n",
    "            'Positive emotion':'positive',\n",
    "            'No emotion toward brand or product':'neutral', \n",
    "            \"I can't tell\":'unknown'}\n",
    "data['emotion'] = data['emotion'].replace(emo_dict)\n",
    "del emo_dict\n",
    "\n",
    "emo_int_dict = {'unknown':0, 'negative':1, 'neutral':2, 'positive':3 } \n",
    "data['emot_int'] = data['emotion'].replace(emo_int_dict)\n",
    "del emo_int_dict\n",
    "\n",
    "##  define company and product\n",
    "##  first convert NaN to a string\n",
    "data['direction'] = data['direction'].map('{}'.format)\n",
    "\n",
    "##  define company\n",
    "comp_dict = {'iPhone':'Apple', \n",
    "             'iPad or iPhone App':'Apple', \n",
    "             'iPad':'Apple', \n",
    "             'Google':'Google', \n",
    "             'nan':'unknown', \n",
    "             'Android':'Google',\n",
    "             'Apple':'Apple',\n",
    "             'Android App':'Google', \n",
    "             'Other Google product or service':'Google',\n",
    "             'Other Apple product or service':'Apple'}\n",
    "data['company'] = data['direction'].replace(comp_dict)\n",
    "del comp_dict\n",
    "\n",
    "##  define product\n",
    "prod_dict = {'iPhone':'device', \n",
    "             'iPad or iPhone App':'software', \n",
    "             'iPad':'device', \n",
    "             'Google':'company', \n",
    "             'nan':'unknown', \n",
    "             'Android':'device',\n",
    "             'Apple':'company',\n",
    "             'Android App':'software', \n",
    "             'Other Google product or service':'other',\n",
    "             'Other Apple product or service':'other'}\n",
    "data['product'] = data['direction'].replace(prod_dict)\n",
    "del prod_dict\n",
    "\n",
    "##  let's take a look\n",
    "# data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  prepare stop word list\n",
    "stopwords_list = []\n",
    "# stopwords_list += stopwords.words('english')\n",
    "stopwords_list += list(string.punctuation)\n",
    "# stopwords_list += ['0','1','2','3','4','5','6','7','8','9']\n",
    "\n",
    "##  how to process tweets\n",
    "def process_tweets(tweet):\n",
    "    tokens = word_tokenize(tweet)\n",
    "    no_stop_tokens = [token.lower() for token in tokens if token.lower() not in stopwords_list]\n",
    "    token_string = ' '.join(no_stop_tokens)\n",
    "    return token_string\n",
    "\n",
    "##  process tweets\n",
    "data['tweet'] = list(map(process_tweets, list(data['tweet'])))\n",
    "\n",
    "##  bind tweets and labels together\n",
    "tweets_labels = [[text, int(label)] for text, label in zip(data['tweet'], data['emot_int'])]\n",
    "\n",
    "##  split between training set and validation set\n",
    "train_dataset, valid_dataset = nlp.data.train_valid_split(tweets_labels, 0.20)\n",
    "# len(train_dataset), len(valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  ClipSequence takes list as input and returns list with max. length 100\n",
    "length_clip = nlp.data.ClipSequence(100)\n",
    "\n",
    "def preprocess(dataset):\n",
    "    ##  get data and label\n",
    "    data, label = dataset[0], dataset[1]\n",
    "    \n",
    "    ##  clip the length of review words\n",
    "    data = length_clip(word_tokenize(data))\n",
    "    return data, label\n",
    "\n",
    "def get_length(dataset):\n",
    "    return float(len(dataset[0]))\n",
    "\n",
    "def preprocess_dataset(dataset):\n",
    "    with mp.Pool() as pool:\n",
    "        # Each sample is processed in an asynchronous manner.\n",
    "        dataset = gluon.data.SimpleDataset(pool.map(preprocess, dataset))\n",
    "        lengths = gluon.data.SimpleDataset(pool.map(get_length, dataset))\n",
    "    return dataset, lengths\n",
    "\n",
    "## preprocess training set and validation set\n",
    "train_dataset, train_data_lengths = preprocess_dataset(train_dataset)\n",
    "valid_dataset, valid_data_lengths = preprocess_dataset(valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  create vocabulary\n",
    "train_seqs = [sample[0] for sample in train_dataset]\n",
    "counter = nlp.data.count_tokens(list(itertools.chain.from_iterable(train_seqs)))\n",
    "\n",
    "##  limit vocabulary to 5,000 words\n",
    "vocab = nlp.Vocab(counter, max_size=5000)\n",
    "\n",
    "##  load pre-trained GloVe (word) embedding with 300 dimensions\n",
    "embedding_weights = nlp.embedding.GloVe(embedding_root='dataset',source='glove.6B.300d')\n",
    "vocab.set_embedding(embedding_weights)\n",
    "# print(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  token to index\n",
    "def token_to_idx(dataset):\n",
    "    return vocab[dataset[0]], dataset[1]\n",
    "\n",
    "##  return token index (or list of token indices) according to the vocabulary\n",
    "with mp.Pool() as pool:\n",
    "    train_dataset = pool.map(token_to_idx, train_dataset)\n",
    "    valid_dataset = pool.map(token_to_idx, valid_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bucketing, mini-batches, and the `DataLoader`\n",
    "\n",
    "Since each sentence may have a different length, we need to use `Pad` to fill the sentences in a mini-batch to equal lengths so that the data can be quickly tensored."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FixedBucketSampler:\n",
      "  sample_num=7273, batch_num=230\n",
      "  key=[11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 41]\n",
      "  cnt=[827, 642, 768, 923, 1009, 976, 840, 682, 386, 155, 51, 12, 1, 1]\n",
      "  batch_size=[39, 33, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/eryk/.local/lib/python3.7/site-packages/gluonnlp/data/sampler.py:354: UserWarning: Some buckets are empty and will be removed. Unused bucket keys=[37, 39]\n",
      "  str(unused_bucket_keys))\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "bucket_num = 16\n",
    "bucket_ratio = 0.33\n",
    "\n",
    "\n",
    "def get_dataloader():\n",
    "\n",
    "    # Construct the DataLoader Pad data, stack label and lengths\n",
    "    batchify_fn = nlp.data.batchify.Tuple(\n",
    "        nlp.data.batchify.Pad(axis=0, pad_val=0),\n",
    "        nlp.data.batchify.Stack())\n",
    "\n",
    "    # In this example, we use a FixedBucketSampler,\n",
    "    # which assigns each data sample to a fixed bucket based on its length.\n",
    "    batch_sampler = nlp.data.sampler.FixedBucketSampler(\n",
    "        train_data_lengths,\n",
    "        batch_size=batch_size,\n",
    "        num_buckets=bucket_num,\n",
    "        ratio=bucket_ratio,\n",
    "        shuffle=True)\n",
    "    print(batch_sampler.stats())\n",
    "\n",
    "    # Training set DataLoader\n",
    "    train_dataloader = gluon.data.DataLoader(\n",
    "        dataset=train_dataset,\n",
    "        batch_sampler=batch_sampler,\n",
    "        batchify_fn=batchify_fn)\n",
    "    # Validation set DataLoader\n",
    "    valid_dataloader = gluon.data.DataLoader(\n",
    "        dataset=valid_dataset,\n",
    "        batch_size=batch_size,\n",
    "        shuffle=False,\n",
    "        batchify_fn=batchify_fn)\n",
    "    return train_dataloader, valid_dataloader\n",
    "\n",
    "train_dataloader, valid_dataloader = get_dataloader()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constructing the model and outlining the model's structure\n",
    "\n",
    "construct the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A custom attention layer\n",
    "class SelfAttention(nn.HybridBlock):\n",
    "    def __init__(self, att_unit, att_hops, **kwargs):\n",
    "        super(SelfAttention, self).__init__(**kwargs)\n",
    "        with self.name_scope():\n",
    "            self.ut_dense = nn.Dense(att_unit, activation='tanh', flatten=False)\n",
    "            self.et_dense = nn.Dense(att_hops, activation=None, flatten=False)\n",
    "\n",
    "    def hybrid_forward(self, F, x):\n",
    "        # x shape: [batch_size, seq_len, embedding_width]\n",
    "        # ut shape: [batch_size, seq_len, att_unit]\n",
    "        ut = self.ut_dense(x)\n",
    "        # et shape: [batch_size, seq_len, att_hops]\n",
    "        et = self.et_dense(ut)\n",
    "\n",
    "        # att shape: [batch_size,  att_hops, seq_len]\n",
    "        att = F.softmax(F.transpose(et, axes=(0, 2, 1)), axis=-1)\n",
    "        # output shape [batch_size, att_hops, embedding_width]\n",
    "        output = F.batch_dot(att, x)\n",
    "\n",
    "        return output, att"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When the number of samples for labels are very unbalanced, applying different weights on different labels may improve the performance of the model significantly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WeightedSoftmaxCE(nn.Block):\n",
    "    def __init__(self, sparse_label=True, from_logits=False,  **kwargs):\n",
    "        super(WeightedSoftmaxCE, self).__init__(**kwargs)\n",
    "        with self.name_scope():\n",
    "            self.sparse_label = sparse_label\n",
    "            self.from_logits = from_logits\n",
    "\n",
    "    def forward(self, pred, label, class_weight, depth=None):\n",
    "        if self.sparse_label:\n",
    "            label = nd.reshape(label, shape=(-1, ))\n",
    "            label = nd.one_hot(label, depth)\n",
    "        if not self.from_logits:\n",
    "            pred = nd.log_softmax(pred, -1)\n",
    "\n",
    "        weight_label = nd.broadcast_mul(label, class_weight)\n",
    "        loss = -nd.sum(pred * weight_label, axis=-1)\n",
    "\n",
    "        # return nd.mean(loss, axis=0, exclude=True)\n",
    "        return loss\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now define the basic model characteristics in a self-attentive bi-LSTM model, and configure the layers and dropout, as well as how the model feeds forward."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SelfAttentiveBiLSTM(nn.HybridBlock):\n",
    "    def __init__(self, vocab_len, embsize, nhidden, nlayers, natt_unit, natt_hops, nfc, nclass,\n",
    "                 drop_prob, pool_way, prune_p=None, prune_q=None, **kwargs):\n",
    "        super(SelfAttentiveBiLSTM, self).__init__(**kwargs)\n",
    "        with self.name_scope():\n",
    "            self.embedding_layer = nn.Embedding(vocab_len, embsize)\n",
    "            self.bilstm = rnn.LSTM(nhidden, num_layers=nlayers, dropout=drop_prob, bidirectional=True)\n",
    "            self.att_encoder = SelfAttention(natt_unit, natt_hops)\n",
    "            self.dense = nn.Dense(nfc, activation='tanh')\n",
    "            self.output_layer = nn.Dense(nclass)\n",
    "\n",
    "            self.dense_p, self.dense_q = None, None\n",
    "            if all([prune_p, prune_q]):\n",
    "                self.dense_p = nn.Dense(prune_p, activation='tanh', flatten=False)\n",
    "                self.dense_q = nn.Dense(prune_q, activation='tanh', flatten=False)\n",
    "\n",
    "            self.drop_prob = drop_prob\n",
    "            self.pool_way = pool_way\n",
    "\n",
    "    def hybrid_forward(self, F, inp):\n",
    "        # input_embed: [batch, len, emsize]\n",
    "        inp_embed = self.embedding_layer(inp)\n",
    "        h_output = self.bilstm(F.transpose(inp_embed, axes=(1, 0, 2)))\n",
    "        # att_output: [batch, att_hops, emsize]\n",
    "        att_output, att = self.att_encoder(F.transpose(h_output, axes=(1, 0, 2)))\n",
    "\n",
    "        dense_input = None\n",
    "        if self.pool_way == 'flatten':\n",
    "            dense_input = F.Dropout(F.flatten(att_output), self.drop_prob)\n",
    "        elif self.pool_way == 'mean':\n",
    "            dense_input = F.Dropout(F.mean(att_output, axis=1), self.drop_prob)\n",
    "        elif self.pool_way == 'prune' and all([self.dense_p, self.dense_q]):\n",
    "            # p_section: [batch, att_hops, prune_p]\n",
    "            p_section = self.dense_p(att_output)\n",
    "            # q_section: [batch, emsize, prune_q]\n",
    "            q_section = self.dense_q(F.transpose(att_output, axes=(0, 2, 1)))\n",
    "            dense_input = F.Dropout(F.concat(F.flatten(p_section), F.flatten(q_section), dim=-1), self.drop_prob)\n",
    "\n",
    "        dense_out = self.dense(dense_input)\n",
    "        output = self.output_layer(F.Dropout(dense_out, self.drop_prob))\n",
    "\n",
    "        return output, att"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuring the parameters and assembling the model\n",
    "\n",
    "The resulting `M` is a matrix, and the way to classify this matrix is `flatten`-ing, `mean`-ing or `prune`-ing. Pruning is an effective way of trimming parameters that was proposed in the original paper, and has been implemented for our example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_len = len(vocab)\n",
    "emsize = 300           # word embedding size\n",
    "nhidden = 150    # 300  # lstm hidden_dim\n",
    "nlayers = 2            # lstm layers\n",
    "natt_unit = 150  # 300  # the hidden_units of attention layer\n",
    "natt_hops = 2          # the channels of attention\n",
    "nfc = 256       # 512\n",
    "nclass = 4\n",
    "\n",
    "drop_prob = 0.5\n",
    "pool_way = 'flatten'  # The way to handle M\n",
    "prune_p = None\n",
    "prune_q = None\n",
    "\n",
    "model = SelfAttentiveBiLSTM(vocab_len, emsize, nhidden, nlayers,\n",
    "                            natt_unit, natt_hops, nfc, nclass,\n",
    "                            drop_prob, pool_way, prune_p, prune_q)\n",
    "\n",
    "model.initialize(init=init.Xavier(), ctx=mx.cpu())\n",
    "model.hybridize()\n",
    "\n",
    "# Attach a pre-trained glove word vector to the embedding layer\n",
    "model.embedding_layer.weight.set_data(vocab.embedding.idx_to_vec)\n",
    "# fixed the layer\n",
    "model.embedding_layer.collect_params().setattr('grad_req', 'null')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "standard loss function below includes penalty coefficient "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_loss(x, y, model, loss, class_weight, penal_coeff):\n",
    "    pred, att = model(x)\n",
    "    if loss_name == 'sce':\n",
    "        l = loss(pred, y)\n",
    "    elif loss_name == 'wsce':\n",
    "        l = loss(pred, y, class_weight, class_weight.shape[0])\n",
    "\n",
    "    # penalty\n",
    "    diversity_penalty = nd.batch_dot(att, nd.transpose(att, axes=(0, 2, 1))\n",
    "                        ) - nd.eye(att.shape[1], ctx=att.context)\n",
    "    l = l + penal_coeff * diversity_penalty.norm(axis=(1, 2))\n",
    "\n",
    "    return pred, l"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "define what one epoch of training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_epoch(data_iter, model, loss, trainer, ctx, is_train, epoch,\n",
    "              penal_coeff=0.0, clip=None, class_weight=None, loss_name='wsce'):\n",
    "\n",
    "    loss_val = 0.\n",
    "    total_pred = []\n",
    "    total_true = []\n",
    "    n_batch = 0\n",
    "\n",
    "    for batch_x, batch_y in data_iter:\n",
    "        batch_x = batch_x.as_in_context(ctx)\n",
    "        batch_y = batch_y.as_in_context(ctx)\n",
    "\n",
    "        if is_train:\n",
    "            with autograd.record():\n",
    "                batch_pred, l = calculate_loss(batch_x, batch_y, model, loss, class_weight, penal_coeff)\n",
    "\n",
    "            # backward calculate\n",
    "            l.backward()\n",
    "\n",
    "            # clip gradient\n",
    "            clip_params = [p.data() for p in model.collect_params().values()]\n",
    "            if clip is not None:\n",
    "                norm = nd.array([0.0], ctx)\n",
    "                for param in clip_params:\n",
    "                    if param.grad is not None:\n",
    "                        norm += (param.grad ** 2).sum()\n",
    "                norm = norm.sqrt().asscalar()\n",
    "                if norm > clip:\n",
    "                    for param in clip_params:\n",
    "                        if param.grad is not None:\n",
    "                            param.grad[:] *= clip / norm\n",
    "\n",
    "            # update parmas\n",
    "            trainer.step(batch_x.shape[0])\n",
    "\n",
    "        else:\n",
    "            batch_pred, l = calculate_loss(batch_x, batch_y, model, loss, class_weight, penal_coeff)\n",
    "\n",
    "        # keep result for metric\n",
    "        batch_pred = nd.argmax(nd.softmax(batch_pred, axis=1), axis=1).asnumpy()\n",
    "        batch_true = np.reshape(batch_y.asnumpy(), (-1, ))\n",
    "        total_pred.extend(batch_pred.tolist())\n",
    "        total_true.extend(batch_true.tolist())\n",
    "\n",
    "        batch_loss = l.mean().asscalar()\n",
    "\n",
    "        n_batch += 1\n",
    "        loss_val += batch_loss\n",
    "\n",
    "        # check the result of traing phase\n",
    "        if is_train and n_batch % 25 == 0:\n",
    "            print('epoch %d, batch %d, batch_train_loss %.4f, batch_train_acc %.3f' %\n",
    "                  (epoch, n_batch, batch_loss, accuracy_score(batch_true, batch_pred)))\n",
    "\n",
    "    # metric\n",
    "    F1 = f1_score(np.array(total_true), np.array(total_pred), average='weighted')\n",
    "    acc = accuracy_score(np.array(total_true), np.array(total_pred))\n",
    "    loss_val /= n_batch\n",
    "\n",
    "    if is_train:\n",
    "        # save parameters\n",
    "        ot_file = 'params/params.' + f'{epoch:02}'\n",
    "        model.save_parameters(ot_file)\n",
    "        print('saved parameters to: ' + ot_file)\n",
    "\n",
    "        ##  print statistics\n",
    "        print('epoch %d, learning_rate %.5f \\n\\t train_loss %.4f, acc_train %.3f, F1_train %.3f, ' %\n",
    "              (epoch, trainer.learning_rate, loss_val, acc, F1))\n",
    "        \n",
    "        ## reduce the learning rate\n",
    "        if epoch % 2 == 0:\n",
    "            trainer.set_learning_rate(trainer.learning_rate * 0.9)\n",
    "    else:\n",
    "        ##  print statistics\n",
    "        print('\\t valid_loss %.4f, acc_valid %.3f, F1_valid %.3f, ' % (loss_val, acc, F1))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In addition, we include a helper method `train_valid` which combines the one epoch for the training data as well as the validation data, using the `is_train` boolean to swap between the two modes we discussed above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_valid(data_iter_train, data_iter_valid, model, loss, trainer, ctx, nepochs,\n",
    "                penal_coeff=0.0, clip=None, class_weight=None, loss_name='wsce'):\n",
    "    \n",
    "    print('-'*40)\n",
    "    for epoch in range(1, nepochs+1):\n",
    "        print('BEGIN epoch '+str(epoch))\n",
    "        start = time.time()\n",
    "        # train\n",
    "        is_train = True\n",
    "        one_epoch(data_iter_train, model, loss, trainer, ctx, is_train,\n",
    "                  epoch, penal_coeff, clip, class_weight, loss_name)\n",
    "\n",
    "        # valid\n",
    "        is_train = False\n",
    "        one_epoch(data_iter_valid, model, loss, trainer, ctx, is_train,\n",
    "                  epoch, penal_coeff, clip, class_weight, loss_name)\n",
    "        end = time.time()\n",
    "        print('time %.2f sec' % (end-start))\n",
    "        print('END epoch '+str(epoch))\n",
    "        print('-'*40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the model\n",
    "\n",
    "Now that we are actually training the model, we use `WeightedSoftmaxCE` to alleviate the problem of data categorical imbalance. We perform statistical analysis on the data in advance to retrieve a set of `class_weight`s."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_weight = None\n",
    "loss_name = 'wsce'\n",
    "optim = 'adam'\n",
    "lr = 0.000729\n",
    "penal_coeff = 0.05\n",
    "clip = 0.5\n",
    "nepochs = 4\n",
    "\n",
    "trainer = gluon.Trainer(model.collect_params(), optim, {'learning_rate': lr})\n",
    "\n",
    "if loss_name == 'sce':\n",
    "    loss = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "elif loss_name == 'wsce':\n",
    "    loss = WeightedSoftmaxCE()\n",
    "    ##  the value of class_weight is obtained by counting data in advance. \n",
    "    ##  It can be seen as a hyperparameter.\n",
    "    class_weight = nd.array([0.02, 1.0, 3.0, 2.0], ctx=mx.cpu())\n",
    "    \n",
    "##  data['emot_int'].value_counts() / data['emot_int'].value_counts().sum()\n",
    "##  2    0.592609\n",
    "##  3    0.327541\n",
    "##  1    0.062692\n",
    "##  0    0.017158\n",
    "##  Name: emot_int, dtype: float64\n",
    "\n",
    "##  data['emotion'].value_counts() / data['emotion'].value_counts().sum()\n",
    "##  neutral     0.592609\n",
    "##  positive    0.327541\n",
    "##  negative    0.062692\n",
    "##  unknown     0.017158\n",
    "##  Name: emotion, dtype: float64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  train and validate\n",
    "\n",
    "We've simplified our lives earlier by creating the necessary helper methods so our training is as simple as the below line of code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "BEGIN epoch 1\n",
      "epoch 1, batch 25, batch_train_loss 3.4905, batch_train_acc 0.156\n",
      "epoch 1, batch 50, batch_train_loss 2.5511, batch_train_acc 0.303\n",
      "epoch 1, batch 75, batch_train_loss 1.8807, batch_train_acc 0.688\n",
      "epoch 1, batch 100, batch_train_loss 1.8038, batch_train_acc 0.656\n",
      "epoch 1, batch 125, batch_train_loss 2.0564, batch_train_acc 0.406\n",
      "epoch 1, batch 150, batch_train_loss 1.7180, batch_train_acc 0.688\n",
      "epoch 1, batch 175, batch_train_loss 2.0205, batch_train_acc 0.406\n",
      "epoch 1, batch 200, batch_train_loss 1.6888, batch_train_acc 0.625\n",
      "epoch 1, batch 225, batch_train_loss 1.6218, batch_train_acc 0.656\n",
      "saved parameters to: params/params.01\n",
      "epoch 1, learning_rate 0.00073 \n",
      "\t train_loss 2.1333, acc_train 0.527, F1_train 0.474, \n",
      "\t valid_loss 1.8561, acc_valid 0.618, F1_valid 0.473, \n",
      "time 198.15 sec\n",
      "END epoch 1\n",
      "----------------------------------------\n",
      "BEGIN epoch 2\n",
      "epoch 2, batch 25, batch_train_loss 2.1851, batch_train_acc 0.531\n",
      "epoch 2, batch 50, batch_train_loss 0.3891, batch_train_acc 1.000\n",
      "epoch 2, batch 75, batch_train_loss 2.0014, batch_train_acc 0.625\n",
      "epoch 2, batch 100, batch_train_loss 1.8494, batch_train_acc 0.594\n",
      "epoch 2, batch 125, batch_train_loss 1.4238, batch_train_acc 0.781\n",
      "epoch 2, batch 150, batch_train_loss 1.7018, batch_train_acc 0.594\n",
      "epoch 2, batch 175, batch_train_loss 1.5640, batch_train_acc 0.688\n",
      "epoch 2, batch 200, batch_train_loss 1.5930, batch_train_acc 0.719\n",
      "epoch 2, batch 225, batch_train_loss 2.0508, batch_train_acc 0.594\n",
      "saved parameters to: params/params.02\n",
      "epoch 2, learning_rate 0.00073 \n",
      "\t train_loss 1.9163, acc_train 0.579, F1_train 0.438, \n",
      "\t valid_loss 1.8823, acc_valid 0.618, F1_valid 0.473, \n",
      "time 202.07 sec\n",
      "END epoch 2\n",
      "----------------------------------------\n",
      "BEGIN epoch 3\n",
      "epoch 3, batch 25, batch_train_loss 1.9758, batch_train_acc 0.375\n",
      "epoch 3, batch 50, batch_train_loss 1.8443, batch_train_acc 0.656\n",
      "epoch 3, batch 75, batch_train_loss 1.8851, batch_train_acc 0.590\n",
      "epoch 3, batch 100, batch_train_loss 2.5400, batch_train_acc 0.312\n",
      "epoch 3, batch 125, batch_train_loss 2.5302, batch_train_acc 0.562\n",
      "epoch 3, batch 150, batch_train_loss 2.0980, batch_train_acc 0.375\n",
      "epoch 3, batch 175, batch_train_loss 1.9755, batch_train_acc 0.531\n",
      "epoch 3, batch 200, batch_train_loss 1.8859, batch_train_acc 0.515\n",
      "epoch 3, batch 225, batch_train_loss 2.0024, batch_train_acc 0.556\n",
      "saved parameters to: params/params.03\n",
      "epoch 3, learning_rate 0.00066 \n",
      "\t train_loss 1.9630, acc_train 0.579, F1_train 0.453, \n",
      "\t valid_loss 2.0008, acc_valid 0.618, F1_valid 0.473, \n",
      "time 202.22 sec\n",
      "END epoch 3\n",
      "----------------------------------------\n",
      "BEGIN epoch 4\n",
      "epoch 4, batch 25, batch_train_loss 2.0820, batch_train_acc 0.500\n",
      "epoch 4, batch 50, batch_train_loss 1.8559, batch_train_acc 0.656\n",
      "epoch 4, batch 75, batch_train_loss 1.9499, batch_train_acc 0.562\n",
      "epoch 4, batch 100, batch_train_loss 1.8028, batch_train_acc 0.564\n",
      "epoch 4, batch 125, batch_train_loss 1.8654, batch_train_acc 0.469\n",
      "epoch 4, batch 150, batch_train_loss 1.6157, batch_train_acc 0.688\n",
      "epoch 4, batch 175, batch_train_loss 2.0368, batch_train_acc 0.594\n",
      "epoch 4, batch 200, batch_train_loss 7.2409, batch_train_acc 0.438\n",
      "epoch 4, batch 225, batch_train_loss 2.9550, batch_train_acc 0.500\n",
      "saved parameters to: params/params.04\n",
      "epoch 4, learning_rate 0.00066 \n",
      "\t train_loss 2.3300, acc_train 0.585, F1_train 0.437, \n",
      "\t valid_loss 1.9514, acc_valid 0.618, F1_valid 0.473, \n",
      "time 199.47 sec\n",
      "END epoch 4\n",
      "----------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# train and validate\n",
    "train_valid(train_dataloader, valid_dataloader, model, loss, trainer, mx.cpu(), nepochs,\n",
    "            penal_coeff=penal_coeff, clip=clip, class_weight=class_weight, loss_name=loss_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictions and sampling using our model\n",
    "\n",
    "Now that the model has been trained, we can randomly input a sentence into the model and predict its emotional value tag. The range of emotional markers (or the labels) is zero, one or two -- negative, neutral, positive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2]\n",
      "\n",
      "[[[0.1961866  0.26331416 0.26740205 0.27309716]\n",
      "  [0.00527297 0.02690275 0.2075112  0.76031303]]]\n",
      "<NDArray 1x2x4 @cpu(0)>\n"
     ]
    }
   ],
   "source": [
    "input_ar = nd.array(vocab[['this','phone','is','amazing']], ctx=mx.cpu()).reshape((1, -1))\n",
    "pred, att = model(input_ar)\n",
    "\n",
    "label = np.argmax(nd.softmax(pred, axis=1).asnumpy(), axis=1)\n",
    "print(label)\n",
    "print(att)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to intuitively understand the role of the attention mechanism, we visualize the output of the model's attention on the predicted samples using the `matplotlib` and `seaborn` modules."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAboAAABZCAYAAABIUOEdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHm9JREFUeJzt3Xl8VNUZ8PHfM5MFgZCEJCA7hH2RXUBQQAuCti5oVWzdXhcqiLZUrcUVra1L36oVrYpKXZHyYkVEAVkEBIqyCLLIvsgiS0KAsGSd5/3j3gyTkMlMJHue7+czn8y995y5557c5Mw999zniKpijDHGVFWe8i6AMcYYU5qsoTPGGFOlWUNnjDGmSrOGzhhjTJVmDZ0xxpgqzRo6Y4wxVZo1dMYYY8qciAwVkU0islVE/lzI9hdFZLX72iwiRwK25QZsmx5yX/YcnTHGmLIkIl5gMzAY2AMsB25U1Q1B0t8LdFPV293l46paO9z92RWdMcaYstYL2Kqq21U1C5gMXFVE+huBj37uziJ+bsZwvTX/G7tkLGXN6yWUdxGqPI+UdwmqB4/HvnuXtoHtk0vtbN5y4RAFaL14dqh9NAJ2ByzvAXoXllBEmgEtgPkBq2uIyAogB3hWVacVtbNSb+iMMcZUEx4vACIyAhgRsGWCqk4IWC6sIQx2UTQcmKqquQHrmqrqPhFJBuaLyFpV3RasWNbQGWOMKRGe6CgA3EZtQhFJ9wBNApYbA/uCpB0O3BO4QlX3uT+3i8gCoBsQtKGzfgJjjDElQmrUQGrUCCfpcqC1iLQQkSicxuyM0ZMi0haIB/4XsC5eRKLd94lAP6DQQSx57IrOGGNMiZDIyLDSqWqOiIwGZgNeYKKqrheRp4AVqprX6N0ITNb8jwe0B94QER/OxdqzwUZr5rGGzhhjTInw1IgOO62qfgF8UWDd4wWWxxWSbylwXnHKZQ2dMcaYEiHuPbqKxho6Y4wxJSLcrsuyZoNRjDHGlIhiDEYJGQLMTXO9iGwQkfUiMilg/a0issV93RpqX9X6im7H+u+ZN+UDVH107jeA3kOuyLd9+dyZrF2yEPF6qVk7hqE330lsQmI5lbbyWP/dCqZOnIDP56PfLy7l0muuz7d93vRPWDpvNh6Pl9qxsdw06g8k1KsHwOFDB/nwtZdJSzmEiDDqkSdJqFe/PA6jQlu/agVTJk5AfT76DbqUIQXqeO70T1gydzZer5fadWK5+R6njjetXcPUf7/pT7d/7x7u+ONDdO19QVkfQoW3btUKprz1Oj6fjwsHD2XotfnreM6n/2XJnFl43Dq+9d4xJNSrz6a1a5jy9umR9fv37uau+/9M1z59y/oQypwnzK5LNwTYqwSEABOR6YGDSkSkNTAW6KeqaSJSz11fF3gC6Inz7N1KN29asP1V24bO5/MxZ/J7XH/fn4iJr8v7zz5By87dSWzQyJ+mfpNmdB37JJFR0Xy3cB4LP5nMlXeOLsdSV3y+3FymvPka9z7+NHEJiTz/0BjOO78PDZo09adp0iKZh55/iajoGiya9TnT3p/IHfc7X+jeG/8CQ669gfZdupFx6hQeC0lyBl9uLpPffI37nnia+IREnv3TGDoXUsdj/+7U8cJZn/PJexO584E/0/a8LjzywisAnEhP5/F77qRD127ldSgVli83l4/eeJU/PPk34hMSeebB39O5V28aNmnmT9M0uSUD/vGyU8czZ/DxuxMZ8eBY2p7XhcdeehVw6vjRkbfToVv38jqUMiXRYQ9G8YcAAxCRvBBggaMn7wJezWvAVPWgu34IMEdVD7t55wBDKSJEWLXtuvxp5zbik+oRl1QPb0QE7Xr2YeuaVfnSNG3bgcgo5xfXMLkl6WlBvzAY186tm0k6tyGJ5zYgIjKSHhf25/vly/KlaXNeF6Kine6NFm3acSQ1BYCfdv9Ibm4u7bs4/3hrnHOOP505befWzSQ1aEiSW8c9L+zPmm/z13HbgDpObtOONLeOA63632I6dutpdVyIHVs2Uy9fHQ9gzTfB67hF29PncaCVS7+mU/fqU8cSFYlEhXWfrrAQYI0KpGkDtBGRJSKyTESGFiNvPiGv6EQkFqe1bIRzmbgPmK2qR4rMWMEdP5JGTPzpGJEx8XX5aUfQB+tZu2QRyR07l0XRKrUjh1OJTzzdvRtXN5GdWzYFTb903pd06N4TgIP79lKzVi0mPP80qQcP0Pa8rlx90214vN5SL3dlciQ1lfiALvT4hER2FFHHS+Z9SUe3jgOtWLyIQVcOK5UyVnZHDqcQn5jkXw5Zx3OtjgE87hVdCYUAiwBaAwNxIqd8LSKdwsybv1xFbRSRW4BV7o5qArWAi3H6RG8pKm+FV1i1BOklW//NEvbv2sH5gy8v1SJVCYVM+yRB6vXbhfP5cdsWBl11LQC5vly2/rCea265gz899xKpB/az7Ku5pVnaSkkLOXmDdfB+s3A+u7ZuYfDV1+Zbf/TwYfb9uJMOXatHl1qxFSMU/bIF89m1dTOXDjuzjvfu2kHHbj1KuHAVl0RHI9HRqOoEVe0Z8CoYDiycEGB7gE9VNVtVdwCbcBq+4oQPA0J3XT4C9FDVkar6tPu6G+cm4KPBMonICBFZISIrFs0oMqh0uakdH096Wqp/OT3tMLVj489It/OHdSybNZ1hI8cQUUGHzlYkcQmJpKWc7sI5cjiF2Lpnzq6wcc13zPr4P9w99nEi3XqNT0ikSYuWJJ7bAK/XS+deF7C7iKvs6io+ITFfV2RaauF1/MOa75g19T+MDKjjPCuXfk3X3hfgjai2t+mL5JzHh/zLaakpxAWp45lTJzPq4XFERuYfiLFiySK69u5brepYoqKQqLAGpIQTAmwazoVVXqivNsB2nGgql7qhwOKBS911QYVq6ITCv9v4CP4lMl9r3v9XV4fYRflo0CyZtIMHOJJyiNycHDauWEarzvlvyh/YvZMvJ73DNSPHUKtOnXIqaeXSrFUbDv60l5QD+8nJzmbl4kWc1zP/7Bu7t2/jozde4e4/P05MbNzpvC1bc/L4cdKPHgVg87o1nNu4KSa/gnW8YvEiOp9/Zh1Pev0VRo59nDpxcWd8xvKvF9LzwgFlVeRKp3nrNhz8aV9AHS+kS68++dL8uH0rH/zrZUY9/ESQOl5Ar/4Dy6jEFYNER4X10Liq5gB5IcB+AKbkhQATkSvdZLOBVBHZAHwFPKiqqe4glL/gNJbLgafyBqYEE+qrxl+BVSLyJadv/jXFGRL6dMijqcA8Xi+Dht/C1PHP4/Mp5/XtT2LDxiz+7GPObdqCVl26s+DjyWRnZvDpm84otTrxCVwzakw5l7xi83q9XH/nSF79y2P4fD4uuGQwDZs2Y8ZH79O0VWs6n9+HT957m8yMDN76xzMA1E1M4u6xT+Dxehl26x28PO5hQGmS3Ip+g4aU7wFVQF6vl+F3jmT8U04d9/2FU8efffQ+TVu2pkuvPnzs1vGb/9ep4/jEJEY9/AQAqQcPkJaaQuuOxYqiVK14vV6G3zWSfz75KL7cXPoNupSGTZsxfdJ7NGvVxqnjd5w6nvD83wCom5TEPY+MAyDlwAHSUqpfHZdkCDA3vuUf3VfBvBOBieHuS7SQeyr5EjiXhkNwBqMITv/o7KKeWQhkE6+WPpt4tfTZUw5lwyZeLX2lOfHqkSmfKEDc9cMq1F9MyM5j90G9bqr6UOB6EXmu4DpjjDHVlxTjiq4shfv1aXAh6y4ryYIYY4yp3CQqGokKr7ELJwSYm+7XIqIi0tNdbi4ip0Rktft6PdS+iryiE5GRwCigpYh8H7ApBlga1tEYY4ypFsK9RxdOCDA3XQxwH/BNgY/Ypqpdwy1XqK7LScBM4BkgsMVNDzXKxRhjTPVSjNkLwgkBBs7oyueBB86mXEV2XarqUVXdCeSo6q6A12ERef9sdmyMMaZqCffxAsII4yUi3YAmqjqjkPwtROQ7EVkoIheF2lm4TzJ2LFCACKD6PO5vjDEmJI87Rc/ZhgATEQ/wInBbIel+ApqqaqqI9ACmiUhHVT0WrFyh7tGNBR4GzhGRwA/JBgqGdDHGGFON5XVduo1aUW1EqDBeMUAnYIE4MQTPBaaLyJWqugLIdPezUkS24URNWRFsZ0U2dKr6DPCMiDyD00/aBsgLw23PxxljjPErxuMF/hBgwF6cEGC/yduoqkcBf+RyEVkAPKCqK0QkCTisqrkikowT/3J7UTsLt+tyO7AIp9VdDfQB/gdcEmZ+Y4wxVVyY9+dQ1RwRyQsB5gUm5oUAA1aoasG4l4H6A0+JSA6QC9wdanBkyMgoACKyFjgfWKaqXUWkHfCkqt4QKu/Aca/YlV8pqxnmyWV+vhv6WZT/stD34UfKuwhVXuvFs0stakl6eroCxMTEVKjIKOE+MJ6hqhkAIhKtqhuBtqVXLGOMMaZkhNt1uUdE4nCmTZgjImmEmP/HGGNM9ZLlqZhTmYV1Raeqw1T1iKqOAx4D3gYq5vw7xhhjykVmdg6Z2TlhpQ0VAkxE7haRtW6Yr8Ui0iFg21g33yYRCTnFSbFnBFTVhcXNY4wxpurLyg27kQsnBNgkVX3dTX8l8AIw1G3whuM8390QmCsibVQ1N9j+bE4MY4wxJaIYV3T+EGCqmgXkhQDzK/AAeC1OP9J2FTBZVTNVdQew1f28oKrPHO/GGGNKVbjdlhQeAqx3wUQicg/OxKtRnH6crRGwrEDeRhTBruiMMcaUiLwrOhEZISIrAl4jCiQtMgSYf4Xqq6raEngIeLQ4eQPZFZ0xxpgSkZXj3CYrgRBgBU0GXvuZeatmQ9erVVNGD70Ir0f4fNUGJi1elW97pNfD2GGDadswiaMnM3hq6mz2H0kH4DcX9uCX3duT61PGz/ya5dt+BGDyH27hZGY2PvWR61N+N2EKALdf3Jt+7VqgqqSdOMWz0+aRmn6ibA+4nPVIbszvBvXF4xFmr97I/1u2Jt/2CK+HB351Ma0aJJJ+KpNnps3l4NHjdGveiNsG9iLS6yU7N5eJX33Dml37OCcqkudvusKfPzGmNl+t38KEuf8r60OrsLatW8Ocye+jPh9dLhpI38uuzLf9my+/YPXir/B4vNSMqcOvbruL2IQkACa/9Bx7t2+lSas2XH/fg+VR/EqhZu+eJP3+bvB4OTZjJmkfTMm3PfHe31GzexfACX3ljYtj+2XXAhBRP4l6D40hsl4Sqsq+Bx8jZ/+BMj+GspaZnR1u0iJDgAGISGtV3eIu/hLIez8dmCQiL+AMRmkNfFvUzqpcQ+cR4feXD+CB9z/l0LHjvH7X9SzZtINdh9L8aS7v3oHjGZn89uUPuKRTa0YM6stTU2fTLCmeSzq15rZXJ5EQU4t/3HI1N4//AJ8bPWbMu59w9GRGvv1NXrqKiV85cwJe07sztw44nxdmLCiz4y1vHhFGXXohj0z+nJRjJ3jptmEs27KL3alH/GmGdGnH8YxM7nz9P/Rv35LbB/bm2U/ncfRUBk9Onc3h4ydplhjPX4Zfzi2vfMiprGzunfhff/5/3jaMpZt2lMfhVUg+n4/Zk97hxjFjqRNfl3//9TFad+lOUsPG/jT1mzbj9keeJjI6mpUL5jJ/6kcM+919APQe8ktysrL4buG88jqEis/jIemP97B3zFhyDqbQ9K3xnFi8jKydP/qTpIx/w/8+9toriW7Tyr9c/9EHSXt3MidXrELOqQG+6hEgKjMnvHt0YYYAGy0ig3AmEUgDbnXzrheRKThz1+UA9xQ14hKq4D26do3qs/fwUX5KO0ZOro/567bQr21yvjT92iYza/VGABZu2EqP5Mb+9fPXbSE718f+I+nsPXyUdo3qF7m/k5mnv8HUiIwknJBqVUmbhknsSzvK/iPp5Ph8LPphGxe0aZ4vTZ/WzZi7bjMAizdup0tz577x9gOpHD5+EoBdKWlERXiJ8OY/JRvG1yGu1jms272/9A+mkti3YxvxSfWJT6qHNyKCDuf3YcvqlfnSNG/XkchoJ8Buo+RWpKedDgXYon0nomrUwARXo31bsvfsI2fffsjJIX3uAmpdeEHQ9DGDLub4nAUARDVvini9nFzh9CTpqQw0M7Msil3usrJzycouss3xU9UvVLWNqrZU1b+66x7Pi3Opqr9X1Y6q2lVVL1bV9QF5/+rma6uqM0Ptq8pd0SXVqcWhY+n+5UPHjtOhcf2gaXJ9yvGMLGJr1iCpTi027NmfL29SnVoAqMLfb74SVfhs5XpmrPTXOXdc0ochXdpyIjOLP7zzSWkeXoWTULsWKcdOd9WmpJ+gbcN6+dPE1OKQm8anysnMLOqcE82xU6f/+Pu1bcG2/Snk5Pry5R3QoRWLfthWikdQ+aQfOUydugn+5Zj4uuzbEbyO1ixeQHKnLmVRtCojIimBnIOH/Ms5h1Ko0aFd4Wnr1yOyQX1OrloNQGSTRvjST9Dgr48R0eBcTq34jpTXJ4LPV2j+qiTcK7qy9rOv6ERkcEkWpDSFc5EVLE3e+tETP2bEG1N46MPPuPr88+jcrKE/zdvzl3H9i+8y5/vNDOvVuQRKXHlIIeOfCl7Vhhoi1TQxntsv7s34WV+fkW5Ah5Ys3GANXT6FnquFx9Bdt2wxP+3cTp8hvyrVIlU5hZ/YhSaNGTSQ4wsW+xsy8Xqp0aUTh159k9133UtkwwbUuazS/Ls8KxnZOWSE/4hBmTmbrsu3g20IHFq6b+WSs9hF8R06doKkOjH+5aQ6tUkpMDgkMI3XI9SuEcWxUxlF5s0bYHLkxCkWb9xO+0K6NOet3cyADi1L/JgqspT0EyS6V70AiTG1/N2RgWnyrow9ItSMjiLdvZpLiKnFY9cO5h+ffeUfEJSnRb26eD3C1v0ppXwUlUtMfF2OHU71L6enHSYmLu6MdDs2rGPJ559y3ej7iYismDEIK6qcgylE1EvyL0ckJZKTklpo2tq/GED63AWn8x5KIXPLVqfbM9fH8a+XEt22VaF5q5qs7ByySi4EWH8RWSUiOSLy6wLbct3QYKtFpKgpfYAQDZ2ITA/y+gxICJZPVSeoak9V7dmwR79QZShRm/YdoHFCLOfGxRDh9XBJp9ZnDGRYumkHQ7s63RADOrRi1Y49/vWXdGpNpNfDuXExNE6IZePeA9SIjOCcKOcfRY3ICHq2bMKOg85J36hurP9z+7ZtwY8paVQnm/cdomF8LPVjY4jweOjfviXLtuzKl+abLbsY1KkNABe2S+b7XXsBqBUdxZPXDeWdBcvZsPfMEWkDOrRigV3NnaFh82TSDu7nyKGD5ObksGH5Mlp36ZEvzf4fdzLzg7e5bvT91KoTG+STTDAZGzcR1aQREQ3qQ0QEMYMGcmLJsjPSRTZpjDemNhnrTkeuyvhhM96YGLxxTr3X7N413yCWqiwzJyes7suAEGCXAR2AGwNjWbp+BG4DJhXyEafce3ddVfXKQrbnE+oe3UXATcDxguUkRMiV8pLrU/75xSL+fvNVeESY+d0Gdh46zP+5uBeb9h1k6aadfPHdBh4eNpgP77uJY6cyeWrqbAB2HjrMgvVbeOee35Lr8/HS5wvxqRJfuyZ/ueFywLkCnLd2M99udU7cEYP60jQxDp8qB46kV6sRl+Dcc3ttzhKeHn4ZHvHw5feb+DEljZsu6sGWn1L4ZusuZq/ZxANXXMxbd99A+qlMnvvUGe13RY+ONIyvw/B+3RjerxsAj07+wj+y9aJ2yTwxJeR95mrH4/Vy6W9uY/JLz+FTH136DSCpUWMWfjqVBs1a0KZrD+ZPnURWRgb/ff2fAMQmJHLd6PsBeO+5p0jdv4/szAzGPziaX946guRO1avLPaRcHwdfeJVGL/wNPB6Off4lWTt2UfeOW8jcuNnf6MUMHkj6vALhf30+Ul55k0YvPQsiZG7awtHp1eM8DvdqjoAQYAAikhcCzP+NQVV3utvO+uZmkROvishM4HlV/aqQbYtUtX+oHdjEq6XPJl4tfTbxatmwiVdLX2lOvPrMtHkK8PCwQb8DAqOhTHAfIgfA7Yocqqp3uss3A71VdXTBzxSRd4AZqjo1YF0OsBrn8YJnVXVaUeUq8opOVS8rYlvIRs4YY0z1kZ0bdmSUYofxKqCpqu4TkWRgvoisVdWg9znCGowiIs+Fs84YY0z1VYzZC4odxiuQqu5zf24HFgDdikof7qjLwsbGBr3aM8YYU/0U4/ECfwgwEYnCCQEWcvQkgIjEi0i0+z4R6EfAvb3CFNl1KSIjgVFASxH5PmBTDLA0nEIZY4ypHrJLMASYiJwPfALEA1eIyJOq2hFoD7zhDlLx4Nyj+/kNHc6wzpnAM0Dgcw7pqnq48CzGGGOqo+I8LK6qXwBfFFj3eMD75ThdmgXzLQXOK065Qg1GOQocFZEcVc33cJSIvK+qNxdnZ8YYY6quyh4CrGPggohEAD2CpDXGGFMNZWblkJlVYpFRokXkP+72b0SkecC2se76TSIyJNS+QkVGGSsi6UBnETmW9wIOAJ+GdTTGGGOqhazcXLJyQ89eEGZklDuANFVtBbwIPOfm7YAzeKUjMBT4l/t5QRXZ0KnqM6oaA/wdaIYz+vIK4Bqce3fGGGMM4Ey8Gubkq/7IKKqahTOD+FUF0lwFvOu+nwr8QkTEXT9ZVTNVdQewlRCRusKdpmc7sAjnxuBqoA/wP+CSMPMbY4yp4sJ8hg6gEbA7YHkP0DtYGneU5lGcGMuNgGUF8jYqamfhNnT3AecDy1T1YhFpBzwZTsYF40aXWriZ0iIiIwLD1ZiSZ3Vc+iplHS+eXd4lKLZKWc+lZO5jowScOqGIEGCEFxklWJpiR1UJdzBKhqpmgHODUFU3Am3DzFsZjQidxJwlq+PSZ3VcNqyeCwicwcZ9FfwiEE5kFH8adwBkLHA4zLz5hNvQ7RGROGAaMEdEPg31wcYYY0wQ4URGmQ7c6r7/NTBfnVkIpgPD3VGZLYDWwLdF7SysrktVHea+HSciX+G0rLPCyWuMMcYECicyCs7k3u+LyFacK7nhbt71IjIFJ+xXDnCPqhY51LPIaXqqK+tzL31Wx6XP6rhsWD1XfNbQGWOMqdLCvUdnjDHGVErVrqETkTgRGeW+HygiM4Kke6uQJ/VNGERkpzt9hiljImKzilQQ9ruoOKpdQwfE4Uw9VCRVvTPU1A/GVDSq2re8y2Ac9ruoOKpjQ/cszvx6q3FCm9UWkakislFEPnRDzCAiC0Skp4h4ReQdEVknImtFZEy5lr4CEZHmbr29KyLfu/VY0918r4iscuusnZu+rohMc9MuE5HO7vpxIjLRrfPtInJfwD5uEpFvRWS1iLwRKqZddScix92fDURkkVtv60TkovIuW0XhnoMrRWS9+2AzInJcRJ5z188VkV4B5+OVbprmIvK1e16vEpG+7vqn3HpeLSJ7ReTfeZ/p/hzoflZh/2cud9ctFpGXg/UwmbOkqtXqBTQH1rnvBwJHcR449OCENbvQ3bYA6IkzS8OcgPxx5X0MFeXl1qUC/dzlicADwE7gXnfdKOAt9/144An3/SXAavf9OJyJfKOBRCAViMSZYPEzINJN9y/glvI+7or8Ao67P+8HHnHfe4GY8i5bRXkBdd2f5wDrcMJKKXCZu/4T4Ev3HOwScJ7WBGq471vjDIMP/NxY4HugR4HfRaH/Z4AaOCGuWrjpPgJmlHf9VMVXuCHAqrJvVXUPgHuV1xxYHLB9O5AsIuOBz3H+AMxpu1V1ifv+A5xwcQD/dX+uxAkCDs4f97UAqjpfRBJEJNbd9rmqZgKZInIQqA/8AueLxnL3C/A5wMHSPJgqZDkwUUQigWmqurq8C1SB3Cciec8GN8FptLI4/WzwWiBTVbNFZC3O/wRwGr5XRKQrkAu0yftA9wrtQ+BFVV1ZyD4L+z9zHNiuTmBicBo6i7JSCqpj12VBmQHvcynwEL2qpuF8q1sA3AO8VWYlqxwKPp+St5xXr4F1WlSMusJ+DwK8q6pd3VdbVR139kWu+lR1EdAf2Ivz0O0t5VykCkFEBgKDgAtUtQvwHc6VVba6l1WAD/d8VFUfp8/fMThTlHXB6e2JCvjoccAeVf13kF0HO79NGaiODV06EBNuYnf0oEdVPwYeA7qXVsEqqaYicoH7/kbyXw0XtAj4Lfj/4aSo6rEi0s8Dfi0i9dw8dUWk2dkXuepz6+mgqr6JE2HCzltHLM4cZyfde8d9ipn3J7fxuxmnSxgR+RXOFGb3FZG3MBtxeouau8s3FDO/CVO167pU1VQRWSIi64BTON/QitII+LeI5H0pGFuqBax8fgBuFZE3gC3Aa8C9QdKOw6nL74GTnI5jVyhV3SAijwJfuvWfjXNVvauEyl6VDQQeFJFsnC4yu6JzzALuds/BTeSf7iWUfwEfi8h1wFfACXf9/UBD4Fu3i326qj4e6sNU9ZT7qNMsEUkhRLxG8/NZZBTzs7nfRGeoaqdyLooxlZKI1FbV4+49vleBLar6YnmXq6qpjl2XxhhTUdzlDk5Zj9M1+kY5l6dKsis6Y4wxVZpd0RljjKnSrKEzxhhTpVlDZ4wxpkqzhs4YY0yVZg2dMcaYKs0aOmOMMVXa/wf1JNCwEjAhtAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x72 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualizing the attention layer\n",
    "np.squeeze(att.asnumpy(), 0).shape\n",
    "plt.figure(figsize=(8,1))\n",
    "cmap = sns.diverging_palette(220, 10, as_cmap=True)\n",
    "sns.heatmap(np.squeeze(att.asnumpy(), 0), cmap=cmap, annot=True,\n",
    "            xticklabels=['this','phone','is','amazing'], yticklabels=['att0', 'att1'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  pick a better model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  pick a better model\n",
    "model = SelfAttentiveBiLSTM(vocab_len, emsize, nhidden, nlayers,\n",
    "                            natt_unit, natt_hops, nfc, nclass,\n",
    "                            drop_prob, pool_way, prune_p, prune_q)\n",
    "model.load_parameters('params/params.03')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2]\n",
      "\n",
      "[[[0.11517888 0.16674337 0.288203   0.42987478]\n",
      "  [0.05860164 0.10938538 0.28133833 0.5506747 ]]]\n",
      "<NDArray 1x2x4 @cpu(0)>\n"
     ]
    }
   ],
   "source": [
    "input_ar = nd.array(vocab[['this','phone','is','amazing']], ctx=mx.cpu()).reshape((1, -1))\n",
    "pred, att = model(input_ar)\n",
    "\n",
    "label = np.argmax(nd.softmax(pred, axis=1).asnumpy(), axis=1)\n",
    "print(label)\n",
    "print(att)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbQAAABZCAYAAABWmdGuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAG31JREFUeJzt3Xl8VNXZwPHfM1khgSxkY5N9EVBAEFlkEcGtitq6VRG1KloRfbX6KrWvRVtLqX2hYl1Q1KpI1cKrIKK4souyIzuWNQmQQPY9M3PeP+ZmSEKSuSlJJpk8389nPpl777n3nLlc5txz5tzniDEGpZRSqqlz+LsASimlVF3QCk0ppVRA0ApNKaVUQNAKTSmlVEDQCk0ppVRA0ApNKaVUQNAKTSmlVEDQCk0ppVRA0ApNKaVUQAiu7wx+PW+hhiKpZ73aJfi7CAHv1pGD/V2EZsHxznv+LkLAi5tyr9TXsfdffLkB6LFmeb3lUZN6r9CUUko1E44g/2bv19yVUkoFDEdYKI6wUFtpReQKEdkrIj+JyJNVbL9TRNJFZKv1usfXMbWFppRSqk5IeLi9dCJBwEvAeCAZ2CAiS4wxuyol/cAY86Dd/LWFppRSqk5ISAgSEmIn6RDgJ2PMAWNMCfA+cO3Z5q8VmlJKqTrhCA/DER5mJ2l74Gi55WRrXWW/EJHtIrJQRDr6zN9eMZVSSqmaSVio5yUyWUQ2lntNrpy0it0rj4j/BOhsjDkf+Ap421f++huaUkqpOlHW3WiMeQ14rYakyUD5FlcHILV8AmPMqXKLrwMzfeWvLTSllFJ1QsLD7Q4M2QD0EJEuIhIK3AIsqXAskbblFicAu30dtFm10Pp0SOSmoQMQEdbuPcgX2/dW2N49KY4bh/anfWwUb3zzPVsOpQDQITaKX464gPDQYNzG8PnWPWw6kOyPj9AkHNm9gzUfvY8xbs69aCQXjLuywvbUf+9j7UcfcOpYMuNvn0y3AYMASNm/h7Uff+BNl5V2nPGTJtPlvIENWv6m4Pt1a3nhr8/jdru5+rrrmHjnrypsf3/+uyxd/BFBQcFEx8Qw7enfk9S2HQCvzHmB79asBuCOe+7l0ssub/DyNwXrDx3kb6u+wW0M1/Q9j9sHX1Rlum/37+V3n33CvJsncm5iEruOH2PmN194t//qouGM7tajoYrtV3aH7BtjnCLyILAcCALeNMbsFJFngY3GmCXAQyIyAXACGcCdvo7bbCo0Ebhl+EDmfLaazPwCnrz2UrYfSeV4Vq43TUZeAe+s2si483pW2LfE6eIfKzeQnpNHVMtwpl13KbuST1BYUtrQH6PRc7vdrF60gGvuf4SI6BgWzX6Ozv36E5vUzpsmMiaWsbfexdZvl1fYt32P3tz0+O8BKMrPZ8GffkuHXn0atPxNgcvlYtbMPzP7pVeIT0zk3km3MWLUaLp07eZN07N3b+bd8B7h4S34aOGHvDLnBZ6ZMZN1a1azb89u3lzwPqWlpUydfDdDh48gIjLSj5+o8XG53fzviq/42/U3khDZins+mM/FXbrRpU1chXT5JSX8a9sW+iSebkx0bRPHG7fcTrDDwcn8PO5Y8DYjunQj2BH4HWISZmtACADGmGXAskrrni73fhowrTb5B/4ZtnSOjyU9J4+Tufm43IaNB47Sv1O7Cmky8gpIycjGmIq/Tabl5JGekwdAdkERuYXFRNobydPspB05SFRcPK3j4gkKDqb7wAs5tGNrhTStY+No064DItVHxzmwbRPn9O5HSKie58p279xB+44dadehAyEhIVx62eWsWbmiQpoLBl9IeHgLAPr2O5+0EycAOHTgAAMuGERwcDAtWrSge4+efP/duob+CI3e7hPH6RAdQ/uoaEKCgri0R29WH/j3GeleX7+G2wZdSFjw6QgZ4SEh3sqrxOlEqhz/EJgkNAQJtTVsv174rNBEJEpEbhaRR0XkEet9dEMUri5Ft2xBZn6hdzkzv5Doli1qfZxO8TEEBTk4aVVwqqL8rCwiomO9yxFRMeRnZ9X6OPu3/ED3C4bUZdECRnpaGgmJid7l+IRETqalV5v+08UfM3T4CAC69+zJ+nVrKSoqJCsrk82bNpJ24ni9l7mpSc/LJSGylXc5ITKS9PzcCmn2pZ0gLTeXEV26Vd6dncePcdv8t5i04G0eHzu+WbTOABxhYThq0Uqr8/xr2igik4DNwBigJRABXAJssrY1GVU1BmobNbl1i3DuGj2Ed1durPW+zcfZn5n87CwyjqXQsXffOihPM1FNI2D5sk/Zs3sXv5x0BwBDhg5j2IiL+fWv7uSZ306j33nnExTUbH55sK2qq7h8S8ttDHNWf8vUkWOq3L9vUlvem3gX826eyLsbv6fY6ayfgjYyEhZmu9vRV+irculuEBEjIj4jhPu6kp8CBhljKtxii0gM8D3wTjUFmAxMBhh1+2T6jBrvqxz1LjO/kJiI0y2ymIgWZBcU1rBHReEhwUy5fARLNu3gYHpGfRQxIEREx5Cfdfr85GdnEhFVuwb9v7dupMt5A/WLthrxCQneLkSA9LQTxMXHn5Fu4/freffNN3jxtXmEhp7+sX7S3fcw6W5PWLxnnppGh44+n1dtdhIiW5GWd7pFlpaXR1zE6d8ZC0pKOHDqFA8u8gxiyijI54mlHzHz6us5NzHJm65zbBvCg0M4cOpkhfWBSkJtx3G0FfpKRFoBD+Gpb3zy1Q4Wqr5ZcVPtPaHnGQRjzGBjzODGUJkBHE7PJKF1JG0iWxLkEAZ37cj2w8ds7RvkEO4bN5zv9x9m88GUei5p05bQsTNZ6WnknErH5XTy05YNdO7bv1bH2L/lB3pod2O1evfpS/LRI6SmpFBaWsrXXyzn4lFjKqTZt2cPz//pOWbMmk1M7OkuYJfLRXaW5/70p/37+Pf+/Vw4dFhDFr9J6J2YRHJWJqnZWZS6XHy9fw8Xlxt0ExkWxrLJU1h012QW3TWZvkltvZVZanYWTrcbgOM52RzJyqBt69b++igNquzBahvshr76A/AXoMjOQX3dAj8HbBaRLzgdpuQcPLXqH+1k0Fi4jeH9dVuZeuVIHCKs23eIY1k5XH1BH46czGT7kWN0iovhvvHDaBkaynnntOXqQX34w6IvGdS1Iz3axhERHsrQnp0BeGflBpIzsv37oRohR1AQI39xK0vn/g3jNvS+aASxbdvzw2eLie/YiS79BpB25CCfv/kyxYUFHNq5nQ2fL+aWJ58FICfjJPlZmbTr1tNHTs1XcHAwjzz+BL+Z+gBul5ufTbiWLt26Me/Vl+l9bh8uHj2Gl+fMprCwgKef/G8AEhOT+PPsF3A6nUy51zPEPyIikv/5w3MEB2tLuLJgh4NHxlzKo4sX4XK7ubrveXRtE8fr69fQOyGJkV27V7vv9tQU3t30EcEOBw4RHhszjugWLRuw9P5jM+wVVB36qsJzESIyEOhojFkqIo/ZOahUHtF3RgJP9+LlVgHEyni5MSbTTgY6wWf90wk+659O8NkwdILP+lefE3xmffiRAYi5+ef3Yf3sZHnNih4CgIjcCFxujLnHWr4dGGKMmWotO4BvgDuNMYdEZAXwmDFmY035+7w1M8ZkishAY8wT5deLyMzK65RSSjVfYrXQ6iD0VSugH7DCerwnCVgiIhNqqtTsjiWt6oewK6tYp5RSqpmS0DDE3rOjNYa+MsZkG2PijDGdjTGdgfVAjZUZ+GihicivgQeAbiKyvdymVoA+jamUUsrL7m9oNkNf1ZqvLscFwGfADKD8cwK5xhgdu66UUsrL5uSegO/QV5XWj7FzzBorNGNMNpAtIk5jzOHy20TkXWPM7XYyUUopFfhsDtmvN3bH61YI2SAiwcCgui+OUkqppsphb+qY+su/po0iMk1EcoHzRSSn7AWcABY3SAmVUko1CRISYrvb0VfoKxG5X0R+FJGtIrJGRHxOveGry3EGMENEZuB5WrsnUFYF6/NlSimlvMTmoBCboa8WGGNetdJPAGYBV9R0XLtdjgeAVXieFdgKDAW+A8ba3F8ppVSAq8VvaN7QVwAiUhb6yluhGWNyyqWPwEYjym6F9hBwIbDeGHOJiPQGnrGz4+5knZqivjldLn8XIeCVOvUcN4SCfy70dxECXtyUe+vt2EURnhBfNjodfYa+AhCRKcCjQCg2GlB2H6wuMsYUWRmEGWP2AL1s7quUUqoZEZHJIrKx3Gty5SRV7HZGC8wY85IxphvwBPA7X/nabaElW5N6fgx8KSKZVAxTopRSqpkrcXjaZnUQ+qqy94FXfOVvq0IzxlxvvZ0uIt8CUcDndvZVSinVPBSX2p7I1Bv6CkjBE/rq1vIJRKSHMWa/tfgzYD8+1HreCGPMytruo5RSKvCVuOxVaDZDXz0oIuOAUiATuMPXcXUiJKWUUnWiFi00n6GvjDEP1zZ/rdCUUkrVidpUaPVBKzSllFJ1wt8Vmt1h+0oppVSNSpwuSmw+s2kj9NWjIrJLRLaLyNci0snXMQO+hTak+zk8eMVIghzCp5t3sWDN5grbQ4IcTLt+PL3axZNdUMSzC5dzPCuXpOhWvD3lNo6eygRgV/IJZi1dAcAlfbszcdRgHCKs33+YuV/q1HDl9e2YxC3DB+IQYfWeA3y+dU+F7T3axnPzsIF0aBPFa199x+aDyd5tD181iq4Jbfjp+Ele/Hx1Qxe9yfjhu3X8ffZfcbvdXDXhOm6ddGeF7f9aMJ9lSxYTFBREVEwMjz/1NElt2wIw98UXWL9uLcbtZtCQi3jw0cewZgVW5bS8aDDxD98PjiByln5G5vwPK2xvdeV44h64B9fJUwBkLVpCzlLP4O/uK5dRcuAQAKUn0jj25PSGLLrfFJeW2kpnM/TVFmCwMabAmpvzL8DNNR03oCs0hwgPXzWax95dTHpOHq/eexNr9x7kcHqmN81VF/Qhr6iY2+bMZ2y/HkweN5xnFy4HIDUzm3te/aDCMVu3COf+y0Ywee4HZBcU8eR147igS4cKX8rNmYhw64hBzP50BZn5hTz18/FsO5TKsazTUWwycvN5a8X3XN6/9xn7L9+2h9DgYEaf260hi92kuFwuXvjrTJ6f8xLxCYn8+q5JDB85is5dunrTdO/Vm1f+cQPh4eEsXrSQ1/4+h6efm8GO7dvYsX0b8+b/E4CH77uHbZs3MWDQYH99nMbJ4SD+0SmkPDINZ9pJzpn3Ivlr1lNy6EiFZHnfrCJ99ktn7G6KSzhy1wMNVdpGo9hpu8vRTuirb8ulXw9M9HXQgO5y7N0+kZSMbI5l5uB0uflmx35G9OpaIc2IXl29LYiVu35iUNcONR6zbUxrkk9lkV1QBMCmA0cZ1Ue/fMt0SYglPSeXk7n5uNxuNvx0hAGd21dIcyqvgJSMbIw5MzTbnpQ0ikrs3eU1V3t27aR9h460a9+BkJAQxo6/jHWrKj5NM3DQYMKtqTz69OtHetoJwHPDUVJSgrO0lNLSUpxOJzGxbRr8MzR24ef2ojQ5FWfqcXA6yf1qBREXD/N3sRq9klIXJaW2uhyrCn3Vvpq0AHfjmWy6RgHdQotvHUF6Tq53OT0njz4dEqtN43Ib8opKiGrp+SJIim7N6/fdTH5xCW98s54fjxwjJSObc+JiSIpuRXpOHhf37kpIUEDfF9RKdMsWZOQVepcz8wvokqBfmHXpZHoaCQmnr+O4hAR279xRbfplnyxmyLDhAPQ973wGDBrMDVdfAcZw3Q030alLl3ovc1MTHN8GZ1q6d9mZfpLwPmf2KESOHkGL/v0oOZrCyRfneveR0FA6znsR43KROf8D8ld/12Bl96eyFpoV6qp8uKvXrOghZWyFvrKONREYDIz2lf9/XKGJyHhjzJf/6f7+UkWjoMo0p3LzuXn22+QUFtGzbTx/vOUq7nx5AXlFxcxauoKnb7gcY2DH0WO0i2ld/wVvIvSnmPpX1TUsVX4/wJefLWPf7t3MfsXzXZJy9ChHDh3kwyWex38ef2gK27Zspv/AC+qtvE1SVRdypROfv3Y9eV+twJSWEnXtz0h86jFSHn4CgIO/mIjrVAbB7ZLo8MJMSv59iNLUYw1Rcr8qskY51lXoK+vB6qeA0caYYl/5n03T4o3qNpQPTJm6ae1ZZHF20nPyiW/dyrsc3zqSk7n51aYJcgiR4aHkFBZR6nKTU+jpVtx3LJ3UzBw6tokB4Lt9h3hg3kKmvLGQo6eySM7IbqBP1Phl5hcSG9nCuxwT0ZKs/MIa9lC1FZ+QQJrVhQhwMi2NuPj4M9Jt+uF73vvHm/zx+VmEhnqm9Vi98lv69DuPFi1b0qJlS4YMG87uHT82WNmbCmfaSYITTp/T4Pg4nNbgjzLunFyMNQgi+5PPCOvVw7vNdSrDc5zU4xRu2U5Yz+bxs0RJqZMSe0P3vaGvRCQUT+irJeUTiMhAYC4wwRiTZuegvmasXlLN6xOg2n4kY8xrxpjBxpjB7QaNsFOOerE39QQd2kSRFN2K4CAHY/v1YN3egxXSrNt7kCsGeLoSRvfp7h3cEdUyHId1l9Y2pjXtY6NIzfRUXNERni/syPAwrruwH59u3oXyOJSWQUJUK+JaRRDkcHBh93PYdjjF38UKKL3P7UPK0aMcS02htLSUb778gmEjR1VIs3/vHmbN/BN/fH4WMbGx3vWJiUls27wZl9OJ0+lk25bNnNNZuxwrK9qzl9CO7QlumwjBwbQaN4b8tesrpAlqc/q8Rlw8lJLDngEjjlaR3lmbHVGtCT+v7xmDSQJVsdNpa2CIMcYJlIW+2g18WBb6yprME+B5IBL4lzVr9ZJqDuflq8txJJ6RJXmV1gueUSqNmstteGHZKp6//VocIny2ZReH0jO465Ih7E1NY93eQyzbsovfXj+e9x6aSE5hsXeEY/9O7bnrkiG43Aa3cTNr6QpyCz0t3qlXjKRbUhwA76zcQPKpLL99xsbGbQwL1mzmv64ajYiwdu8BUjNzmDC4H4fTM9h2OJXO8bE8cNkIWoaFcn6ndlw7uB+//5dnuPN/TxhLUnQrwkKC+ctt1/D2yg3s1Dn1KggKDmbqY4/zxMNTcbldXHn1BLp07cZbr71Kz97nMmLUaOa+OIeigkKeecrzeE9CYiLP/XU2o8ZeypZNG7j7tlsQES4cOozhlSpDBbjcpM16ifaz/gQOBzmffkHJwcPE3j2J4j37yF+7nugbrvUMFHG5cOXkcuK5/wUgtNM5JDz+kKeLUoTM+R80mwrNZusMsBX6alxt85eqRpp5N4p8Bvyl0vDJsm2rjDE+/yeMmf53G79aqbPRo+2Z3U2qbk2/6Sp/F6FZKLjmBn8XIeD1WLO83n7pnvHx1wZg2nWX+uXX9BpbaMaYK2vYprd1SimlvEpd/p3Z3dagEBGZaWedUkqp5qu41Gk7nqON0FejRGSziDhFxFbT3e4ox/FVrKu29aaUUqr5KSp1eofu16Rc6KsrgT7AL0WkT6VkR4A7gQV286+xy9GKn/UA0E1Etpfb1ArQAIZKKaW8Sus29NUha5vb7kF9jXJcgCfcyAygfJMw1xiTYTcTpZRSgc9O68xSVeiri842/xq7HI0x2VYt6TTGHC73yhCRd882c6WUUoGj7Dm08sE1rNfkSklth76qDbuhr/pWKIlIMDDobDNXSikVOIpL6jb0VW35ihQyTURygfNFJKfsBZwAFp9t5koppQJHictFib2h+z5DX/0nfHU5zjDGtMITgqQTntGO1wA/x0Yof6WUUs1HcWmprUk+7YS+EpELRSQZuBGYKyI7fR3XbpfjAWAVnmbhVmAo8B0w1ub+SimlApzdZ9DAVuirDXjqHNtqDH3lTSTyI3AhsN4YM0BEegPPGGNqnA67qRKRyZXm7lF1TM9x/dNz3DD0PDcedh+sLjLGFAGISJgxZg/Qq/6K5XeVR+SouqfnuP7pOW4Yep4bCbtdjskiEg18DHwpIpnUwYgUpZRSqq7YqtCMMddbb6eLyLdAFPB5vZVKKaWUqiW7LTQvY8zK+ihII6P94fVPz3H903PcMPQ8NxK2BoUopZRSjZ3dQSFKKaVUo9bsKjQRiRaRB6z3Y0RkaTXp5lUxnYGyQUQOiUicv8vRHImIzoLRSOi/RcNrdhUaEI1nSpwaGWPuMcbs8pVOqcbEGDPc32VQHvpv0fCaY4X2Zzzzu23FE9IrUkQWisgeEXlPRARARFaIyGARCRKRf4jIDhH5UUQe8WvpGxER6Wydt7dFZLt1Hltam6das83+aD2Ij4jEisjHVtr1InK+tX66iLxpnfMDIvJQuTwmisgPIrJVROZaEwOqaohInvW3rYisss7bDhEZ6e+yNRbWNbhJRHaWRYEXkTwRmWmt/0pEhpS7HstCMXUWkdXWdb1ZRIZb65+1zvNWEUkRkbfKjmn9HWMdq6rvmausdWtEZE51PUbKJmNMs3oBnYEd1vsxQDae8CoOPOG8Lra2rQAG45lV4Mty+0f7+zM0lpd1Lg0wwlp+E3gMOARMtdY9AMyz3r8I/N56PxbYar2fjmfC2DAgDjgFhADnAp8AIVa6l4FJ/v7cjfkF5Fl/fwM8Zb0PAlr5u2yN5QXEWn9bADuANtZ1fKW1/iPgC+sa7F/uOm0JhFvvewAbKx03CtgODKr0b1Hl9wwQjmdOsC5Wun8CS/19fpryq9bD9gPQD8aYZACr1dYZWFNu+wGgq4i8CHyK50JXpx01xqy13s8HylpX/2f93YQnmDV4/hP/AsAY842ItBGRKGvbp8aYYqBYRNKAROBSPDcUG6wb2hZAWn1+mACyAXhTREKAj40xW/1doEbkIREpe7a2I57KqYTTz9b+CBQbY0qtsH+drfUhwN9FZADgAnqWHdBqcb0HzDbGbKoiz6q+Z/KAA8aYg1aaf6JRR85Kc+xyrKy43HsXlZ7NM8Zk4rlLWwFMAeY1WMmahsrPfZQtl53X8ue0pkn9qvp3EOBtY8wA69XLGDP97Isc+Iwxq4BRQArwrohM8nORGgURGQOMA4YZY/oDW/C0lEqN1UwC3FjXozHGzenr9xE8U2f1x9N7E1ru0NOBZGPMW9VkXd31repQc6zQcoFWdhNbo/UcxphFwP8AF9RXwZqoc0RkmPX+l1Rs3Va2CrgNvF8sJ40xOTWk/xq4QUQSrH1iRaTT2Rc58FnnKc0Y8zrwBnrdlokCMo0xBdZvu0Nrue8xq5K7HU9XLiJyNZ6ptR6qYd+q7MHT+9PZWg7IYO8Nqdl1ORpjTonIWhHZARTiueOqSXvgLREpq/yn1WsBm57dwB0iMhfYD7wCTK0m7XQ853I7UADcUdOBjTG7ROR3wBfW+S/F00o+XEdlD2RjgMdFpBRP15a20Dw+B+63rsG9wPpa7PsysEhEbgS+BfKt9b8B2gE/WF3jS0y5aVCqY4wptB4h+lxETgI/1KIsqgoaKUT9x6w7y6XGmH5+LopSTZKIRBpj8qzf4F4C9htjZvu7XE1Vc+xyVEqpxuJea5DITjxdmnP9XJ4mTVtoSimlAoK20JRSSgUErdCUUkoFBK3QlFJKBQSt0JRSSgUErdCUUkoFBK3QlFJKBYT/B9Ho9fnnol3nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x72 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualizing the attention layer\n",
    "np.squeeze(att.asnumpy(), 0).shape\n",
    "plt.figure(figsize=(8,1))\n",
    "cmap = sns.diverging_palette(220, 10, as_cmap=True)\n",
    "sns.heatmap(np.squeeze(att.asnumpy(), 0), cmap=cmap, annot=True,\n",
    "            xticklabels=['this','phone','is','amazing'], yticklabels=['att0', 'att1'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}