attention_model_validation_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Boilerplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Functions to accomplish attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def batch_matmul_bias(seq, weight, bias, nonlinearity=''):\n",
    "    s = None\n",
    "    bias_dim = bias.size()\n",
    "    for i in range(seq.size(0)):\n",
    "        _s = torch.mm(seq[i], weight) \n",
    "        _s_bias = _s + bias.expand(bias_dim[0], _s.size()[0]).transpose(0,1)\n",
    "        if(nonlinearity=='tanh'):\n",
    "            _s_bias = torch.tanh(_s_bias)\n",
    "        _s_bias = _s_bias.unsqueeze(0)\n",
    "        if(s is None):\n",
    "            s = _s_bias\n",
    "        else:\n",
    "            s = torch.cat((s,_s_bias),0)\n",
    "    return s.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def batch_matmul(seq, weight, nonlinearity=''):\n",
    "    s = None\n",
    "    for i in range(seq.size(0)):\n",
    "        _s = torch.mm(seq[i], weight)\n",
    "        if(nonlinearity=='tanh'):\n",
    "            _s = torch.tanh(_s)\n",
    "        _s = _s.unsqueeze(0)\n",
    "        if(s is None):\n",
    "            s = _s\n",
    "        else:\n",
    "            s = torch.cat((s,_s),0)\n",
    "    return s.squeeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def attention_mul(rnn_outputs, att_weights):\n",
    "    attn_vectors = None\n",
    "    for i in range(rnn_outputs.size(0)):\n",
    "        h_i = rnn_outputs[i]\n",
    "        a_i = att_weights[i].unsqueeze(1).expand_as(h_i)\n",
    "        h_i = a_i * h_i\n",
    "        h_i = h_i.unsqueeze(0)\n",
    "        if(attn_vectors is None):\n",
    "            attn_vectors = h_i\n",
    "        else:\n",
    "            attn_vectors = torch.cat((attn_vectors,h_i),0)\n",
    "    return torch.sum(attn_vectors, 0).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Word attention model with bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class AttentionWordRNN(nn.Module):\n",
    "    \n",
    "    \n",
    "    def __init__(self, batch_size, num_tokens, embed_size, word_gru_hidden, bidirectional= True):        \n",
    "        \n",
    "        super(AttentionWordRNN, self).__init__()\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.num_tokens = num_tokens\n",
    "        self.embed_size = embed_size\n",
    "        self.word_gru_hidden = word_gru_hidden\n",
    "        self.bidirectional = bidirectional\n",
    "        \n",
    "        self.lookup = nn.Embedding(num_tokens, embed_size)\n",
    "        if bidirectional == True:\n",
    "            self.word_gru = nn.GRU(embed_size, word_gru_hidden, bidirectional= True)\n",
    "            self.weight_W_word = nn.Parameter(torch.Tensor(2* word_gru_hidden,2*word_gru_hidden))\n",
    "            self.bias_word = nn.Parameter(torch.Tensor(2* word_gru_hidden,1))\n",
    "            self.weight_proj_word = nn.Parameter(torch.Tensor(2*word_gru_hidden, 1))\n",
    "        else:\n",
    "            self.word_gru = nn.GRU(embed_size, word_gru_hidden, bidirectional= False)\n",
    "            self.weight_W_word = nn.Parameter(torch.Tensor(word_gru_hidden, word_gru_hidden))\n",
    "            self.bias_word = nn.Parameter(torch.Tensor(word_gru_hidden,1))\n",
    "            self.weight_proj_word = nn.Parameter(torch.Tensor(word_gru_hidden, 1))\n",
    "            \n",
    "        self.softmax_word = nn.Softmax()\n",
    "        self.weight_W_word.data.uniform_(-0.1, 0.1)\n",
    "        self.weight_proj_word.data.uniform_(-0.1,0.1)\n",
    "\n",
    "        \n",
    "        \n",
    "    def forward(self, embed, state_word):\n",
    "        # embeddings\n",
    "        embedded = self.lookup(embed)\n",
    "        # word level gru\n",
    "        output_word, state_word = self.word_gru(embedded, state_word)\n",
    "#         print output_word.size()\n",
    "        word_squish = batch_matmul_bias(output_word, self.weight_W_word,self.bias_word, nonlinearity='tanh')\n",
    "        word_attn = batch_matmul(word_squish, self.weight_proj_word)\n",
    "        word_attn_norm = self.softmax_word(word_attn.transpose(1,0))\n",
    "        word_attn_vectors = attention_mul(output_word, word_attn_norm.transpose(1,0))        \n",
    "        return word_attn_vectors, state_word, word_attn_norm\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        if self.bidirectional == True:\n",
    "            return Variable(torch.zeros(2, self.batch_size, self.word_gru_hidden))\n",
    "        else:\n",
    "            return Variable(torch.zeros(1, self.batch_size, self.word_gru_hidden))        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Sentence Attention model with bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class AttentionSentRNN(nn.Module):\n",
    "    \n",
    "    \n",
    "    def __init__(self, batch_size, sent_gru_hidden, word_gru_hidden, n_classes, bidirectional= True):        \n",
    "        \n",
    "        super(AttentionSentRNN, self).__init__()\n",
    "        \n",
    "        self.batch_size = batch_size\n",
    "        self.sent_gru_hidden = sent_gru_hidden\n",
    "        self.n_classes = n_classes\n",
    "        self.word_gru_hidden = word_gru_hidden\n",
    "        self.bidirectional = bidirectional\n",
    "        \n",
    "        \n",
    "        if bidirectional == True:\n",
    "            self.sent_gru = nn.GRU(2 * word_gru_hidden, sent_gru_hidden, bidirectional= True)        \n",
    "            self.weight_W_sent = nn.Parameter(torch.Tensor(2* sent_gru_hidden ,2* sent_gru_hidden))\n",
    "            self.bias_sent = nn.Parameter(torch.Tensor(2* sent_gru_hidden,1))\n",
    "            self.weight_proj_sent = nn.Parameter(torch.Tensor(2* sent_gru_hidden, 1))\n",
    "            self.final_linear = nn.Linear(2* sent_gru_hidden, n_classes)\n",
    "        else:\n",
    "            self.sent_gru = nn.GRU(word_gru_hidden, sent_gru_hidden, bidirectional= True)        \n",
    "            self.weight_W_sent = nn.Parameter(torch.Tensor(sent_gru_hidden ,sent_gru_hidden))\n",
    "            self.bias_sent = nn.Parameter(torch.Tensor(sent_gru_hidden,1))\n",
    "            self.weight_proj_sent = nn.Parameter(torch.Tensor(sent_gru_hidden, 1))\n",
    "            self.final_linear = nn.Linear(sent_gru_hidden, n_classes)\n",
    "        self.softmax_sent = nn.Softmax()\n",
    "        self.final_softmax = nn.Softmax()\n",
    "        self.weight_W_sent.data.uniform_(-0.1, 0.1)\n",
    "        self.weight_proj_sent.data.uniform_(-0.1,0.1)\n",
    "        \n",
    "        \n",
    "    def forward(self, word_attention_vectors, state_sent):\n",
    "        output_sent, state_sent = self.sent_gru(word_attention_vectors, state_sent)        \n",
    "        sent_squish = batch_matmul_bias(output_sent, self.weight_W_sent,self.bias_sent, nonlinearity='tanh')\n",
    "        sent_attn = batch_matmul(sent_squish, self.weight_proj_sent)\n",
    "        sent_attn_norm = self.softmax_sent(sent_attn.transpose(1,0))\n",
    "        sent_attn_vectors = attention_mul(output_sent, sent_attn_norm.transpose(1,0))        \n",
    "        # final classifier\n",
    "        final_map = self.final_linear(sent_attn_vectors.squeeze(0))\n",
    "        return F.log_softmax(final_map), state_sent, sent_attn_norm\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        if self.bidirectional == True:\n",
    "            return Variable(torch.zeros(2, self.batch_size, self.sent_gru_hidden))\n",
    "        else:\n",
    "            return Variable(torch.zeros(1, self.batch_size, self.sent_gru_hidden))   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Functions to train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "word_attn = AttentionWordRNN(batch_size=64, num_tokens=81132, embed_size=300, \n",
    "                             word_gru_hidden=100, bidirectional= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sent_attn = AttentionSentRNN(batch_size=64, sent_gru_hidden=100, word_gru_hidden=100, \n",
    "                             n_classes=10, bidirectional= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def train_data(mini_batch, targets, word_attn_model, sent_attn_model, word_optimizer, sent_optimizer, criterion):\n",
    "    state_word = word_attn_model.init_hidden().cuda()\n",
    "    state_sent = sent_attn_model.init_hidden().cuda()\n",
    "    max_sents, batch_size, max_tokens = mini_batch.size()\n",
    "    word_optimizer.zero_grad()\n",
    "    sent_optimizer.zero_grad()\n",
    "    s = None\n",
    "    for i in xrange(max_sents):\n",
    "        _s, state_word, _ = word_attn_model(mini_batch[i,:,:].transpose(0,1), state_word)\n",
    "        if(s is None):\n",
    "            s = _s\n",
    "        else:\n",
    "            s = torch.cat((s,_s),0)            \n",
    "    y_pred, state_sent, _ = sent_attn_model(s, state_sent)\n",
    "    loss = criterion(y_pred.cuda(), targets) \n",
    "    loss.backward()\n",
    "    \n",
    "    word_optimizer.step()\n",
    "    sent_optimizer.step()\n",
    "    \n",
    "    return loss.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_predictions(val_tokens, word_attn_model, sent_attn_model):\n",
    "    max_sents, batch_size, max_tokens = val_tokens.size()\n",
    "    state_word = word_attn_model.init_hidden().cuda()\n",
    "    state_sent = sent_attn_model.init_hidden().cuda()\n",
    "    s = None\n",
    "    for i in xrange(max_sents):\n",
    "        _s, state_word, _ = word_attn_model(val_tokens[i,:,:].transpose(0,1), state_word)\n",
    "        if(s is None):\n",
    "            s = _s\n",
    "        else:\n",
    "            s = torch.cat((s,_s),0)            \n",
    "    y_pred, state_sent, _ = sent_attn_model(s, state_sent)    \n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 1e-1\n",
    "momentum = 0.9\n",
    "word_optmizer = torch.optim.SGD(word_attn.parameters(), lr=learning_rate, momentum= momentum)\n",
    "sent_optimizer = torch.optim.SGD(sent_attn.parameters(), lr=learning_rate, momentum= momentum)\n",
    "criterion = nn.NLLLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AttentionWordRNN (\n",
       "  (lookup): Embedding(81132, 300)\n",
       "  (word_gru): GRU(300, 100, bidirectional=True)\n",
       "  (softmax_word): Softmax ()\n",
       ")"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_attn.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AttentionSentRNN (\n",
       "  (sent_gru): GRU(200, 100, bidirectional=True)\n",
       "  (final_linear): Linear (200 -> 10)\n",
       "  (softmax_sent): Softmax ()\n",
       "  (final_softmax): Softmax ()\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_attn.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "d = pd.read_json('/data1/sandeep/datasets/imdb_final.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "d['rating'] = d['rating'] - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sandeep/miniconda3/envs/py2/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "#from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "d = d[['tokens','rating']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = d.tokens\n",
    "y = d.rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X.values, y.values, test_size = 0.3, random_state= 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59443,)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def pad_batch(mini_batch):\n",
    "    mini_batch_size = len(mini_batch)\n",
    "    max_sent_len = int(np.mean([len(x) for x in mini_batch]))\n",
    "    max_token_len = int(np.mean([len(val) for sublist in mini_batch for val in sublist]))\n",
    "    main_matrix = np.zeros((mini_batch_size, max_sent_len, max_token_len), dtype= np.int)\n",
    "    for i in xrange(main_matrix.shape[0]):\n",
    "        for j in xrange(main_matrix.shape[1]):\n",
    "            for k in xrange(main_matrix.shape[2]):\n",
    "                try:\n",
    "                    main_matrix[i,j,k] = mini_batch[i][j][k]\n",
    "                except IndexError:\n",
    "                    pass\n",
    "    return Variable(torch.from_numpy(main_matrix).transpose(0,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def test_accuracy_mini_batch(tokens, labels, word_attn, sent_attn):\n",
    "    y_pred = get_predictions(tokens, word_attn, sent_attn)\n",
    "    _, y_pred = torch.max(y_pred, 1)\n",
    "    correct = np.ndarray.flatten(y_pred.data.cpu().numpy())\n",
    "    labels = np.ndarray.flatten(labels.data.cpu().numpy())\n",
    "    num_correct = sum(correct == labels)\n",
    "    return float(num_correct) / len(correct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def test_accuracy_full_batch(tokens, labels, mini_batch_size, word_attn, sent_attn):\n",
    "    p = []\n",
    "    l = []\n",
    "    g = gen_minibatch(tokens, labels, mini_batch_size)\n",
    "    for token, label in g:\n",
    "        y_pred = get_predictions(token.cuda(), word_attn, sent_attn)\n",
    "        _, y_pred = torch.max(y_pred, 1)\n",
    "        p.append(np.ndarray.flatten(y_pred.data.cpu().numpy()))\n",
    "        l.append(np.ndarray.flatten(label.data.cpu().numpy()))\n",
    "    p = [item for sublist in p for item in sublist]\n",
    "    l = [item for sublist in l for item in sublist]\n",
    "    p = np.array(p)\n",
    "    l = np.array(l)\n",
    "    num_correct = sum(p == l)\n",
    "    return float(num_correct)/ len(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def test_data(mini_batch, targets, word_attn_model, sent_attn_model):    \n",
    "    state_word = word_attn_model.init_hidden().cuda()\n",
    "    state_sent = sent_attn_model.init_hidden().cuda()\n",
    "    max_sents, batch_size, max_tokens = mini_batch.size()\n",
    "    s = None\n",
    "    for i in xrange(max_sents):\n",
    "        _s, state_word, _ = word_attn_model(mini_batch[i,:,:].transpose(0,1), state_word)\n",
    "        if(s is None):\n",
    "            s = _s\n",
    "        else:\n",
    "            s = torch.cat((s,_s),0)            \n",
    "    y_pred, state_sent,_ = sent_attn_model(s, state_sent)\n",
    "    loss = criterion(y_pred.cuda(), targets)     \n",
    "    return loss.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def iterate_minibatches(inputs, targets, batchsize, shuffle=False):\n",
    "    assert inputs.shape[0] == targets.shape[0]\n",
    "    if shuffle:\n",
    "        indices = np.arange(inputs.shape[0])\n",
    "        np.random.shuffle(indices)\n",
    "    for start_idx in range(0, inputs.shape[0] - batchsize + 1, batchsize):\n",
    "        if shuffle:\n",
    "            excerpt = indices[start_idx:start_idx + batchsize]\n",
    "        else:\n",
    "            excerpt = slice(start_idx, start_idx + batchsize)\n",
    "        yield inputs[excerpt], targets[excerpt]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def gen_minibatch(tokens, labels, mini_batch_size, shuffle= True):\n",
    "    for token, label in iterate_minibatches(tokens, labels, mini_batch_size, shuffle= shuffle):\n",
    "        token = pad_batch(token)\n",
    "        yield token.cuda(), Variable(torch.from_numpy(label), requires_grad= False).cuda()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def check_val_loss(val_tokens, val_labels, mini_batch_size, word_attn_model, sent_attn_model):\n",
    "    val_loss = []\n",
    "    for token, label in iterate_minibatches(val_tokens, val_labels, mini_batch_size, shuffle= True):\n",
    "        val_loss.append(test_data(pad_batch(token).cuda(), Variable(torch.from_numpy(label), requires_grad= False).cuda(), \n",
    "                                  word_attn_model, sent_attn_model))\n",
    "    return np.mean(val_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import math\n",
    "\n",
    "def timeSince(since):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def train_early_stopping(mini_batch_size, X_train, y_train, X_test, y_test, word_attn_model, sent_attn_model, \n",
    "                         word_attn_optimiser, sent_attn_optimiser, loss_criterion, num_epoch, \n",
    "                         print_val_loss_every = 1000, print_loss_every = 50):\n",
    "    start = time.time()\n",
    "    loss_full = []\n",
    "    loss_epoch = []\n",
    "    accuracy_epoch = []\n",
    "    loss_smooth = []\n",
    "    accuracy_full = []\n",
    "    epoch_counter = 0\n",
    "    g = gen_minibatch(X_train, y_train, mini_batch_size)\n",
    "    for i in xrange(1, num_epoch + 1):\n",
    "        try:\n",
    "            tokens, labels = next(g)\n",
    "            loss = train_data(tokens, labels, word_attn_model, sent_attn_model, word_attn_optimiser, sent_attn_optimiser, loss_criterion)\n",
    "            acc = test_accuracy_mini_batch(tokens, labels, word_attn_model, sent_attn_model)\n",
    "            accuracy_full.append(acc)\n",
    "            accuracy_epoch.append(acc)\n",
    "            loss_full.append(loss)\n",
    "            loss_epoch.append(loss)\n",
    "            # print loss every n passes\n",
    "            if i % print_loss_every == 0:\n",
    "                print 'Loss at %d minibatches, %d epoch,(%s) is %f' %(i, epoch_counter, timeSince(start), np.mean(loss_epoch))\n",
    "                print 'Accuracy at %d minibatches is %f' % (i, np.mean(accuracy_epoch))\n",
    "            # check validation loss every n passes\n",
    "            if i % print_val_loss_every == 0:\n",
    "                val_loss = check_val_loss(X_test, y_test, mini_batch_size, word_attn_model, sent_attn_model)\n",
    "                print 'Average training loss at this epoch..minibatch..%d..is %f' % (i, np.mean(loss_epoch))\n",
    "                print 'Validation loss after %d passes is %f' %(i, val_loss)\n",
    "                if val_loss > np.mean(loss_full):\n",
    "                    print 'Validation loss is higher than training loss at %d is %f , stopping training!' % (i, val_loss)\n",
    "                    print 'Average training loss at %d is %f' % (i, np.mean(loss_full))\n",
    "        except StopIteration:\n",
    "            epoch_counter += 1\n",
    "            print 'Reached %d epocs' % epoch_counter\n",
    "            print 'i %d' % i\n",
    "            g = gen_minibatch(X_train, y_train, mini_batch_size)\n",
    "            loss_epoch = []\n",
    "            accuracy_epoch = []\n",
    "    return loss_full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at 50 minibatches, 0 epoch,(0m 51s) is 2.110739\n",
      "Accuracy at 50 minibatches is 0.195937\n",
      "Loss at 100 minibatches, 0 epoch,(1m 41s) is 2.108910\n",
      "Accuracy at 100 minibatches is 0.190469\n",
      "Loss at 150 minibatches, 0 epoch,(2m 31s) is 2.109402\n",
      "Accuracy at 150 minibatches is 0.196042\n",
      "Loss at 200 minibatches, 0 epoch,(3m 16s) is 2.107046\n",
      "Accuracy at 200 minibatches is 0.196719\n",
      "Loss at 250 minibatches, 0 epoch,(3m 50s) is 2.102653\n",
      "Accuracy at 250 minibatches is 0.198313\n",
      "Loss at 300 minibatches, 0 epoch,(4m 42s) is 2.101659\n",
      "Accuracy at 300 minibatches is 0.198437\n",
      "Loss at 350 minibatches, 0 epoch,(5m 31s) is 2.101384\n",
      "Accuracy at 350 minibatches is 0.197679\n",
      "Loss at 400 minibatches, 0 epoch,(6m 7s) is 2.100627\n",
      "Accuracy at 400 minibatches is 0.197734\n",
      "Loss at 450 minibatches, 0 epoch,(6m 41s) is 2.099645\n",
      "Accuracy at 450 minibatches is 0.197951\n",
      "Loss at 500 minibatches, 0 epoch,(7m 16s) is 2.099830\n",
      "Accuracy at 500 minibatches is 0.198156\n",
      "Loss at 550 minibatches, 0 epoch,(7m 51s) is 2.098680\n",
      "Accuracy at 550 minibatches is 0.198835\n",
      "Loss at 600 minibatches, 0 epoch,(8m 25s) is 2.097876\n",
      "Accuracy at 600 minibatches is 0.198906\n",
      "Loss at 650 minibatches, 0 epoch,(8m 59s) is 2.097118\n",
      "Accuracy at 650 minibatches is 0.199111\n",
      "Loss at 700 minibatches, 0 epoch,(9m 35s) is 2.094303\n",
      "Accuracy at 700 minibatches is 0.199576\n",
      "Loss at 750 minibatches, 0 epoch,(10m 9s) is 2.091025\n",
      "Accuracy at 750 minibatches is 0.200313\n",
      "Loss at 800 minibatches, 0 epoch,(10m 44s) is 2.087079\n",
      "Accuracy at 800 minibatches is 0.201914\n",
      "Loss at 850 minibatches, 0 epoch,(11m 18s) is 2.082600\n",
      "Accuracy at 850 minibatches is 0.203603\n",
      "Loss at 900 minibatches, 0 epoch,(11m 54s) is 2.077223\n",
      "Accuracy at 900 minibatches is 0.205087\n",
      "Reached 1 epocs\n",
      "i 929\n",
      "Loss at 950 minibatches, 1 epoch,(12m 29s) is 1.963912\n",
      "Accuracy at 950 minibatches is 0.252976\n",
      "Loss at 1000 minibatches, 1 epoch,(13m 4s) is 1.963951\n",
      "Accuracy at 1000 minibatches is 0.247579\n",
      "Average training loss at this epoch..minibatch..1000..is 1.963951\n",
      "Validation loss after 1000 passes is 1.944949\n",
      "Loss at 1050 minibatches, 1 epoch,(15m 13s) is 1.948366\n",
      "Accuracy at 1050 minibatches is 0.248967\n",
      "Loss at 1100 minibatches, 1 epoch,(15m 49s) is 1.942707\n",
      "Accuracy at 1100 minibatches is 0.247533\n",
      "Loss at 1150 minibatches, 1 epoch,(16m 24s) is 1.936455\n",
      "Accuracy at 1150 minibatches is 0.249081\n",
      "Loss at 1200 minibatches, 1 epoch,(17m 1s) is 1.932495\n",
      "Accuracy at 1200 minibatches is 0.247867\n",
      "Loss at 1250 minibatches, 1 epoch,(17m 37s) is 1.928141\n",
      "Accuracy at 1250 minibatches is 0.249805\n",
      "Loss at 1300 minibatches, 1 epoch,(18m 13s) is 1.920876\n",
      "Accuracy at 1300 minibatches is 0.253201\n",
      "Loss at 1350 minibatches, 1 epoch,(18m 48s) is 1.915464\n",
      "Accuracy at 1350 minibatches is 0.254899\n",
      "Loss at 1400 minibatches, 1 epoch,(19m 24s) is 1.910669\n",
      "Accuracy at 1400 minibatches is 0.258360\n",
      "Loss at 1450 minibatches, 1 epoch,(19m 59s) is 1.907335\n",
      "Accuracy at 1450 minibatches is 0.257498\n",
      "Loss at 1500 minibatches, 1 epoch,(20m 34s) is 1.900382\n",
      "Accuracy at 1500 minibatches is 0.259824\n",
      "Loss at 1550 minibatches, 1 epoch,(21m 9s) is 1.896294\n",
      "Accuracy at 1550 minibatches is 0.260945\n",
      "Loss at 1600 minibatches, 1 epoch,(21m 44s) is 1.893232\n",
      "Accuracy at 1600 minibatches is 0.261829\n",
      "Loss at 1650 minibatches, 1 epoch,(22m 19s) is 1.889603\n",
      "Accuracy at 1650 minibatches is 0.262721\n",
      "Loss at 1700 minibatches, 1 epoch,(22m 54s) is 1.884576\n",
      "Accuracy at 1700 minibatches is 0.265098\n",
      "Loss at 1750 minibatches, 1 epoch,(23m 30s) is 1.880259\n",
      "Accuracy at 1750 minibatches is 0.266348\n",
      "Loss at 1800 minibatches, 1 epoch,(24m 6s) is 1.877118\n",
      "Accuracy at 1800 minibatches is 0.268459\n",
      "Loss at 1850 minibatches, 1 epoch,(24m 43s) is 1.874356\n",
      "Accuracy at 1850 minibatches is 0.269510\n",
      "Reached 2 epocs\n",
      "i 1858\n",
      "Loss at 1900 minibatches, 2 epoch,(25m 21s) is 1.790677\n",
      "Accuracy at 1900 minibatches is 0.296503\n",
      "Loss at 1950 minibatches, 2 epoch,(25m 56s) is 1.789148\n",
      "Accuracy at 1950 minibatches is 0.301461\n",
      "Loss at 2000 minibatches, 2 epoch,(26m 30s) is 1.775413\n",
      "Accuracy at 2000 minibatches is 0.302487\n",
      "Average training loss at this epoch..minibatch..2000..is 1.775413\n",
      "Validation loss after 2000 passes is 1.787492\n",
      "Loss at 2050 minibatches, 2 epoch,(28m 41s) is 1.764106\n",
      "Accuracy at 2050 minibatches is 0.308838\n",
      "Loss at 2100 minibatches, 2 epoch,(29m 17s) is 1.766308\n",
      "Accuracy at 2100 minibatches is 0.309207\n",
      "Loss at 2150 minibatches, 2 epoch,(29m 56s) is 1.759367\n",
      "Accuracy at 2150 minibatches is 0.312982\n",
      "Loss at 2200 minibatches, 2 epoch,(30m 44s) is 1.756264\n",
      "Accuracy at 2200 minibatches is 0.315698\n",
      "Loss at 2250 minibatches, 2 epoch,(31m 23s) is 1.757062\n",
      "Accuracy at 2250 minibatches is 0.315928\n",
      "Loss at 2300 minibatches, 2 epoch,(31m 58s) is 1.752403\n",
      "Accuracy at 2300 minibatches is 0.317555\n",
      "Loss at 2350 minibatches, 2 epoch,(32m 34s) is 1.752327\n",
      "Accuracy at 2350 minibatches is 0.317518\n",
      "Loss at 2400 minibatches, 2 epoch,(33m 10s) is 1.750582\n",
      "Accuracy at 2400 minibatches is 0.318064\n",
      "Loss at 2450 minibatches, 2 epoch,(33m 45s) is 1.748897\n",
      "Accuracy at 2450 minibatches is 0.318544\n",
      "Loss at 2500 minibatches, 2 epoch,(34m 22s) is 1.746259\n",
      "Accuracy at 2500 minibatches is 0.319826\n",
      "Loss at 2550 minibatches, 2 epoch,(34m 57s) is 1.742573\n",
      "Accuracy at 2550 minibatches is 0.321374\n",
      "Loss at 2600 minibatches, 2 epoch,(35m 34s) is 1.738115\n",
      "Accuracy at 2600 minibatches is 0.322039\n",
      "Loss at 2650 minibatches, 2 epoch,(36m 10s) is 1.737979\n",
      "Accuracy at 2650 minibatches is 0.321970\n",
      "Loss at 2700 minibatches, 2 epoch,(36m 45s) is 1.735540\n",
      "Accuracy at 2700 minibatches is 0.322669\n",
      "Loss at 2750 minibatches, 2 epoch,(37m 23s) is 1.731871\n",
      "Accuracy at 2750 minibatches is 0.323938\n",
      "Reached 3 epocs\n",
      "i 2787\n",
      "Loss at 2800 minibatches, 3 epoch,(37m 59s) is 1.683922\n",
      "Accuracy at 2800 minibatches is 0.344952\n",
      "Loss at 2850 minibatches, 3 epoch,(38m 34s) is 1.633321\n",
      "Accuracy at 2850 minibatches is 0.371280\n",
      "Loss at 2900 minibatches, 3 epoch,(39m 10s) is 1.630799\n",
      "Accuracy at 2900 minibatches is 0.370437\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-fcc858b92244>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m loss_full= train_early_stopping(64, X_train, y_train, X_test, y_test, word_attn, sent_attn, word_optmizer, sent_optimizer, \n\u001b[0;32m----> 2\u001b[0;31m                             criterion, 5000, 1000, 50)\n\u001b[0m",
      "\u001b[0;32m<ipython-input-31-bf7c414268dc>\u001b[0m in \u001b[0;36mtrain_early_stopping\u001b[0;34m(mini_batch_size, X_train, y_train, X_test, y_test, word_attn_model, sent_attn_model, word_attn_optimiser, sent_attn_optimiser, loss_criterion, num_epoch, print_val_loss_every, print_loss_every)\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_attn_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msent_attn_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_attn_optimiser\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msent_attn_optimiser\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_criterion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m             \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_accuracy_mini_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_attn_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msent_attn_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0maccuracy_full\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0macc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-978e85add15a>\u001b[0m in \u001b[0;36mtrain_data\u001b[0;34m(mini_batch, targets, word_attn_model, sent_attn_model, word_optimizer, sent_optimizer, criterion)\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate_sent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msent_attn_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate_sent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mword_optimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sandeep/miniconda3/envs/py2/lib/python2.7/site-packages/torch/autograd/variable.pyc\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_variables)\u001b[0m\n\u001b[1;32m    154\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'backward should be called only on a scalar (i.e. 1-element tensor) or with gradient w.r.t. the variable'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m             \u001b[0mgradient\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresize_as_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfill_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 156\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_execution_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_backward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_variables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    158\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sandeep/miniconda3/envs/py2/lib/python2.7/site-packages/torch/autograd/_functions/pointwise.pyc\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, grad_output)\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m         \u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaved_tensors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mgrad_output\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sandeep/miniconda3/envs/py2/lib/python2.7/site-packages/torch/tensor.pyc\u001b[0m in \u001b[0;36m__mul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m    410\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    411\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__mul__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 412\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    413\u001b[0m     \u001b[0m__rmul__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m__mul__\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    414\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "loss_full= train_early_stopping(64, X_train, y_train, X_test, y_test, word_attn, sent_attn, word_optmizer, sent_optimizer, \n",
    "                            criterion, 5000, 1000, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.33860709798994976"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_accuracy_full_batch(X_test, y_test, 64, word_attn, sent_attn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.35774178340517243"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_accuracy_full_batch(X_train, y_train, 64, word_attn, sent_attn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}