fasttext+elmo+tf_idf+transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import pandas as pd\n",
    "from torch import optim\n",
    "from torchtext import data,vocab\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "SEED = 1111\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "device = ('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15744, 1024)\n",
      "(11335, 1024)\n",
      "(1260, 1024)\n",
      "(12595, 1024)\n",
      "(3149, 1024)\n",
      "          id                                               text  \\\n",
      "0  ta_sent_1         Yarayellam FDFS ppga ippove ready agitinga   \n",
      "1  ta_sent_2  Ennada viswasam mersal sarkar madhri time la l...   \n",
      "2  ta_sent_3  yuvan vera level ya .... valuable script. SK i...   \n",
      "3  ta_sent_4  70 vayasulayum thanoda rasigargala sandhosapad...   \n",
      "4  ta_sent_5      all the best anna...Telugu makkal selvan fans   \n",
      "\n",
      "         category  \n",
      "0        Positive  \n",
      "1        Positive  \n",
      "2        Positive  \n",
      "3  Mixed_feelings  \n",
      "4        Positive  \n"
     ]
    }
   ],
   "source": [
    "lang='tamil'\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "df_train = pd.read_csv(\"data1/\"+lang+\"/\"+lang+\"_train.tsv\",sep=\"\\t\")\n",
    "df_dev = pd.read_csv(\"data1/\"+lang+\"/\"+lang+\"_dev.tsv\",sep=\"\\t\")\n",
    "df_test = pd.read_csv(\"data1/\"+lang+\"/\"+lang+\"_test.tsv\",sep=\"\\t\")\n",
    "\n",
    "import pickle\n",
    "file = open(\"data1/elmo_embeddings/elmo_train_\"+lang+\".pickle\",'rb')\n",
    "elmo_array = pickle.load(file)\n",
    "file.close()\n",
    "print(elmo_array.shape)\n",
    "\n",
    "elmo_train = elmo_array[:len(df_train),:]\n",
    "print(elmo_train.shape)\n",
    "elmo_valid = elmo_array[len(df_train):len(df_train)+len(df_dev),:]\n",
    "print(elmo_valid.shape)\n",
    "\n",
    "elmo_train_valid = elmo_array[:len(df_train)+len(df_dev),:]\n",
    "print(elmo_train_valid.shape)\n",
    "\n",
    "elmo_test = elmo_array[len(df_train)+len(df_dev):,:]\n",
    "print(elmo_test.shape)\n",
    "\n",
    "print(df_test.head())\n",
    "\n",
    "train_sentences = list(df_train['text'].values)\n",
    "dev_sentences = list(df_dev['text'].values)\n",
    "\n",
    "train_labels = list(df_train['category'].values)\n",
    "dev_labels = list(df_dev['category'].values)\n",
    "\n",
    "test_sentences = list(df_test['text'].values)\n",
    "\n",
    "\n",
    "total_sentences = []\n",
    "total_sentences.extend(train_sentences)\n",
    "total_sentences.extend(dev_sentences)\n",
    "total_sentences.extend(test_sentences)\n",
    "\n",
    "total_labels = []\n",
    "total_labels.extend(train_labels)\n",
    "total_labels.extend(dev_labels)\n",
    "\n",
    "len(total_sentences)\n",
    "\n",
    "fout = open(lang+\"_total_input_sentences.txt\",\"w+\",encoding='utf-8')\n",
    "\n",
    "for line in total_sentences:\n",
    "    fout.write(line+\"\\n\")\n",
    "fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "spm.SentencePieceTrainer.train(input=lang+\"_total_input_sentences.txt\", model_prefix='t_m')\n",
    "\n",
    "sp = spm.SentencePieceProcessor(model_file='t_m.model')\n",
    "\n",
    "tokenized_file = sp.encode(train_sentences, out_type=str)\n",
    "\n",
    "valid_tokenized_file = sp.encode(dev_sentences,out_type=str)\n",
    "\n",
    "test_tokenized_file = sp.encode(test_sentences,out_type=str)\n",
    "\n",
    "def join_token_to_line(token_file):\n",
    "    new_sentence_tokenized_file = []\n",
    "    \n",
    "    for line in token_file:\n",
    "        new_sentence_tokenized_file.append(\" \".join(line))\n",
    "        \n",
    "    return new_sentence_tokenized_file\n",
    "\n",
    "new_tokenized_train = join_token_to_line(tokenized_file)\n",
    "new_tokenized_valid = join_token_to_line(valid_tokenized_file)\n",
    "new_tokenized_test = join_token_to_line(test_tokenized_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11335, 6398)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(sublinear_tf = True,\n",
    "                             use_idf = True)\n",
    "\n",
    "train_vectors = vectorizer.fit_transform(new_tokenized_train)\n",
    "valid_vectors = vectorizer.transform(new_tokenized_valid)\n",
    "test_vectors = vectorizer.transform(new_tokenized_test)\n",
    "\n",
    "train_vectors = np.array(train_vectors.todense())\n",
    "valid_vectors = np.array(valid_vectors.todense())\n",
    "test_vectors = np.array(test_vectors.todense())\n",
    "\n",
    "train_elmo_tf_idf = np.concatenate((elmo_train,train_vectors),axis=1)\n",
    "valid_elmo_tf_idf = np.concatenate((elmo_valid,valid_vectors),axis=1)\n",
    "test_elmo_tf_idf = np.concatenate((elmo_test,test_vectors),axis=1)\n",
    "\n",
    "concatenated_column_size = train_elmo_tf_idf.shape[1]\n",
    "\n",
    "print(train_elmo_tf_idf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_v_columns = ['text','label']\n",
    "\n",
    "df_train1 = pd.DataFrame(zip(new_tokenized_train,train_labels),columns=t_v_columns)\n",
    "df_valid1 = pd.DataFrame(zip(new_tokenized_valid,dev_labels),columns=t_v_columns)\n",
    "\n",
    "df_test1 = pd.DataFrame(new_tokenized_test,columns=['text'])\n",
    "\n",
    "df_train1.to_csv(lang+\"_df_train1.tsv\",sep=\"\\t\")\n",
    "df_valid1.to_csv(lang+\"_df_valid1.tsv\",sep=\"\\t\")\n",
    "df_test1.to_csv(lang+\"_df_test1.tsv\",sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11335 1260 3149\n"
     ]
    }
   ],
   "source": [
    "print(len(df_train1),len(df_valid1),len(df_test1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of training examples: 11335\n",
      "Number of validation examples: 1260\n",
      "Number of testing examples: 3149\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15210717b80e4ae3994bb4fa0a287f01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=8211.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<unk>\n",
      "<pad>\n",
      "<sos>\n",
      "<eos>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\ipykernel_launcher.py:37: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "..\\torch\\csrc\\utils\\tensor_numpy.cpp:141: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program.\n"
     ]
    }
   ],
   "source": [
    "tokenize = lambda x: x.split()\n",
    "TEXT= data.Field(sequential=True,tokenize=tokenize,use_vocab=True,batch_first=True,init_token='<sos>', eos_token='<eos>')\n",
    "LABEL = data.LabelField()\n",
    "INDEX = data.LabelField()\n",
    "train_valid_fields = [('index',INDEX),('text',TEXT),('label',LABEL)]\n",
    "test_field = [('index',INDEX),('text',TEXT)]\n",
    "\n",
    "train_data = data.TabularDataset(lang+\"_df_train1.tsv\",format=\"tsv\",skip_header=True,fields=train_valid_fields)\n",
    "valid_data = data.TabularDataset(lang+\"_df_valid1.tsv\",format='tsv',skip_header=True,fields=train_valid_fields)\n",
    "test_data = data.TabularDataset(lang+\"_df_test1.tsv\",format='tsv',skip_header=True,fields=test_field)\n",
    "\n",
    "#print(vars(train_data.examples[len(df_train)-1]))\n",
    "\n",
    "print(f\"Number of training examples: {len(train_data)}\")\n",
    "print(f\"Number of validation examples: {len(valid_data)}\")\n",
    "print(f\"Number of testing examples: {len(test_data)}\")\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_data, valid_data, test_data), \n",
    "     batch_size = BATCH_SIZE,\n",
    "     device = device,sort=False)\n",
    "\n",
    "INDEX.build_vocab(train_data,valid_data,test_data)\n",
    "TEXT.build_vocab(train_data,valid_data,test_data)\n",
    "LABEL.build_vocab(train_data,valid_data)\n",
    "\n",
    "from gensim.models import FastText\n",
    "model_ft = FastText.load(\"fast_text_models/\"+lang+\"_300/model_ft_15\")\n",
    "\n",
    "W2V_SIZE=300\n",
    "from tqdm.notebook import tqdm\n",
    "word2vec_vectors = []\n",
    "for token, idx in tqdm(TEXT.vocab.stoi.items()):\n",
    "    if token in model_ft.wv.vocab.keys():\n",
    "        word2vec_vectors.append(torch.FloatTensor(model_ft[token]))\n",
    "    else:\n",
    "        print(token)\n",
    "        word2vec_vectors.append(torch.zeros(W2V_SIZE))\n",
    "TEXT.vocab.set_vectors(TEXT.vocab.stoi, word2vec_vectors, W2V_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self,src_vocab_size,output_dim,embedding_dim,hidden_dim,n_layers,n_heads,pf_dim,dropout,device,bidirectional=True,max_length=100):\n",
    "        super().__init__()\n",
    "        self.device = device\n",
    "        self.tok_embed_part1 = nn.Embedding(src_vocab_size,embedding_dim)\n",
    "        self.tok_embed_part2 = nn.Linear(embedding_dim,hidden_dim)\n",
    "        self.pos_embed = nn.Embedding(max_length,hidden_dim)\n",
    "        self.layers = nn.ModuleList([EncoderLayer(hidden_dim,n_heads,pf_dim,dropout,device) for _ in range(n_layers)])\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.scale = torch.sqrt(torch.FloatTensor([hidden_dim])).to(device)\n",
    "        \n",
    "        self.rnn = nn.GRU(hidden_dim,\n",
    "                          hidden_dim,\n",
    "                          num_layers = 2,\n",
    "                          bidirectional = bidirectional,\n",
    "                          batch_first = True,\n",
    "                          dropout = 0 if n_layers < 2 else dropout)\n",
    "        \n",
    "        self.out1 = nn.Linear(hidden_dim * 2 + concatenated_column_size,output_dim)\n",
    "    \n",
    "    def forward(self,src,src_mask,elmo_embed):\n",
    "        batch_size = src.shape[0]\n",
    "        seq_length = src.shape[1]\n",
    "        position = torch.arange(0,seq_length).unsqueeze(0).repeat(batch_size,1).to(device)\n",
    "        src = self.tok_embed_part1(src)\n",
    "        embedded = self.dropout(self.tok_embed_part2(src) * self.scale +self.pos_embed(position))\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            encoded = layer(embedded,src_mask)\n",
    "        \n",
    "        _, hidden = self.rnn(encoded)\n",
    "        \n",
    "        if self.rnn.bidirectional:\n",
    "            hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
    "        else:\n",
    "            hidden = self.dropout(hidden[-1,:,:])\n",
    "        \n",
    "        hidden_with_elmo = torch.cat((hidden,elmo_embed),dim=1)\n",
    "        \n",
    "        #print(hidden_with_elmo.shape)\n",
    "                \n",
    "        output = self.out1(hidden_with_elmo)\n",
    "            \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self,hidden_dim,n_heads,pf_dim,dropout,device):\n",
    "        super().__init__()\n",
    "        self.attention_norm = nn.LayerNorm(hidden_dim)\n",
    "        self.pf_norm = nn.LayerNorm(hidden_dim)\n",
    "        self.attention = MultiHeadedAttention(hidden_dim,n_heads,dropout,device)\n",
    "        self.pf = PositionwiseFeedForward(hidden_dim,pf_dim,dropout)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    def forward(self,src,src_mask):\n",
    "        _src,_ = self.attention(src,src,src,src_mask)\n",
    "        src = self.attention_norm(src+self.dropout(_src))\n",
    "        _src = self.pf(src)\n",
    "        src = self.pf_norm(src+self.dropout(_src))\n",
    "        return src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadedAttention(nn.Module):\n",
    "    def __init__(self,hidden_dim,n_heads,dropout,device):\n",
    "        super().__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.n_heads = n_heads\n",
    "        self.head_dim = hidden_dim//n_heads\n",
    "        \n",
    "        self.fc_q = nn.Linear(hidden_dim,hidden_dim)\n",
    "        self.fc_k = nn.Linear(hidden_dim,hidden_dim)\n",
    "        self.fc_v = nn.Linear(hidden_dim,hidden_dim)\n",
    "        \n",
    "        self.fc_o = nn.Linear(hidden_dim,hidden_dim)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.scale = torch.sqrt(torch.FloatTensor([self.hidden_dim])).to(device)\n",
    "        \n",
    "    def forward(self,query,key,value,mask=None):\n",
    "        batch_size = query.shape[0]\n",
    "                \n",
    "        Q = self.fc_q(query)\n",
    "        K = self.fc_k(key)\n",
    "        V = self.fc_v(value)\n",
    "        \n",
    "        Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\n",
    "        \n",
    "        energy = torch.matmul(Q, K.permute(0, 1, 3, 2))\n",
    "        energy = energy/self.scale\n",
    "        if mask is not None:\n",
    "            energy = energy.masked_fill(mask == 0, -1e5)\n",
    "        \n",
    "        attention = torch.softmax(energy, dim = -1)\n",
    "        x = torch.matmul(self.dropout(attention), V)\n",
    "        x = x.permute(0, 2, 1, 3).contiguous()\n",
    "        x = x.view(batch_size, -1, self.hidden_dim)\n",
    "        x = self.fc_o(x)\n",
    "       \n",
    "        return x, attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PositionwiseFeedForward(nn.Module):\n",
    "    def __init__(self,hidden_dim,pf_dim,dropout):\n",
    "        super().__init__()\n",
    "        self.pf1 = nn.Linear(hidden_dim,pf_dim)\n",
    "        self.pf2 = nn.Linear(pf_dim,hidden_dim)\n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "    def forward(self,x):\n",
    "        x = self.dropout(torch.relu(self.pf1(x)))\n",
    "        x = self.pf2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class seq2seq(nn.Module):\n",
    "    def __init__(self,encoder,src_pad_idx,device):\n",
    "        super().__init__()\n",
    "        self.encoder = encoder\n",
    "        self.src_pad_idx = src_pad_idx\n",
    "        self.device = device\n",
    "        \n",
    "    def make_src_mask(self, src):\n",
    "        src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)\n",
    "        return src_mask\n",
    "\n",
    "    def forward(self,src,elmo_embed):\n",
    "        src_mask = self.make_src_mask(src)\n",
    "        enc_src = self.encoder(src, src_mask,elmo_embed)\n",
    "        return enc_src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seq2seq(\n",
      "  (encoder): Encoder(\n",
      "    (tok_embed_part1): Embedding(8211, 300)\n",
      "    (tok_embed_part2): Linear(in_features=300, out_features=256, bias=True)\n",
      "    (pos_embed): Embedding(100, 256)\n",
      "    (layers): ModuleList(\n",
      "      (0): EncoderLayer(\n",
      "        (attention_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (pf_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
      "        (attention): MultiHeadedAttention(\n",
      "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (pf): PositionwiseFeedForward(\n",
      "          (pf1): Linear(in_features=256, out_features=512, bias=True)\n",
      "          (pf2): Linear(in_features=512, out_features=256, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (dropout): Dropout(p=0.1, inplace=False)\n",
      "      )\n",
      "    )\n",
      "    (dropout): Dropout(p=0.1, inplace=False)\n",
      "    (rnn): GRU(256, 256, num_layers=2, batch_first=True, bidirectional=True)\n",
      "    (out1): Linear(in_features=6910, out_features=5, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "INPUT_DIM = len(TEXT.vocab)\n",
    "OUTPUT_DIM = 5\n",
    "EMBEDDING_DIM = TEXT.vocab.vectors.shape[1]\n",
    "HID_DIM = 256\n",
    "ENC_LAYERS = 1\n",
    "ENC_HEADS = 8\n",
    "ENC_PF_DIM = 512\n",
    "ENC_DROPOUT = 0.1\n",
    "BIDIRECIONAL = True\n",
    "\n",
    "BOS_TAG_ID = TEXT.vocab.stoi['<sos>']\n",
    "EOS_TAG_ID = TEXT.vocab.stoi['<eos>']\n",
    "PAD_TAG_ID = TEXT.vocab.stoi['<pad>']\n",
    "\n",
    "\n",
    "encoder = Encoder(INPUT_DIM,OUTPUT_DIM,EMBEDDING_DIM,HID_DIM, \n",
    "              ENC_LAYERS, \n",
    "              ENC_HEADS, \n",
    "              ENC_PF_DIM, \n",
    "              ENC_DROPOUT,\n",
    "              device)\n",
    "\n",
    "PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]\n",
    "model = seq2seq(encoder, PAD_IDX, device).to(device)\n",
    "\n",
    "print(model)\n",
    "\n",
    "LEARNING_RATE = 0.0005\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index = PAD_IDX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 5,099,839 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(f'The model has {count_parameters(model):,} trainable parameters')\n",
    "\n",
    "def initialize_weights(m):\n",
    "    if hasattr(m, 'weight') and m.weight.dim() > 1:\n",
    "        nn.init.xavier_uniform_(m.weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def categorical_accuracy(preds, y):\n",
    "    \"\"\"\n",
    "    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8\n",
    "    \"\"\"\n",
    "    max_preds = preds.argmax(dim = 1, keepdim = True) # get the index of the max probability\n",
    "    correct = max_preds.squeeze(1).eq(y)\n",
    "    return correct.sum() / torch.FloatTensor([y.shape[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, clip):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc= 0.0\n",
    "    \n",
    "    for i, batch in enumerate(iterator):\n",
    "        text = batch.text\n",
    "        label = batch.label\n",
    "        index_array = batch.index\n",
    "        \n",
    "        batch_elmo_vector = []\n",
    "        \n",
    "        for index_value in index_array:\n",
    "            batch_elmo_vector.append(train_elmo_tf_idf[index_value])\n",
    "        \n",
    "        batch_elmo_vector = torch.Tensor(batch_elmo_vector).to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        output = model(text,batch_elmo_vector)\n",
    "        \n",
    "        loss = criterion(output, label)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        \n",
    "        acc = categorical_accuracy(output,label)\n",
    "        \n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc /len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc= 0.0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for i, batch in enumerate(iterator):\n",
    "            text = batch.text\n",
    "            label = batch.label\n",
    "            \n",
    "            index_array = batch.index\n",
    "        \n",
    "            batch_elmo_vector = []\n",
    "\n",
    "            for index_value in index_array:\n",
    "                batch_elmo_vector.append(valid_elmo_tf_idf[index_value])\n",
    "\n",
    "            batch_elmo_vector = torch.Tensor(batch_elmo_vector).to(device)\n",
    "\n",
    "            output = model(text,batch_elmo_vector)\n",
    "\n",
    "            loss = criterion(output, label)\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "\n",
    "            acc = categorical_accuracy(output,label)\n",
    "\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc /len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 0m 16s\n",
      "\tTrain Loss: 0.747 | Train Acc: 67.31%\n",
      "\t Val. Loss: 0.658 |  Val. Acc: 68.78%\n",
      "model saved\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\torch\\serialization.py:402: UserWarning: Couldn't retrieve source code for container of type seq2seq. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\torch\\serialization.py:402: UserWarning: Couldn't retrieve source code for container of type Encoder. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\torch\\serialization.py:402: UserWarning: Couldn't retrieve source code for container of type EncoderLayer. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\torch\\serialization.py:402: UserWarning: Couldn't retrieve source code for container of type MultiHeadedAttention. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n",
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\torch\\serialization.py:402: UserWarning: Couldn't retrieve source code for container of type PositionwiseFeedForward. It won't be checked for correctness upon loading.\n",
      "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 02 | Epoch Time: 0m 16s\n",
      "\tTrain Loss: 0.679 | Train Acc: 68.10%\n",
      "\t Val. Loss: 0.650 |  Val. Acc: 68.78%\n",
      "model saved\n",
      "Epoch: 03 | Epoch Time: 0m 16s\n",
      "\tTrain Loss: 0.642 | Train Acc: 68.59%\n",
      "\t Val. Loss: 0.666 |  Val. Acc: 68.31%\n",
      "Epoch: 04 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.595 | Train Acc: 69.12%\n",
      "\t Val. Loss: 0.661 |  Val. Acc: 68.70%\n",
      "Epoch: 05 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.555 | Train Acc: 70.06%\n",
      "\t Val. Loss: 0.688 |  Val. Acc: 68.46%\n",
      "Epoch: 06 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.510 | Train Acc: 71.43%\n",
      "\t Val. Loss: 0.650 |  Val. Acc: 67.21%\n",
      "Epoch: 07 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.450 | Train Acc: 73.02%\n",
      "\t Val. Loss: 0.682 |  Val. Acc: 65.31%\n",
      "Epoch: 08 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.401 | Train Acc: 74.63%\n",
      "\t Val. Loss: 0.744 |  Val. Acc: 64.74%\n",
      "Epoch: 09 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.336 | Train Acc: 76.59%\n",
      "\t Val. Loss: 0.832 |  Val. Acc: 62.16%\n",
      "Epoch: 10 | Epoch Time: 0m 16s\n",
      "\tTrain Loss: 0.285 | Train Acc: 78.41%\n",
      "\t Val. Loss: 0.864 |  Val. Acc: 64.71%\n",
      "Epoch: 11 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.252 | Train Acc: 79.48%\n",
      "\t Val. Loss: 0.908 |  Val. Acc: 64.06%\n",
      "Epoch: 12 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.200 | Train Acc: 81.18%\n",
      "\t Val. Loss: 1.047 |  Val. Acc: 64.30%\n",
      "Epoch: 13 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.173 | Train Acc: 81.67%\n",
      "\t Val. Loss: 1.292 |  Val. Acc: 60.89%\n",
      "Epoch: 14 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.164 | Train Acc: 82.19%\n",
      "\t Val. Loss: 1.150 |  Val. Acc: 65.13%\n",
      "Epoch: 15 | Epoch Time: 0m 15s\n",
      "\tTrain Loss: 0.143 | Train Acc: 83.07%\n",
      "\t Val. Loss: 1.250 |  Val. Acc: 62.11%\n"
     ]
    }
   ],
   "source": [
    "CLIP = 1\n",
    "epochs = 15\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss,train_acc = train(model, train_iterator, optimizer, criterion, CLIP)\n",
    "    valid_loss,valid_acc = evaluate(model, valid_iterator, criterion)\n",
    "    \n",
    "    end_time = time.time()\n",
    "        \n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model,'model_'+lang+'.pt')\n",
    "        print(\"model saved\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "del model\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score,classification_report,f1_score\n",
    "\n",
    "def metrics_test(y_pred, y):\n",
    "    accuracy = accuracy_score(y_pred,y)\n",
    "    print(\"accuracy score is: \",accuracy)\n",
    "    print(\"weighted_f1 is: \",f1_score(y_pred,y,average='weighted'))\n",
    "    print(\"classification metric\")\n",
    "    print(classification_report(y_pred,y))\n",
    "    \n",
    "    return accuracy\n",
    "\n",
    "def return_y_predictions(predictions):\n",
    "    max_preds = predictions.argmax(dim = 1, keepdim = True)\n",
    "    max_preds = max_preds.squeeze().cpu()\n",
    "    #print(max_preds.shape)\n",
    "    return max_preds\n",
    "\n",
    "def evaluate_f1_accuracy_valid(model, iterator, criterion):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc= 0.0\n",
    "    \n",
    "    y_preds_list=[]\n",
    "    y_true_list=[]\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for i, batch in enumerate(iterator):\n",
    "            text = batch.text\n",
    "            label = batch.label\n",
    "            \n",
    "            index_array = batch.index\n",
    "        \n",
    "            batch_elmo_vector = []\n",
    "\n",
    "            for index_value in index_array:\n",
    "                batch_elmo_vector.append(valid_elmo_tf_idf[index_value])\n",
    "\n",
    "            batch_elmo_vector = torch.Tensor(batch_elmo_vector).to(device)\n",
    "\n",
    "            output = model(text,batch_elmo_vector)\n",
    "\n",
    "            loss = criterion(output, label)\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            \n",
    "            for true_value in label.squeeze().cpu():\n",
    "                y_true_list.append(true_value.item())\n",
    "            \n",
    "            max_preds_tensor = return_y_predictions(output)\n",
    "            for max_value in max_preds_tensor:\n",
    "                y_preds_list.append(max_value.item())\n",
    "            \n",
    "        \n",
    "    acc = metrics_test(y_preds_list,y_true_list)    \n",
    "        \n",
    "    return epoch_loss/len(iterator),y_preds_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(None, {'Positive ': 0, 'Negative ': 1, 'Mixed_feelings ': 2, 'unknown_state ': 3, 'not-Tamil ': 4})\n",
      "accuracy score is:  0.6880952380952381\n",
      "weighted_f1 is:  0.8072992396997715\n",
      "classification metric\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.69      0.82      1235\n",
      "           1       0.00      0.00      0.00         0\n",
      "           2       0.00      0.00      0.00         0\n",
      "           3       0.03      0.40      0.05         5\n",
      "           4       0.41      0.60      0.49        20\n",
      "\n",
      "   micro avg       0.69      0.69      0.69      1260\n",
      "   macro avg       0.29      0.34      0.27      1260\n",
      "weighted avg       0.98      0.69      0.81      1260\n",
      "\n",
      "\t Val. Loss: 0.650\n",
      "Counter({'Positive ': 857, 'Negative ': 165, 'Mixed_feelings ': 141, 'unknown_state ': 68, 'not-Tamil ': 29})\n",
      "Counter({0: 1235, 4: 20, 3: 5})\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\sklearn\\metrics\\classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n",
      "C:\\Users\\suman\\Miniconda3\\envs\\py3_env\\lib\\site-packages\\sklearn\\metrics\\classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(LABEL.vocab.stoi)\n",
    "model = torch.load('model_'+lang+'.pt')\n",
    "model = model.to(device)\n",
    "valid_loss,y_preds_list = evaluate_f1_accuracy_valid(model, valid_iterator, criterion)\n",
    "print(f'\\t Val. Loss: {valid_loss:.3f}')\n",
    "\n",
    "from collections import Counter\n",
    "print(Counter(list(df_valid1['label'].values)))\n",
    "print(Counter(y_preds_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_test(model, iterator):\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    y_preds_list=[]\n",
    "    index_list = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for i, batch in enumerate(iterator):\n",
    "            text = batch.text\n",
    "            #label = batch.label\n",
    "            \n",
    "            index_array = batch.index\n",
    "            #print(index_array)\n",
    "        \n",
    "            batch_elmo_vector = []\n",
    "\n",
    "            for index_value in index_array:\n",
    "                batch_elmo_vector.append(test_elmo_tf_idf[index_value])\n",
    "\n",
    "            batch_elmo_vector = torch.Tensor(batch_elmo_vector).to(device)\n",
    "\n",
    "            output = model(text,batch_elmo_vector)\n",
    "            \n",
    "            max_preds_tensor = return_y_predictions(output)\n",
    "            \n",
    "            for index_value in index_array:\n",
    "                index_list.append(index_value.cpu().item())\n",
    "            \n",
    "            for max_value in max_preds_tensor:\n",
    "                y_preds_list.append(max_value.item())\n",
    "        \n",
    "    return index_list,y_preds_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "index_list,y_preds_list = evaluate_test(model, test_iterator)\n",
    "\n",
    "dict_index_y_value = {}\n",
    "\n",
    "for index,y_value in zip(index_list,y_preds_list):\n",
    "    dict_index_y_value[index] = LABEL.vocab.itos[y_value]\n",
    "\n",
    "od = OrderedDict(sorted(dict_index_y_value.items()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_y_predicts = []\n",
    "for k,v in od.items():\n",
    "    sorted_y_predicts.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sentences = list(df_test['text'].values)\n",
    "test_id = list(df_test['id'].values)\n",
    "test_final = pd.DataFrame(zip(test_id,test_sentences,sorted_y_predicts),columns=['id','text','label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_final.to_csv(\"CMSA11-One_\"+lang+\".tsv\",sep=\"\\t\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (py3_env)",
   "language": "python",
   "name": "py3_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}