Sub-task-B.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sub-task-B.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "xYJAFZB7_orN"
      },
      "source": [
        "#### Git cloning the Training data \n",
        "!git clone https://github.com/ncg-task/training-data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5MkeZihuCD3C"
      },
      "source": [
        "#### Git cloning the Validation data\n",
        "!git clone https://github.com/ncg-task/trial-data.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-56AAQklwkKs"
      },
      "source": [
        "#### Load the required packages and setting the seed\n",
        "import os\n",
        "import copy\n",
        "import random\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.autograd as autograd\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import time\n",
        "seed_val = 66\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1BGgoMLPzYxP"
      },
      "source": [
        "#### Training dataset reading\n",
        "input_dir = \"/content/training-data/\" #Point it to the directory where your Training Data is present\n",
        "list_of_folders = [\"query_wellformedness\", \"passage_re-ranking\", \"part-of-speech_tagging\", \n",
        "         \"sentence_compression\", \"sentiment_analysis\", \"temporal_information_extraction\", \n",
        "         \"phrase_grounding\", \"text_generation\", \"text-to-speech_synthesis\", \n",
        "         \"smile_recognition\", \"topic_models\", \"question_generation\", \n",
        "         \"relation_extraction\", \"paraphrase_generation\", \"question_similarity\", \n",
        "         \"question_answering\", \"sentence_classification\", \"prosody_prediction\", \n",
        "         \"semantic_role_labeling\", \"text_summarization\", \"semantic_parsing\", \n",
        "         \"sarcasm_detection\", \"natural_language_inference\", \"negation_scope_resolution\"] #List all the folders present in the directory\n",
        "input_stanza_list = [] #Stores individual lines from Stanza_out.txt file\n",
        "input_sent_num_list = [] #Stores individual lines from sentences.txt file\n",
        "input_entity_list = [] #Stores the list of all phrases from entities.txt\n",
        "file_name_list = [] #Stores the name of individual files\n",
        "total_phrases_truth = 0\n",
        "for fls in list_of_folders:\n",
        "  count=0\n",
        "  for i in os.listdir(input_dir + fls + '/'):\n",
        "    count=count+1\n",
        "    for files in os.listdir(input_dir + fls + '/' + str(i)):\n",
        "      if files.endswith(\"Stanza-out.txt\"):\n",
        "        stanza_file = open(input_dir + fls + '/' + str(i) + '/' + files, \"r\")\n",
        "        stanza_lines = (stanza_file.read()).lower()\n",
        "        stanza_lines_list = list(filter(None,stanza_lines.splitlines())) # filter empty strings and split into lines\n",
        "        stanza_lines_list_trimmed = [x.strip() for x in stanza_lines_list]\n",
        "        input_stanza_list.append(stanza_lines_list_trimmed)\n",
        "      if files.endswith(\"sentences.txt\"):\n",
        "        sentence_file = open(input_dir + fls + '/' + str(i) + '/' + files, \"r\")\n",
        "        sentence_num_list = list(filter(None,(sentence_file.read().lower()).splitlines())) # filter empty strings and split into lines\n",
        "        sentence_num_list_trimmed  = [x.strip() for x in sentence_num_list]\n",
        "        input_sent_num_list.append(list(map(int, sentence_num_list_trimmed)))\n",
        "      if files.endswith(\"entities.txt\"):\n",
        "        entities_file = open(input_dir + fls + '/' + str(i) + '/' + files, \"r\")\n",
        "        entities_list = list(filter(None,(entities_file.read().lower()).splitlines())) # filter empty strings and split into lines\n",
        "        entities_list_trimmed = [x.strip() for x in entities_list]\n",
        "        input_entity_list.append(entities_list_trimmed)\n",
        "        total_phrases_truth = total_phrases_truth + len(entities_list)\n",
        "    file_name_list.append(fls + '/' + str(i))\n",
        "  print(\"completed\",fls,total_phrases_truth, count)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NkJpU55b5lw9"
      },
      "source": [
        "#### Validation dataset reading\n",
        "val_input_dir = \"/content/trial-data/\" #Point it to the directory where your Validation Data is present\n",
        "val_list_of_folders = [\"machine-translation\", \"named-entity-recognition\", \"question-answering\",\n",
        "         \"relation-classification\", \"text-classification\"] #List all the folders present in the directory\n",
        "val_input_stanza_list = [] #Stores individual lines from Stanza_out.txt file\n",
        "val_input_sent_num_list = [] #Stores individual lines from sentences.txt file\n",
        "val_input_entity_list = [] #Stores the list of all phrases from entities.txt\n",
        "val_file_name_list = [] #Stores the name of individual files\n",
        "val_total_phrases_truth = 0\n",
        "for fls in val_list_of_folders:\n",
        "  count=0\n",
        "  for i in os.listdir(val_input_dir + fls + '/'):\n",
        "    count=count+1\n",
        "    for files in os.listdir(val_input_dir + fls + '/' + str(i)):\n",
        "      if files.endswith(\"Stanza-out.txt\"):\n",
        "        stanza_file = open(val_input_dir + fls + '/' + str(i) + '/' + files, \"r\")\n",
        "        stanza_lines = stanza_file.read().lower()\n",
        "        stanza_lines_list = list(filter(None,stanza_lines.splitlines())) # filter empty strings and split into lines\n",
        "        stanza_lines_list_trimmed = [x.strip() for x in stanza_lines_list]\n",
        "        val_input_stanza_list.append(stanza_lines_list_trimmed)\n",
        "      if files.endswith(\"sentences.txt\"):\n",
        "        sentence_file = open(val_input_dir + fls + '/' + str(i) + '/' + files, \"r\")\n",
        "        sentence_num_list = list(filter(None,(sentence_file.read().lower()).splitlines())) # filter empty strings and split into lines\n",
        "        sentence_num_list_trimmed  = [x.strip() for x in sentence_num_list]\n",
        "        val_input_sent_num_list.append(list(map(int, sentence_num_list_trimmed)))\n",
        "      if files.endswith(\"entities.txt\"):\n",
        "        entities_file = open(val_input_dir + fls + '/' + str(i) + '/' + files, \"r\")\n",
        "        entities_list = list(filter(None,(entities_file.read().lower()).splitlines())) # filter empty strings and split into lines\n",
        "        entities_list_trimmed = [x.strip() for x in entities_list]\n",
        "        val_input_entity_list.append(entities_list_trimmed)\n",
        "        val_total_phrases_truth = val_total_phrases_truth + len(entities_list)\n",
        "    val_file_name_list.append(fls + '/' + str(i))\n",
        "  print(\"completed\",fls,val_total_phrases_truth,count)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hf8Tvo_h7MLX"
      },
      "source": [
        "#### It returns BILUO labels based on number of phrases(n)\n",
        "def make_substring(n): \n",
        "  if n==0:\n",
        "    return ''\n",
        "  elif n ==1:\n",
        "    return 'U'\n",
        "  elif n==2:\n",
        "    return 'B L'\n",
        "  else:\n",
        "    t1 = 'I '*(n-2)\n",
        "    return 'B '+t1+'L'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mZx-f4yZPugG"
      },
      "source": [
        "#### Here we replace contribution sentences in Training data with BILUO labels using the helper function make_substring  \n",
        "task2_in = [] #Stores the contribution sentences\n",
        "task2_label = [] #Stores the BILUO labelling of contribution sentences\n",
        "\n",
        "for i in range(len(input_stanza_list)): #Process each file for labels\n",
        "  entity_list = [j.split('\\t') for j in input_entity_list[i]] #Split the entities line\n",
        "  entity_list.sort(key=lambda x: (int(x[0]),int(x[1]))) #Phrases arrangement in ascending order according to their sentence numbers and their starting character numbers\n",
        "  sent_num_list = copy.deepcopy(input_sent_num_list[i]) #Copy of the sentences list\n",
        "  sent_num_list.sort() #Sort the sentences list\n",
        "  sent_list = [] #Temporarily stores contribution sentences\n",
        "  \n",
        "  for x in sent_num_list: \n",
        "    sent_list.append(input_stanza_list[i][x-1])\n",
        "  \n",
        "  sent_dict_list = dict(zip(sent_num_list,sent_list)) #Dictionary of sentence number and their corresponding sentences\n",
        "\n",
        "  for n ,ind_s ,ind_e, ph in entity_list: #BILUO Label Formation using phrases stored in entity_list\n",
        "    if int(n) in sent_num_list:\n",
        "      sent_dict_list[int(n)] = sent_dict_list[int(n)].replace(ph,make_substring(len(ph.split())),1)\n",
        "\n",
        "  sent_label_list = list(sent_dict_list.values())\n",
        "  task2_in.append(sent_list)\n",
        "  task2_label.append(sent_label_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lyzZwVYnP7-y"
      },
      "source": [
        "#### Here we replace contribution sentences in Validation data with BILUO labels using the helper function make_substring  \n",
        "val_task2_in = [] #Stores the contribution sentences\n",
        "val_task2_label = [] #Stores the BILUO labelling of contribution sentences\n",
        "for i in range(len(val_input_stanza_list)): #Process each file for labels\n",
        "  val_entity_list = [j.split('\\t') for j in val_input_entity_list[i]] # Split the entities line\n",
        "  val_entity_list.sort(key=lambda x: (int(x[0]),int(x[1]))) #Phrases arrangement in ascending order according to their sentence numbers and their starting character numbers\n",
        "  val_sent_num_list = copy.deepcopy(val_input_sent_num_list[i]) #Copy of the sentences list\n",
        "  val_sent_num_list.sort() #Sort the sentences list\n",
        "  val_sent_list = []  #Temporarily stores contribution sentences\n",
        "  \n",
        "  for x in val_sent_num_list: \n",
        "    val_sent_list.append(val_input_stanza_list[i][x-1])\n",
        "  \n",
        "  val_sent_dict_list = dict(zip(val_sent_num_list,val_sent_list)) #Dictionary of sentence number and their corresponding sentences\n",
        "  for n ,ind_s ,ind_e, ph in val_entity_list: #BILUO Label Formation using phrases stored in val_entity_list\n",
        "    if int(n) in val_sent_num_list:\n",
        "      val_sent_dict_list[int(n)] = val_sent_dict_list[int(n)].replace(ph,make_substring(len(ph.split())),1)\n",
        "      \n",
        "  val_sent_label_list = list(val_sent_dict_list.values())\n",
        "  val_task2_in.append(val_sent_list)\n",
        "  val_task2_label.append(val_sent_label_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s3hT4uteRYF2"
      },
      "source": [
        "#### Replacing all words which are not \"BILU\", in contribution sentences of Training data with \"O\" token  \n",
        "for i,out in enumerate(task2_label): \n",
        "  for j,line in enumerate(out):\n",
        "    for k,tok in enumerate(line.split()):\n",
        "      if tok not in ['B','I','L','U']:\n",
        "        task2_label[i][j]=task2_label[i][j].replace(tok,'O',1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sm2Gp5TOoAsm"
      },
      "source": [
        "#### Replacing all words which are not \"BILU\", in contribution sentences of Validation data with \"O\" token  \n",
        "for i,out in enumerate(val_task2_label): \n",
        "  for j,line in enumerate(out):\n",
        "    for k,tok in enumerate(line.split()):\n",
        "      if tok not in ['B','I','L','U']:\n",
        "        val_task2_label[i][j]=val_task2_label[i][j].replace(tok,'O',1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p8tYVQRhRqqa"
      },
      "source": [
        "#### Helper Functions\n",
        "\n",
        "#Return the argmax as a python int\n",
        "def argmax(vec):\n",
        "    _, idx = torch.max(vec, 1)\n",
        "    return idx.item()\n",
        "\n",
        "#The function uses SciBERT tokenizer to prepare the sentence to be fed into SciBERT \n",
        "def prepare_sequence(seq):\n",
        "    for count,i in enumerate(seq):\n",
        "       temp = tokenizer.tokenize(i) \n",
        "       if(len(temp)>1): #Only the first token of the word has been considered(same as in NER paper) \n",
        "         seq[count] = temp[0]           \n",
        "    sentences = \" \".join(seq)\n",
        "    inputs = tokenizer(sentences, return_tensors=\"pt\")\n",
        "    return inputs\n",
        "\n",
        "\n",
        "# Compute log sum exp in a numerically stable way for the forward algorithm\n",
        "def log_sum_exp(vec):\n",
        "    max_score = vec[0, argmax(vec)]\n",
        "    max_score_broadcast = max_score.view(1, -1).expand(1, vec.size()[1])\n",
        "    return max_score + \\\n",
        "        torch.log(torch.sum(torch.exp(vec - max_score_broadcast)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B-XL1T2ySEwF"
      },
      "source": [
        "#### Define the Model \n",
        "\n",
        "class BiLSTM_CRF(nn.Module):\n",
        "\n",
        "    def __init__(self, tag_to_ix, embedding_dim, hidden_dim):\n",
        "        super(BiLSTM_CRF, self).__init__()\n",
        "        self.embedding_dim = embedding_dim\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.tag_to_ix = tag_to_ix\n",
        "        self.tagset_size = len(tag_to_ix)\n",
        "\n",
        "        self.lstm = nn.LSTM(embedding_dim, hidden_dim // 2,\n",
        "                            num_layers=1, bidirectional=True)\n",
        "\n",
        "        #Maps the output of the LSTM into tag space.\n",
        "        self.hidden2tag = nn.Linear(hidden_dim, self.tagset_size)\n",
        "\n",
        "        #Pretrained SciBert downloaded from allenai\n",
        "        self.modell = AutoModel.from_pretrained(\"allenai/scibert_scivocab_uncased\")\n",
        "\n",
        "        #Matrix of transition parameters. Entry i,j is the score of transitioning to i from j.\n",
        "        self.transitions = nn.Parameter(\n",
        "            torch.randn(self.tagset_size, self.tagset_size))\n",
        "\n",
        "        #These two statements enforce the constraint that we never transfer to the start tag and we never transfer from the stop tag\n",
        "        self.transitions.data[tag_to_ix[START_TAG], :] = -10000\n",
        "        self.transitions.data[:, tag_to_ix[STOP_TAG]] = -10000\n",
        "\n",
        "        self.hidden = self.init_hidden()\n",
        "\n",
        "    def init_hidden(self): \n",
        "        return (torch.randn(2, 1, self.hidden_dim // 2).to(device),\n",
        "                torch.randn(2, 1, self.hidden_dim // 2).to(device))\n",
        "\n",
        "    def _forward_alg(self, feats):\n",
        "        #Do the forward algorithm to compute the partition function\n",
        "        init_alphas = torch.full((1, self.tagset_size), -10000.).to(device)\n",
        "        #START_TAG has all of the score.\n",
        "        init_alphas[0][self.tag_to_ix[START_TAG]] = 0.\n",
        "\n",
        "        #Wrap in a variable so that we will get automatic backprop\n",
        "        forward_var = init_alphas\n",
        "\n",
        "        #Iterate through the sentence\n",
        "        for feat in feats:\n",
        "            alphas_t = []  #The forward tensors at this timestep\n",
        "            for next_tag in range(self.tagset_size):\n",
        "                #Broadcast the emission score: it is the same regardless of the previous tag\n",
        "                emit_score = feat[next_tag].view(1, -1).expand(1, self.tagset_size)\n",
        "                #The ith entry of trans_score is the score of transitioning to next_tag from i\n",
        "                trans_score = self.transitions[next_tag].view(1, -1)\n",
        "                #The ith entry of next_tag_var is the value for th edge (i -> next_tag) before we do log-sum-exp\n",
        "                next_tag_var = forward_var + trans_score + emit_score\n",
        "                # The forward variable for this tag is log-sum-exp of all the scores.\n",
        "                alphas_t.append(log_sum_exp(next_tag_var).view(1))\n",
        "            forward_var = torch.cat(alphas_t).view(1, -1)\n",
        "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
        "        alpha = log_sum_exp(terminal_var)\n",
        "        return alpha\n",
        "\n",
        "    def _get_lstm_features(self, sentence):\n",
        "        #The tokenized sentence is passed through SciBERT and output from LSTM is mapped into tag space\n",
        "        outputs = self.modell(**sentence, output_hidden_states = True)\n",
        "        scibert_out = ((outputs[2][12])[0]).view(len(sentence[\"input_ids\"][0]), 1, -1)\n",
        "        self.hidden = self.init_hidden()\n",
        "        lstm_out, self.hidden = self.lstm(scibert_out, self.hidden)\n",
        "        lstm_out = lstm_out.view(len(sentence[\"input_ids\"][0]), self.hidden_dim)\n",
        "        lstm_feats = self.hidden2tag(lstm_out)\n",
        "        return lstm_feats\n",
        "\n",
        "    def _score_sentence(self, feats, tags):\n",
        "        #Gives the score of a provided tag sequence\n",
        "        score = torch.zeros(1).to(device)\n",
        "        tags = torch.cat([torch.tensor([self.tag_to_ix[START_TAG]], dtype=torch.long).to(device), tags])\n",
        "        for i, feat in enumerate(feats):\n",
        "            score = score + \\\n",
        "                self.transitions[tags[i + 1], tags[i]] + feat[tags[i + 1]]\n",
        "        score = score + self.transitions[self.tag_to_ix[STOP_TAG], tags[-1]]\n",
        "        return score\n",
        "\n",
        "    def _viterbi_decode(self, feats):\n",
        "        backpointers = []\n",
        "\n",
        "        #Initialize the viterbi variables in log space\n",
        "        init_vvars = torch.full((1, self.tagset_size), -10000.).to(device)\n",
        "        init_vvars[0][self.tag_to_ix[START_TAG]] = 0\n",
        "\n",
        "        #forward_var at step i holds the viterbi variables for step i-1\n",
        "        forward_var = init_vvars\n",
        "        for feat in feats:\n",
        "            bptrs_t = []  #Holds the backpointers for this step\n",
        "            viterbivars_t = [] #Holds the viterbi variables for this step\n",
        "\n",
        "            for next_tag in range(self.tagset_size):\n",
        "                #next_tag_var[i] holds the viterbi variable for tag i at the previous step, plus the score of transitioning\n",
        "                #from tag i to next_tag. We don't include the emission scores here because the max does not depend on them (we add them in below)\n",
        "                next_tag_var = forward_var + self.transitions[next_tag]\n",
        "                best_tag_id = argmax(next_tag_var)\n",
        "                bptrs_t.append(best_tag_id)\n",
        "                viterbivars_t.append(next_tag_var[0][best_tag_id].view(1))\n",
        "            #Now we add in the emission scores, and assign forward_var to the set of viterbi variables we just computed\n",
        "            forward_var = (torch.cat(viterbivars_t) + feat).view(1, -1)\n",
        "            backpointers.append(bptrs_t)\n",
        "\n",
        "        #Transition to STOP_TAG\n",
        "        terminal_var = forward_var + self.transitions[self.tag_to_ix[STOP_TAG]]\n",
        "        best_tag_id = argmax(terminal_var)\n",
        "        path_score = terminal_var[0][best_tag_id]\n",
        "\n",
        "        #Follow the back pointers to decode the best path.\n",
        "        best_path = [best_tag_id]\n",
        "        for bptrs_t in reversed(backpointers):\n",
        "            best_tag_id = bptrs_t[best_tag_id]\n",
        "            best_path.append(best_tag_id)\n",
        "        #Pop off the start tag (we dont want to return that to the caller)\n",
        "        start = best_path.pop()\n",
        "        assert start == self.tag_to_ix[START_TAG]  #Sanity check\n",
        "        best_path.reverse()\n",
        "        return path_score, best_path\n",
        "\n",
        "    def neg_log_likelihood(self, sentence, tags):\n",
        "        #The loss function \n",
        "        feats = self._get_lstm_features(sentence)\n",
        "        forward_score = self._forward_alg(feats)\n",
        "        gold_score = self._score_sentence(feats, tags)\n",
        "        return forward_score - gold_score\n",
        "\n",
        "    def forward(self, sentence):  \n",
        "        #Get the emission scores from the BiLSTM\n",
        "        lstm_feats = self._get_lstm_features(sentence)\n",
        "\n",
        "        #Find the best path, given the features.\n",
        "        score, tag_seq = self._viterbi_decode(lstm_feats)\n",
        "        return score, tag_seq"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xL9ZAlF8qpWh"
      },
      "source": [
        "#### Model hyperparameters defined\n",
        "START_TAG = \"<START>\"\n",
        "STOP_TAG = \"<STOP>\"\n",
        "EMBEDDING_DIM = 768 #Since SciBERT encodes tokens of a sentence in (768,1) dimension.\n",
        "HIDDEN_DIM = 200 #For BiLSTM"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cLTBLDJWq492"
      },
      "source": [
        "#### Maps BILUO labels to index numbers and vice-versa.\n",
        "tag_to_ix = {\"B\": 0, \"I\": 1, \"L\": 2, \"U\": 3, \"O\": 4, START_TAG: 5, STOP_TAG: 6}\n",
        "ix_to_tag = {v: k for k, v in tag_to_ix.items()}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASxEf_1yT3Im"
      },
      "source": [
        "#### Allenai SciBERT\n",
        "!pip install transformers\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"allenai/scibert_scivocab_uncased\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F9s0cfdzQC0d"
      },
      "source": [
        "#### Initialize the Model and Optimizer\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "model = BiLSTM_CRF(tag_to_ix, EMBEDDING_DIM, HIDDEN_DIM)\n",
        "optimizer = optim.AdamW(model.parameters(), lr=2e-5)\n",
        "if (device=='cuda'):\n",
        "  model.cuda()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "27KBpWDwk2LE"
      },
      "source": [
        "#### listp is the list of predicted labels, listg is the list of ground truth labels. This function\n",
        "#calculates the number of correct predictions and the number of total phrases predicted.\n",
        " \n",
        "def calc_eval(listp, listg):\n",
        "  correct = 0\n",
        "  total_p = 0\n",
        "  for i in zip(listp, listg):\n",
        "    flag = 0\n",
        "    for j in zip(i[0], i[1]):\n",
        "      if j[0]=='U' or j[0]=='B':\n",
        "        total_p += 1\n",
        "      if flag==0:\n",
        "        if j[0]=='U' and j[1]=='U':\n",
        "          correct += 1\n",
        "        if j[0]=='B' and j[1]=='B':\n",
        "          flag = 1\n",
        "      else:\n",
        "        if j[0]=='L' and j[1]=='L':\n",
        "          correct += 1\n",
        "          flag = 0\n",
        "        elif j[0]==j[1]:\n",
        "          continue\n",
        "        else:\n",
        "          flag = 0\n",
        "  return correct, total_p"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vpbr0G7vtFQc"
      },
      "source": [
        "#### This function evaluates the precision, recall and F1 score achieved on training data using the helper function calc_eval \n",
        "def training_eval():\n",
        "  #task2_out_label stores the prediction labels made by the model\n",
        "  task2_out_label = []\n",
        "  for i,file in enumerate(task2_in):\n",
        "    output = []\n",
        "    for j,sent in enumerate(file):\n",
        "      with torch.no_grad():\n",
        "        precheck_sent = prepare_sequence(task2_in[i][j].split()).to(device)\n",
        "        sent_out_label = model(precheck_sent)[1]\n",
        "        sent_str_label = [ix_to_tag[t] for t in sent_out_label]   \n",
        "        output.append(sent_str_label)\n",
        "    task2_out_label.append(output)\n",
        "  print(len(task2_out_label))\n",
        "\n",
        "  true_pos = 0 #All correct predictions\n",
        "  total_ph_pred = 0 #All phrases prediction made(BILU)\n",
        "  for i,file in enumerate(task2_label):\n",
        "    file_true_pos, file_total_ph_pred = calc_eval(task2_out_label[i],[[\"O\"] + s.split() + [\"O\"] for s in task2_label[i]])\n",
        "    true_pos = true_pos + file_true_pos\n",
        "    total_ph_pred = total_ph_pred + file_total_ph_pred\n",
        "    if i%20==0:\n",
        "      print(\"Each example\",i,file_true_pos,file_total_ph_pred)\n",
        "  print(true_pos,total_ph_pred,total_phrases_truth)\n",
        "\n",
        "  precision = 0\n",
        "  recall = 0\n",
        "  F1score =  0\n",
        "  if(total_ph_pred!=0):\n",
        "    precision = true_pos/total_ph_pred\n",
        "  if(total_phrases_truth!=0):\n",
        "    recall = true_pos/total_phrases_truth\n",
        "  if((precision + recall)!=0):  \n",
        "    F1score = 2 * precision*recall/(precision+recall)\n",
        "  print(\"Precision is {} and recall is {} and F1 Score is {}\".format(precision,recall,F1score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sow8PPlLtVn8"
      },
      "source": [
        "#### This function evaluates the precision, recall and F1 score achieved on validation data using the helper function calc_eval \n",
        "def validation_eval():\n",
        "  #task2_out_label stores the prediction labels made by the model\n",
        "  val_task2_out_label = []\n",
        "  for i,file in enumerate(val_task2_in):\n",
        "    output = []\n",
        "    for j,sent in enumerate(file):\n",
        "      with torch.no_grad():\n",
        "        precheck_sent = prepare_sequence(val_task2_in[i][j].split()).to(device)\n",
        "        sent_out_label = model(precheck_sent)[1]\n",
        "        sent_str_label = [ix_to_tag[t] for t in sent_out_label]\n",
        "        output.append(sent_str_label)\n",
        "    val_task2_out_label.append(output)\n",
        "  print(len(val_task2_out_label))\n",
        "\n",
        "  val_true_pos = 0  #All correct predictions\n",
        "  val_total_ph_pred = 0 #All phrases prediction made(BILU)\n",
        "  for i,file in enumerate(val_task2_label):\n",
        "    val_file_true_pos, val_file_total_ph_pred = calc_eval(val_task2_out_label[i],[[\"O\"] + s.split() + [\"O\"] for s in val_task2_label[i]])\n",
        "    val_true_pos = val_true_pos + val_file_true_pos\n",
        "    val_total_ph_pred = val_total_ph_pred + val_file_total_ph_pred\n",
        "    print(\"Each example\",i,\"true pos\",val_file_true_pos,\"phrase in pred\",val_file_total_ph_pred)\n",
        "  print(val_true_pos,val_total_ph_pred,val_total_phrases_truth)\n",
        "\n",
        "  val_precision = 0\n",
        "  val_recall = 0\n",
        "  val_F1score =  0\n",
        "  if(val_total_ph_pred!=0):\n",
        "    val_precision = val_true_pos/val_total_ph_pred\n",
        "  if(val_total_phrases_truth!=0):\n",
        "    val_recall = val_true_pos/val_total_phrases_truth\n",
        "  if((val_precision + val_recall)!=0):  \n",
        "    val_F1score = 2 * val_precision*val_recall/(val_precision+val_recall)\n",
        "  print(\"Precision is {} and recall is {} and F1 Score is {}\".format(val_precision,val_recall,val_F1score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BozoCn89SRmm"
      },
      "source": [
        "#### The MAIN TRAINING CELL\n",
        "\n",
        "#Check predictions before training\n",
        "with torch.no_grad():\n",
        "    \n",
        "    precheck_sent = prepare_sequence(task2_in[0][0].split()).to(device) #Tokenize the sentence before passing it into SciBERT\n",
        "    precheck_tags = torch.tensor([4] + [tag_to_ix[t] for t in task2_label[0][0].split()] + [4], dtype=torch.long).to(device) #Ground Truth Labels\n",
        "    print(precheck_tags)\n",
        "    print(model(precheck_sent),'\\n',precheck_tags)\n",
        "    print(\"checkpoint passed\")\n",
        "\n",
        "\n",
        "for epoch in range(1): #Decide the number of epochs\n",
        "    start = time.time() #Starting time measured\n",
        "    model.train() #Model in training mode\n",
        "\n",
        "    for i,file in enumerate(task2_in):\n",
        "      for j,sent in enumerate(file):\n",
        "        \n",
        "        model.zero_grad() #Zero gradients \n",
        "\n",
        "        sentence_in = prepare_sequence(sent.split()).to(device)\n",
        "    \n",
        "        targets = torch.tensor([4] + [tag_to_ix[t] for t in task2_label[i][j].split()] + [4], dtype=torch.long).to(device)\n",
        "        \n",
        "        loss = model.neg_log_likelihood(sentence_in, targets)\n",
        "\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "\n",
        "    end = time.time() #Ending time measured\n",
        "\n",
        "    #Saving the Model, note that it will be very heavy to store(approx 400 Mb)  \n",
        "    if epoch%1 == 0: \n",
        "      torch.save(model,\"/content/mod\" + str(epoch) + \".pt\")\n",
        "\n",
        "    print(\"The epoch completed is\",epoch, \"and time\", end-start)\n",
        "\n",
        "    #Validation Loss computation\n",
        "    if epoch%1 ==0:\n",
        "      model.eval() #In evaluation mode\n",
        "      training_eval() #Training metrics for every epoch\n",
        "      validation_eval() #Validation metrics for every epoch\n",
        "      val_total_loss = 0.\n",
        "      with torch.no_grad():\n",
        "        for k, val_file in enumerate(val_task2_in):\n",
        "          for l, val_sent in enumerate(val_file):\n",
        "            val_sentence_in = prepare_sequence(val_sent.split()).to(device)\n",
        "            val_targets = torch.tensor([4] + [tag_to_ix[t] for t in val_task2_label[k][l].split()] + [4], dtype=torch.long).to(device)\n",
        "            loss = model.neg_log_likelihood(val_sentence_in, val_targets)\n",
        "            val_total_loss += loss.item()\n",
        "      print(\"validation loss after epoch\",epoch,\" is\", val_total_loss)\n",
        "      \n",
        "\n",
        "#Simple check\n",
        "with torch.no_grad():\n",
        "    precheck_sent = prepare_sequence(task2_in[0][0].split()).to(device)\n",
        "    print(model(precheck_sent))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZqRglUUEsYaI"
      },
      "source": [
        "#### Load the saved model and put it into evaluation mode \n",
        "model = torch.load(\"/content/mod0.pt\")\n",
        "model.eval()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPkKBpGYhhIr"
      },
      "source": [
        "#### Test dataset reading\n",
        "\n",
        "#Point it to the directory where your Test Data is present\n",
        "test_input_dir = \"/content/drive/MyDrive/sub1_ph2/\"\n",
        "\n",
        "test_list_of_folders = [\"constituency_parsing\",\"coreference_resolution\",\n",
        "                   \"data-to-text_generation\",\"dependency_parsing\",\n",
        "                   \"document_classification\",\"entity_linking\",\n",
        "                   \"face_alignment\",\"face_detection\", \"hypernym_discovery\",\n",
        "                   \"natural_language_inference\"]\n",
        "\n",
        "test_input_stanza_list = [] #Stores individual lines from Stanza_out.txt file\n",
        "test_input_sent_num_list = [] #Stores individual lines from sentences.txt file\n",
        "test_input_entity_list = [] #Stores the list of all phrases from entities.txt\n",
        "test_file_name_list = [] #Stores the name of individual files\n",
        "test_total_phrases_truth = 0\n",
        "Capital_test_input_stanza_list = [] #Stores individual lines from Stanza_out.txt file in its original case(not lowered case)\n",
        "\n",
        "for fls in test_list_of_folders:\n",
        "  count=0\n",
        "  for i in os.listdir(test_input_dir + fls + '/'):\n",
        "    count=count+1\n",
        "    for files in os.listdir(test_input_dir + fls + '/' + str(i)):\n",
        "      if files.endswith(\"Stanza-out.txt\"):\n",
        "        stanza_file = open(test_input_dir + fls + '/' + str(i) + '/' + files, \"r\")\n",
        "        print(test_input_dir + fls + '/' + str(i))\n",
        "        Capital_stanza_lines = stanza_file.read()\n",
        "        Capital_stanza_lines_list = list(filter(None,Capital_stanza_lines.splitlines())) # filter empty strings and split into lines\n",
        "        Capital_test_input_stanza_list.append(Capital_stanza_lines_list)\n",
        "\n",
        "        stanza_lines = Capital_stanza_lines.lower()\n",
        "        stanza_lines_list = list(filter(None,stanza_lines.splitlines())) # filter empty strings and split into lines\n",
        "        test_input_stanza_list.append(stanza_lines_list)\n",
        "      if files.endswith(\"sentences.txt\"):\n",
        "        sentence_file = open(test_input_dir + fls + '/' + str(i) + '/' + 'sentences.txt', \"r\")\n",
        "        sentence_num_list = list(filter(None,(sentence_file.read().lower()).splitlines())) # filter empty strings and split into lines\n",
        "        test_input_sent_num_list.append(list(map(int, sentence_num_list)))\n",
        "     \n",
        "    test_file_name_list.append(fls + '/' + str(i))\n",
        "  print(\"completed\",fls,count)\n",
        "\n",
        "#Check test data\n",
        "print(\"Test no. of examples for each stanza,sentences and entities\")\n",
        "print(len(test_input_sent_num_list),len(test_file_name_list))\n",
        "\n",
        "print(\"print one example to show all files\")\n",
        "print(len(test_input_sent_num_list[0]),test_file_name_list[0])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D_9-OxSsrxpD"
      },
      "source": [
        "#### Store the contribution sentences in Lower Case and in their Original Case \n",
        "import copy\n",
        "test_task2_in = [] #Stores the contribution sentences\n",
        "Capital_test_task2_in = [] #Stores the contribution sentences in Original Case\n",
        "\n",
        "for i in range(len(test_input_sent_num_list)): #Process each file for labels\n",
        "  \n",
        "  test_sent_num_list = copy.deepcopy(test_input_sent_num_list[i]) #Copy of the sentences list\n",
        "  test_sent_num_list.sort() #Sorting the sentences list\n",
        "  test_sent_list = []  #List containing sentence strings\n",
        "  Capital_test_sent_list = []\n",
        " \n",
        "  for x in test_sent_num_list: \n",
        "    test_sent_list.append(test_input_stanza_list[i][x-1])\n",
        "    Capital_test_sent_list.append(Capital_test_input_stanza_list[i][x-1])\n",
        "  \n",
        "  test_task2_in.append(test_sent_list)\n",
        "  Capital_test_task2_in.append(Capital_test_sent_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YrD-Kcr5LV0S"
      },
      "source": [
        "#### This is where we make a dictionary that can map sentence to its corresponding number in Stanza_out.txt \n",
        "\n",
        "list_of_dict_for_number_to_sentence = []\n",
        "for i in range(len(test_input_stanza_list)): #Process each file for labels\n",
        " \n",
        "  test_sent_num_list = copy.deepcopy(test_input_sent_num_list[i]) #Copy of the sentences list\n",
        "  test_sent_num_list.sort() #Sorting the sentences list\n",
        "  test_sent_list = [] #List containg sentence strings\n",
        "\n",
        "  for x in test_sent_num_list: \n",
        "    test_sent_list.append(test_input_stanza_list[i][x-1])\n",
        "  test_sent_dict_list = dict(zip(test_sent_num_list,test_sent_list)) #Dictionary of sentence number and strings\n",
        "  list_of_dict_for_number_to_sentence.append(test_sent_dict_list)  \n",
        "\n",
        "#This is the mapper list of dictionaries\n",
        "list_of_dict_for_sentence_to_number = [dict((v,k) for k,v in a.items()) for a in list_of_dict_for_number_to_sentence]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eOyoMdEQiWKP"
      },
      "source": [
        "#### Test label Prediction Function\n",
        "\n",
        "test_task2_out_label = [] #Stores the Labels predicted\n",
        "\n",
        "for i,file in enumerate(test_task2_in):\n",
        "  output = []\n",
        "  for j,sent in enumerate(file):\n",
        "    with torch.no_grad():\n",
        "      precheck_sent = prepare_sequence(test_task2_in[i][j].split()).to(device)\n",
        "      sent_out_label = model(precheck_sent)[1]\n",
        "      sent_str_label = [ix_to_tag[t] for t in sent_out_label]\n",
        "      output.append(sent_str_label)\n",
        "  test_task2_out_label.append(output)\n",
        "print(len(test_task2_out_label))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ck4eeCV5NpxM"
      },
      "source": [
        "#### For reference\n",
        "print(len(test_task2_in))\n",
        "print(len(test_task2_in[0]))\n",
        "print(test_task2_in[0])\n",
        "print()\n",
        "print(len(Capital_test_task2_in))\n",
        "print(len(Capital_test_task2_in[0]))\n",
        "print(Capital_test_task2_in[0])\n",
        "print()\n",
        "print(test_file_name_list[0])\n",
        "print()\n",
        "print(len(test_task2_out_label))\n",
        "print(len(test_task2_out_label[0]))\n",
        "print((test_task2_out_label[0]))\n",
        "print()\n",
        "print((test_task2_in[0][2]))\n",
        "print(len(test_task2_out_label[0][2]))\n",
        "print(len(test_task2_in[0][2].split()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCNP8QFpMFJ0"
      },
      "source": [
        "#### Here we convert the predicted labels into phrases and write them into entities.txt file along with its starting and ending character number\n",
        "\n",
        "for i,file in enumerate(test_task2_in):\n",
        "  \n",
        "  print(test_file_name_list[i]) #Print the current file name \n",
        "  \n",
        "  f1 = open(test_input_dir + test_file_name_list[i] + \"/entities.txt\", \"w\") #Open it\n",
        "\n",
        "  for j,sent in enumerate(file):\n",
        "\n",
        "    biluo_list = (test_task2_out_label[i][j])[1:-1] #The start and end labels should be ignored as they correspond to CLS and SEP token respectively.\n",
        "    respective_sentence = Capital_test_task2_in[i][j].split()\n",
        "    sentence_number = (list_of_dict_for_sentence_to_number[i])[test_task2_in[i][j]]\n",
        "\n",
        "    if(len(respective_sentence) != len(biluo_list)):\n",
        "      print(\"Length mismatch in the sentence and BILUO sequence\")\n",
        "      continue\n",
        "\n",
        "    temp_phrase_storer = []\n",
        "    temp_phrase = []\n",
        "    count_of_words_in_sentence = 0\n",
        "\n",
        "    for k in zip(biluo_list,respective_sentence):\n",
        "\n",
        "      if (k[0]==\"U\"): #That means its a single phrase\n",
        "        temp_phrase_storer = temp_phrase_storer + [k[1]]\n",
        "\n",
        "        start_of_word = 0\n",
        "        if(count_of_words_in_sentence == 0):\n",
        "          start_of_word = 0\n",
        "        else:  \n",
        "          start_of_word = len((\" \".join(respective_sentence[0:count_of_words_in_sentence])).strip() + \" \") \n",
        "\n",
        "        end_of_word = start_of_word + len(k[1].strip())\n",
        "\n",
        "        f1.write(str(sentence_number) + \"\\t\" +  str(start_of_word) + \"\\t\" + str(end_of_word) + \"\\t\" + k[1].strip() + \"\\n\")\n",
        "\n",
        "      elif (k[0]==\"B\"):\n",
        "        temp_phrase = temp_phrase + [k[1]] \n",
        "\n",
        "      elif (k[0]==\"I\"):\n",
        "        temp_phrase = temp_phrase + [\" \", k[1]]\n",
        "\n",
        "      elif (k[0]==\"L\"): #We have reached the end of phrase, so store it now\n",
        "        temp_phrase = temp_phrase + [\" \", k[1]]\n",
        "\n",
        "        end_of_words = len((\" \".join(respective_sentence[0:count_of_words_in_sentence])).strip() + \" \") + len(respective_sentence[count_of_words_in_sentence].strip())\n",
        "        start_of_words = end_of_words - len((\"\".join(temp_phrase)).strip())\n",
        "\n",
        "        f1.write(str(sentence_number) + \"\\t\" + str(start_of_words) + \"\\t\" + str(end_of_words) + \"\\t\" + (\"\".join(temp_phrase)).strip() + \"\\n\")\n",
        "        temp_phrase_storer =  temp_phrase_storer + copy.deepcopy([\"\".join(temp_phrase)])\n",
        "        temp_phrase = []\n",
        "\n",
        "      count_of_words_in_sentence += 1 \n",
        "\n",
        "  f1.close()  \n",
        "  print(\"done\")"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}