project1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import csv\n",
    "import re\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "from transformers import BertTokenizer\n",
    "import pandas as pd\n",
    "from torch.utils.data import DataLoader\n",
    "import torch.nn as nn\n",
    "from transformers import BertModel\n",
    "import torch.optim as optim\n",
    "import time\n",
    "import operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_file = r\"data\\train.data.jsonl\"\n",
    "train_label_file = r\"data\\train.label.json\"\n",
    "train_tsv = r\".\\train.tsv\"\n",
    "\n",
    "dev_file = r\"data\\dev.data.jsonl\"\n",
    "dev_label_file = r\"data\\dev.label.json\"\n",
    "dev_tsv = r\".\\dev.tsv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Transfer to TSV File Section\n",
    "#### If transferred, this section is not required to run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract original tweet and reply do not do any processing\n",
    "def transfer_to_tsv(tweet_file, label_file, transferred_file):\n",
    "    contents = []\n",
    "    labels = []\n",
    "    classes = [\"non-rumour\", \"rumour\"]\n",
    "    \n",
    "    # get labels\n",
    "    with open(label_file) as lable_obj:\n",
    "        unprocessed_labels = json.load(lable_obj)\n",
    "    \n",
    "    #get tweets \n",
    "    with open(tweet_file, 'r') as tweet_obj:\n",
    "        json_list = list(tweet_obj)\n",
    "    \n",
    "    # extract the content we want and append to list\n",
    "    for json_str in json_list:\n",
    "        # get one event\n",
    "        event = json.loads(json_str)\n",
    "        \n",
    "        # concat. original tweet and replies\n",
    "        # Insering the CLS and SEP token in the beginning of original tweet and end of the each tweet and reply\n",
    "        content = \"\"\n",
    "        for i in range(len(event)):\n",
    "            if event[i][\"text\"][-1] == \".\":\n",
    "                content = content + event[i][\"text\"] + ' '\n",
    "            else:\n",
    "                content = content + event[i][\"text\"] + '. '\n",
    "                \n",
    "        # remove @, and hashtag\n",
    "        content = re.sub(r'(https|http)?:\\/\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b', '', content, flags=re.MULTILINE)\n",
    "        content = re.sub(r'@(\\w+)?', '', content, flags=re.MULTILINE)\n",
    "        content = re.sub(r'#(\\w+)?', '', content, flags=re.MULTILINE)\n",
    "        \n",
    "        contents.append(content.replace(\"\\n\", \" \").replace(\"\\r\", \" \"))\n",
    "        labels.append(classes.index(unprocessed_labels[event[0]['id_str']]))\n",
    "        \n",
    "    # write into tsv\n",
    "    with open(transferred_file, 'w', encoding=\"utf-8\") as f:\n",
    "        tsv_w = csv.writer(f, delimiter='\\t', lineterminator='\\n')\n",
    "        tsv_w.writerow(['content', 'label'])\n",
    "        for num in range(len(contents)):\n",
    "            tsv_w.writerow([contents[num], labels[num]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transfer jsonl and json to tsv file\n",
    "transfer_to_tsv(train_file, train_label_file, train_tsv)\n",
    "transfer_to_tsv(dev_file, dev_label_file, dev_tsv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transferring TSV to Training Form Section\n",
    "#### If just for prediction please go to Building Model Section."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainDataset(Dataset):\n",
    "\n",
    "    def __init__(self, filename, input_size):\n",
    "\n",
    "        #Store the contents of the file in a pandas dataframe\n",
    "        self.df = pd.read_csv(filename, delimiter = '\\t')\n",
    "\n",
    "        #Initialize the BERT tokenizer\n",
    "        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "        \n",
    "        # the input length for BERT model. Max length is 512\n",
    "        self.input_size = 0\n",
    "        if input_size > 512:\n",
    "            self.input_size = 512\n",
    "        else:\n",
    "            self.input_size = input_size\n",
    "            \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "\n",
    "        #Selecting the content and label at the specified index in the data frame\n",
    "        tweet = self.df.loc[index, 'content']\n",
    "        label = self.df.loc[index, 'label']\n",
    "        \n",
    "        # Tokenize the tweet and insering the CLS and SEP\n",
    "        tokens = ['[CLS]'] + self.tokenizer.tokenize(tweet) + ['[SEP]'] \n",
    "        \n",
    "        if len(tokens) < self.input_size:\n",
    "            #Padding token\n",
    "            tokens = tokens + ['[PAD]' for _ in range(self.input_size - len(tokens))] \n",
    "        else:\n",
    "            # if tokens length > input_size, extract the first input_size-1 and add SEP\n",
    "            tokens = tokens[:self.input_size-1] + ['[SEP]'] \n",
    "        \n",
    "        #Converting the token to a pytorch ID tensor\n",
    "        tokens_ids_tensor = torch.tensor(self.tokenizer.convert_tokens_to_ids(tokens)) \n",
    "\n",
    "        #Obtaining the attention mask\n",
    "        attn_mask = (tokens_ids_tensor != 0).long()\n",
    "        \n",
    "        return tokens_ids_tensor, attn_mask, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating instances of training and development set\n",
    "train_set = TrainDataset(filename = train_tsv, input_size = 512)\n",
    "dev_set = TrainDataset(filename = dev_tsv, input_size = 512)\n",
    "\n",
    "#Creating intsances of training and development dataloaders\n",
    "#batch_size = 8 for 512 input size (GTX-1080Ti 11G)\n",
    "#batch_size = 4 for 512 input size (RTX-2080 8G)\n",
    "train_loader = DataLoader(train_set, batch_size = 4, num_workers = 0)\n",
    "dev_loader = DataLoader(dev_set, batch_size = 4, num_workers = 0)\n",
    "\n",
    "print(\"Done preprocessing training and development data.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Building Model Section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RumorClassifier(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(RumorClassifier, self).__init__()\n",
    "        #Instantiating BERT model object \n",
    "        self.bert_layer = BertModel.from_pretrained('bert-base-uncased', return_dict=False)\n",
    "        \n",
    "        #Linear model. Since the dimension of contextual representation is 768, the input size of linear model is 768\n",
    "        self.cls_layer = nn.Linear(768, 1)\n",
    "\n",
    "    def forward(self, seq, attn_masks):\n",
    "        #Feeding the input to BERT model to obtain contextualized representations\n",
    "        cont_reps, _ = self.bert_layer(seq, attention_mask = attn_masks)\n",
    "\n",
    "        #Obtaining the representation of [CLS] head (the first token)\n",
    "        cls_rep = cont_reps[:, 0]\n",
    "\n",
    "        #Feeding cls_rep to the classifier layer\n",
    "        logits = self.cls_layer(cls_rep)\n",
    "\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu = 0 #gpu ID\n",
    "\n",
    "print(\"Creating the Rumor classifier, initialised with pretrained BERT-BASE parameters...\")\n",
    "net = RumorClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.cuda(gpu) #Enable gpu support for the model\n",
    "print(\"Done creating the Rumor classifier.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.BCEWithLogitsLoss()\n",
    "opti = optim.Adam(net.parameters(), lr = 2e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_accuracy_from_logits(logits, labels):\n",
    "    probs = torch.sigmoid(logits.unsqueeze(-1))\n",
    "    soft_probs = (probs > 0.5).long()\n",
    "    acc = (soft_probs.squeeze() == labels).float().mean()\n",
    "    return acc\n",
    "\n",
    "def evaluate(net, criterion, dataloader, gpu):\n",
    "    net.eval()\n",
    "\n",
    "    mean_acc, mean_loss = 0, 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for seq, attn_masks, labels in dev_loader:\n",
    "            seq, attn_masks, labels = seq.cuda(gpu), attn_masks.cuda(gpu), labels.cuda(gpu)\n",
    "            logits = net(seq, attn_masks)\n",
    "            mean_loss += criterion(logits.squeeze(-1), labels.float()).item()\n",
    "            mean_acc += get_accuracy_from_logits(logits, labels)\n",
    "    \n",
    "    return mean_acc / len(dev_loader), mean_loss / len(dev_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training Section\n",
    "#### If just for prediction please go to Testing Section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(net, criterion, opti, train_loader, dev_loader, max_eps, gpu):\n",
    "\n",
    "    best_acc = 0\n",
    "    st = time.time()\n",
    "    for ep in range(max_eps):\n",
    "        \n",
    "        for it, (seq, attn_masks, labels) in enumerate(train_loader):\n",
    "            #Clear gradients\n",
    "            opti.zero_grad()  \n",
    "            #Converting these to cuda tensors\n",
    "            seq, attn_masks, labels = seq.cuda(gpu), attn_masks.cuda(gpu), labels.cuda(gpu)\n",
    "\n",
    "            #Obtaining the logits from the model\n",
    "            logits = net(seq, attn_masks)\n",
    "\n",
    "            #Computing loss\n",
    "            loss = criterion(logits.squeeze(-1), labels.float())\n",
    "\n",
    "            #Backpropagating the gradients\n",
    "            loss.backward()\n",
    "\n",
    "            #Optimization step\n",
    "            opti.step()\n",
    "              \n",
    "            if it % 100 == 0:\n",
    "                \n",
    "                acc = get_accuracy_from_logits(logits, labels)\n",
    "                print(\"Iteration {} of epoch {} complete. Loss: {}; Accuracy: {}; Time taken (s): {}\".format(it, ep, loss.item(), acc, (time.time()-st)))\n",
    "                st = time.time()\n",
    "\n",
    "        # evaluate for each epoch\n",
    "        dev_acc, dev_loss = evaluate(net, criterion, dev_loader, gpu)\n",
    "        print(\"Epoch {} complete! Development Accuracy: {}; Development Loss: {}\".format(ep, dev_acc, dev_loss))\n",
    "        if dev_acc > best_acc:\n",
    "            print(\"Best development accuracy improved from {} to {}, saving model...\".format(best_acc, dev_acc))\n",
    "            best_acc = dev_acc\n",
    "            torch.save(net.state_dict(), 'sstcls_{}.dat'.format(ep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epoch = 16\n",
    "\n",
    "#fine-tune the model\n",
    "train(net, criterion, opti, train_loader, dev_loader, num_epoch, gpu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing Section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_to_tsv(tweet_file, transferred_file):\n",
    "    contents = []\n",
    "    tweet_ID = []\n",
    "    \n",
    "    #get tweets \n",
    "    with open(tweet_file, 'r') as tweet_obj:\n",
    "        json_list = list(tweet_obj)\n",
    "    \n",
    "    # extract the content we want and append to list\n",
    "    for json_str in json_list:\n",
    "        # get one event\n",
    "        event = json.loads(json_str)\n",
    "        \n",
    "        # concat. original tweet and replies\n",
    "        # Insering the CLS and SEP token in the beginning of original tweet and end of the each tweet and reply\n",
    "        content = \"\"\n",
    "        for i in range(len(event)):\n",
    "            if event[i][\"text\"][-1] == \".\" or event[i][\"text\"][-1] == \"?\" or event[i][\"text\"][-1] == \"!\":\n",
    "                content = content + event[i][\"text\"] + ' '\n",
    "            else:\n",
    "                content = content + event[i][\"text\"] + '. '\n",
    "        \n",
    "        content = re.sub(r'(https|http)?:\\/\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b', '', content, flags=re.MULTILINE)\n",
    "        content = re.sub(r'@(\\w+)?', '', content, flags=re.MULTILINE)\n",
    "        content = re.sub(r'#(\\w+)?', '', content, flags=re.MULTILINE)\n",
    "        \n",
    "        contents.append(content.replace(\"\\n\", \" \").replace(\"\\r\", \" \"))\n",
    "        tweet_ID.append(event[0]['id_str'])\n",
    "        \n",
    "    # write into tsv\n",
    "    with open(transferred_file, 'w', encoding=\"utf-8\") as f:\n",
    "        tsv_w = csv.writer(f, lineterminator='\\n')\n",
    "        tsv_w.writerow(['content'])\n",
    "        for num in range(len(contents)):\n",
    "            tsv_w.writerow([contents[num].replace(\"\\n\", \" \")])\n",
    "            \n",
    "    return tweet_ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file = r\"data\\test.data.jsonl\"\n",
    "test_tsv = r\".\\test.tsv\"\n",
    "dev_for_evaluate =  r\".\\dev_result.tsv\"\n",
    "\n",
    "# test_order used to construct dict. with prediction\n",
    "#test_order = test_to_tsv(test_file, test_tsv)\n",
    "\n",
    "# produce json prediction for dev in order to apply eval.py\n",
    "test_order = test_to_tsv(dev_file, dev_for_evaluate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestDataset(Dataset):\n",
    "\n",
    "    def __init__(self, filename, input_size):\n",
    "\n",
    "        #Store the contents of the file in a pandas dataframe\n",
    "        self.df = pd.read_csv(filename, delimiter = '\\t')\n",
    "\n",
    "        #Initialize the BERT tokenizer\n",
    "        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "        \n",
    "        # the input length for BERT model. Max length is 512\n",
    "        self.input_size = 0\n",
    "        if input_size > 512:\n",
    "            self.input_size = 512\n",
    "        else:\n",
    "            self.input_size = input_size\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "\n",
    "        #Selecting the content and label at the specified index in the data frame\n",
    "        tweet = self.df.loc[index, 'content']\n",
    "        \n",
    "        # Tokenize the tweet and insering the CLS and SEP\n",
    "        tokens = ['[CLS]'] + self.tokenizer.tokenize(tweet) + ['[SEP]'] \n",
    "        \n",
    "        if len(tokens) < self.input_size:\n",
    "            #Padding token\n",
    "            tokens = tokens + ['[PAD]' for _ in range(self.input_size - len(tokens))] \n",
    "        else:\n",
    "            # if tokens length > input_size, extract the first input_size-1 and add SEP\n",
    "            tokens = tokens[:self.input_size-1] + ['[SEP]'] \n",
    "        \n",
    "        #Converting the token to a pytorch ID tensor\n",
    "        tokens_ids_tensor = torch.tensor(self.tokenizer.convert_tokens_to_ids(tokens)) \n",
    "\n",
    "        #Obtaining the attention mask\n",
    "        attn_mask = (tokens_ids_tensor != 0).long()\n",
    "        \n",
    "        return tokens_ids_tensor, attn_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test for dev set\n",
    "test_set = TestDataset(filename = dev_for_evaluate, input_size = 512)\n",
    "\n",
    "# test for testing set\n",
    "#test_set = TestDataset(filename = test_tsv, maxlen = 512)\n",
    "\n",
    "test_loader = DataLoader(test_set, batch_size = 1, num_workers = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(net, test_order, test_loader, weight_file, predict_file):\n",
    "    # load weight\n",
    "    net.load_state_dict(torch.load(weight_file))\n",
    "    \n",
    "    predictions = []\n",
    "    classes = [\"non-rumour\", \"rumour\"]\n",
    "    \n",
    "    # Predict process\n",
    "    with torch.no_grad():\n",
    "        for seq, attn_masks in test_loader:\n",
    "            seq, attn_masks = seq.cuda(gpu), attn_masks.cuda(gpu)\n",
    "            logits = net(seq, attn_masks)\n",
    "            probs = torch.sigmoid(logits.unsqueeze(-1))\n",
    "            soft_probs = (probs > 0.5).long()\n",
    "            predictions.append(classes[soft_probs.cpu().numpy().squeeze()])\n",
    "    \n",
    "    # Write into json file\n",
    "    dictionary = dict(zip(test_order, predictions))\n",
    "    \n",
    "    with open(predict_file, 'w') as result_file:\n",
    "        json.dump(dictionary, result_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_file = r\".\\OR_remove@URL#.dat\"\n",
    "predict_file = \"test-output.json\"\n",
    "prediction = predict(net, test_order, test_loader, weight_file, predict_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Other Processing Method for Convert to tsv function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = re.sub(r'(https|http)?:\\/\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b', '', content, flags=re.MULTILINE) # remove url\n",
    "content = re.sub(r'@(\\w+)?', '', content, flags=re.MULTILINE) # remove @username\n",
    "content = re.sub(r'#(\\w+)?', '', content, flags=re.MULTILINE) # remove hashtag"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Concatenet source and reply tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = \"\"\n",
    "for i in range(len(event)):\n",
    "    if event[i][\"text\"][-1] == \".\" or event[i][\"text\"][-1] == \"?\" or event[i][\"text\"][-1] == \"!\":\n",
    "        content = content + event[i][\"text\"] + ' '\n",
    "    else:\n",
    "        content = content + event[i][\"text\"] + '. '"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Concatenet source and last 10 reply tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "content = \"\"\n",
    "# get the original tweet first\n",
    "if event[0][\"text\"][-1] == \".\" or event[0][\"text\"][-1] == \"?\" or event[0][\"text\"][-1] == \"!\":\n",
    "    content = content + event[0][\"text\"] + ' '\n",
    "else:\n",
    "    content = content + event[0][\"text\"] + '. '\n",
    "\n",
    "#get the last 10 replies\n",
    "if len(event) > 10: \n",
    "# process tweet create time in order to sort tweet by time\n",
    "    for e in event:\n",
    "        e[\"processed_time\"] = time.strftime('%Y-%m-%d %H:%M:%S',time.strptime(e['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))\n",
    "    event.sort(key = operator.itemgetter(\"processed_time\"))\n",
    "\n",
    "    for e in event[-10:]:\n",
    "        if e[\"text\"][-1] == \".\" or e[\"text\"][-1] == \"?\" or e[\"text\"][-1] == \"!\":\n",
    "            content = content + e[\"text\"] + ' '\n",
    "        else:\n",
    "            content = content + e[\"text\"] + '. '\n",
    "else:\n",
    "    # get all replies if there are no enough replies\n",
    "    for i in range(1, len(event)):\n",
    "        if event[i][\"text\"][-1] == \".\" or event[i][\"text\"][-1] == \"?\" or event[i][\"text\"][-1] == \"!\":\n",
    "            content = content + event[i][\"text\"] + ' '\n",
    "        else:\n",
    "            content = content + event[i][\"text\"] + '. '\n",
    "        content = content + event[i][\"text\"] + '. '"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Other Processing Method for Convert to tensor function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### remove token which do not contain alphabet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_tokens = []\n",
    "for token in tokens:\n",
    "    if (re.search('[a-zA-Z.?!]', token)):\n",
    "        new_tokens.append(token)\n",
    "\n",
    "new_tokens = ['[CLS]'] + new_tokens + ['[SEP]']\n",
    "if len(new_tokens) < self.maxlen:\n",
    "    new_tokens = new_tokens + ['[PAD]' for _ in range(self.maxlen - len(new_tokens))] #Padding token\n",
    "else:\n",
    "    new_tokens = new_tokens[:self.maxlen-1] + ['[SEP]'] #Prunning the list to be of specified max length\n",
    "\n",
    "tokens_ids = self.tokenizer.convert_tokens_to_ids(new_tokens) #Obtaining the indices of the tokens in the BERT Vocabulary"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}