pos_bert_multi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8_rzsBs_Egpn"
   },
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "V-9Pw5AwwHDH"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tmI1bkMsDJHZ"
   },
   "outputs": [],
   "source": [
    "pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fCxwmP3eGlpu"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import pickle\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report \n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data.dataloader import DataLoader, Sampler\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "import transformers\n",
    "from transformers import BertTokenizer, BertModel\n",
    "\n",
    "\n",
    "train_data_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/data/bijankhan_corpus.tsv'\n",
    "pickled_data_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/data/bert_data_multi.pickle'\n",
    "model_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/bert_model_multi/model.pickle'\n",
    "last_i_dir = '/content/drive/My Drive/Colab Notebooks/ezafe/bert_model_multi/last_i'\n",
    "\n",
    "hidden_size = 768\n",
    "num_epochs = 100\n",
    "batch_size = 8\n",
    "learning_rate = 2e-5\n",
    "dropout_rate = .1\n",
    "tag_mask_and_pad = 50\n",
    "\n",
    "random.seed(17)\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print('device: ', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Czh-4NV6gzJI"
   },
   "outputs": [],
   "source": [
    "def data_reader(directory):\n",
    "    sents, sent = [], [101]\n",
    "    all_ezafe_tags, ezafe_tags = [], [tag_mask_and_pad]\n",
    "    all_pos_tags, pos_tags = [], [tag_mask_and_pad]\n",
    "    pos_to_id = {}\n",
    "    with open(directory) as bijankhan_corpus:\n",
    "        for line in bijankhan_corpus:\n",
    "            if line != '\\n':\n",
    "                word, pos_tag, ezafe_tag = line.strip().split('\\t')\n",
    "\n",
    "                if pos_tag not in pos_to_id:\n",
    "                    pos_to_id[pos_tag] = len(pos_to_id)\n",
    "\n",
    "                word = word.replace('ي', 'ی').replace('ك', 'ک').replace('ة', 'ه')\n",
    "                word = tokenizer.encode(word, add_special_tokens=False)\n",
    "                sent.extend(word)\n",
    "\n",
    "                ezafe_tags.extend([int(ezafe_tag)] + [tag_mask_and_pad for _ in range(len(word) - 1)])\n",
    "                pos_tags.extend([pos_to_id[pos_tag]] + [tag_mask_and_pad for _ in range(len(word) - 1)])\n",
    "            else:\n",
    "                sents.append(sent + [102])\n",
    "                all_ezafe_tags.append(ezafe_tags + [tag_mask_and_pad])\n",
    "                all_pos_tags.append(pos_tags + [tag_mask_and_pad])\n",
    "                \n",
    "                sent = [101]\n",
    "                ezafe_tags = [tag_mask_and_pad]\n",
    "                pos_tags = [tag_mask_and_pad]\n",
    "\n",
    "    return sents, all_ezafe_tags, all_pos_tags\n",
    "\n",
    "\n",
    "class MySampler(Sampler):\n",
    "    def __init__(self, data, i=0):\n",
    "        random.shuffle(data)\n",
    "        self.seq = list(range(len(data)))[i * batch_size:]\n",
    "\n",
    "    def __iter__(self):\n",
    "        return iter(self.seq)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.seq)\n",
    "\n",
    "\n",
    "class EzafeDataset(Dataset):\n",
    "    def __init__(self, data):\n",
    "       self.samples = data\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.samples)\n",
    "  \n",
    "    def __getitem__(self, idx):\n",
    "        sample = self.samples[idx]\n",
    "        x = torch.tensor(sample[0])\n",
    "        mask = torch.sign(x)\n",
    "        ezafe_tags = torch.tensor(sample[1])\n",
    "        pos_tags = torch.tensor(sample[2])\n",
    "\n",
    "        return x, mask, ezafe_tags, pos_tags\n",
    "\n",
    "\n",
    "def collate_fn(batch):\n",
    "    xs, masks, ezafe_tags, pos_tags = zip(*batch)\n",
    "    padded_xs = pad_sequence(xs, batch_first=True)\n",
    "    padded_masks = pad_sequence(masks, batch_first=True)\n",
    "    padded_ezafe_tags = pad_sequence(ezafe_tags, batch_first=True, padding_value=tag_mask_and_pad)\n",
    "    padded_pos_tags = pad_sequence(pos_tags, batch_first=True, padding_value=tag_mask_and_pad)\n",
    "\n",
    "    return {'input_ids': padded_xs, 'attention_masks': padded_masks}, \\\n",
    "           (padded_ezafe_tags, padded_pos_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZokGlh4adnhp"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    tokenizer = BertTokenizer.from_pretrained(model_dir)\n",
    "    print('tokenizer is loaded')\n",
    "except:\n",
    "    tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')\n",
    "\n",
    "try:\n",
    "    with open(pickled_data_dir, 'rb') as out_file:\n",
    "        sents, ezafe_tags, pos_tags = pickle.load(out_file)\n",
    "except FileNotFoundError:\n",
    "    sents, ezafe_tags, pos_tags = data_reader(train_data_dir) \n",
    "    with open(pickled_data_dir, 'wb') as out_file:\n",
    "        pickle.dump([sents, ezafe_tags, pos_tags], out_file)\n",
    "\n",
    "sents_shuf = []\n",
    "ezafe_tags_shuf = []\n",
    "pos_tags_shuf = []\n",
    "index_shuf = list(range(len(sents)))\n",
    "\n",
    "random.seed(17)\n",
    "random.shuffle(index_shuf)\n",
    "\n",
    "for i in index_shuf:\n",
    "    sents_shuf.append(sents[i])\n",
    "    ezafe_tags_shuf.append(ezafe_tags[i])\n",
    "    pos_tags_shuf.append(pos_tags[i])\n",
    "\n",
    "sents_shuf, ezafe_tags_shuf, pos_tags_shuf = zip(*[(sent, ezafe, pos) for sent, ezafe, pos in zip(sents_shuf, ezafe_tags_shuf, pos_tags_shuf) if len(sent) <= 512])\n",
    "\n",
    "data_split_1 = int(len(sents_shuf) * .1)\n",
    "data_split_2 = int(len(sents_shuf) * .2)\n",
    "\n",
    "test_data = [(sents_shuf[i], ezafe_tags_shuf[i], pos_tags_shuf[i]) for i in range(len(sents_shuf))[:data_split_1]]\n",
    "valid_data = [(sents_shuf[i], ezafe_tags_shuf[i], pos_tags_shuf[i]) for i in range(len(sents_shuf))[data_split_1:data_split_2]]\n",
    "train_data = [(sents_shuf[i], ezafe_tags_shuf[i], pos_tags_shuf[i]) for i in range(len(sents_shuf))[data_split_2:]]\n",
    " \n",
    "print(train_data[3])\n",
    "\n",
    "prev_epochs = 0\n",
    "try:\n",
    "    with open(last_i_dir) as in_file:\n",
    "        last_i = int(in_file.read().strip())\n",
    "    all_steps = math.ceil(len(train_data) / batch_size)\n",
    "    while last_i > all_steps:\n",
    "        last_i -= all_steps\n",
    "        prev_epochs += 1\n",
    "\n",
    "except FileNotFoundError:\n",
    "    last_i = 0\n",
    "\n",
    "print(last_i)\n",
    "    \n",
    "train_dataset = EzafeDataset(train_data)\n",
    "train_sampler = MySampler(train_dataset, last_i)\n",
    "\n",
    "test_dataset = EzafeDataset(test_data)\n",
    "\n",
    "print(len(train_dataset))\n",
    "print(len(test_dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EwFWvRPND67-"
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "\n",
    "        self.bert_model = bert_model\n",
    "        self.dropout = nn.Dropout(dropout_rate)\n",
    "        self.ezafe_classifier = nn.Linear(hidden_size, 2)\n",
    "        self.pos_classifier = nn.Linear(hidden_size, 15)\n",
    "\n",
    "    def forward(self, x, mask):\n",
    "        sequence_output, _ = self.bert_model(x, mask)\n",
    "        sequence_output = self.dropout(sequence_output)\n",
    "        ezafe_pred = self.ezafe_classifier(sequence_output)\n",
    "        pos_pred = self.pos_classifier(sequence_output)\n",
    "        return ezafe_pred, pos_pred\n",
    "\n",
    "\n",
    "try:\n",
    "    model = torch.load(model_dir)    \n",
    "except:\n",
    "    print('model wasn\\'t found')\n",
    "    bert_model = BertModel.from_pretrained('bert-base-multilingual-cased')\n",
    "    model = Net().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SrptV6bQC7_e"
   },
   "outputs": [],
   "source": [
    "train_data_loader = DataLoader(dataset=train_dataset,                                                         \n",
    "                               batch_size=batch_size,                                            \n",
    "                               collate_fn=collate_fn,\n",
    "                               sampler=train_sampler,\n",
    "                               shuffle=False,\n",
    "                               num_workers=4)\n",
    "\n",
    "test_data_loader = DataLoader(dataset=test_dataset,                                                         \n",
    "                              batch_size=batch_size,                                            \n",
    "                              collate_fn=collate_fn,\n",
    "                              shuffle=False,\n",
    "                              num_workers=4)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=tag_mask_and_pad).to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    epoch += prev_epochs\n",
    "    for i, (x, (ezafe_true, pos_true)) in enumerate(train_data_loader):\n",
    "        model.train()\n",
    "        i += last_i\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        ezafe_pred, pos_pred = model(x['input_ids'].to(device), x['attention_masks'].to(device))\n",
    "\n",
    "        ezafe_loss = criterion(ezafe_pred.view(-1, ezafe_pred.shape[-1]).cpu(), ezafe_true.view(-1))\n",
    "        pos_loss = criterion(pos_pred.view(-1, pos_pred.shape[-1]).cpu(), pos_true.view(-1))\n",
    "\n",
    "        loss = 1.0 * ezafe_loss + 2.0 * pos_loss\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if i % 2500 == 0 and (epoch != 0 or i >= 10_000):\n",
    "            print('testing...')\n",
    "            model.evaluate()\n",
    "            ezafe_true_labels, ezafe_pred_labels = [], []\n",
    "            pos_true_labels, pos_pred_labels = [], [] \n",
    "            with torch.no_grad():\n",
    "                for x, (ezafe_true, pos_true) in test_data_loader:\n",
    "                    ezafe_pred, pos_pred = model(x['input_ids'].to(device), x['attention_masks'].to(device))\n",
    "\n",
    "                    ezafe_true_labels.extend(ezafe_true.reshape(-1).numpy().tolist())\n",
    "                    ezafe_pred_labels.extend(torch.argmax(ezafe_pred, -1).reshape(-1).cpu().numpy().tolist())\n",
    "\n",
    "                    pos_true_labels.extend(pos_true.reshape(-1).numpy().tolist())\n",
    "                    pos_pred_labels.extend(torch.argmax(pos_pred, -1).reshape(-1).cpu().numpy().tolist())\n",
    "\n",
    "            ezafe_true_labels_, ezafe_pred_labels_ = np.array(ezafe_true_labels), np.array(ezafe_pred_labels)\n",
    "            pos_true_labels_, pos_pred_labels_ = np.array(pos_true_labels), np.array(pos_pred_labels)\n",
    " \n",
    "            ezafe_pos_mask_or_pad_ids = np.where(ezafe_true_labels_ == tag_mask_and_pad)[0]\n",
    "\n",
    "            ezafe_true_labels_ = np.delete(ezafe_true_labels_, ezafe_pos_mask_or_pad_ids)\n",
    "            ezafe_pred_labels_ = np.delete(ezafe_pred_labels_, ezafe_pos_mask_or_pad_ids)\n",
    "            pos_true_labels_ = np.delete(pos_true_labels_, ezafe_pos_mask_or_pad_ids)\n",
    "            pos_pred_labels_ = np.delete(pos_pred_labels_, ezafe_pos_mask_or_pad_ids)\n",
    "\n",
    "            ezafe_f1 = classification_report(ezafe_true_labels_, ezafe_pred_labels_, digits=4)\n",
    "            pos_f1 = classification_report(pos_true_labels_, pos_pred_labels_, digits=4)\n",
    "          \n",
    "            summary = f'epoch: {epoch + 1} | step: {i}:\\nEzafe_F1:\\n{ezafe_f1}\\n\\nPOS_F1:\\n{pos_f1}\\n\\n'\n",
    "            print(summary)\n",
    "\n",
    "            # save everything\n",
    "            with open(model_dir + 'summary.txt', 'a+') as out_file:\n",
    "                out_file.write(summary + '\\n')           \n",
    "    \n",
    "            with open(model_dir + 'last_i', 'w+') as out_file:\n",
    "                out_file.write(str(i))\n",
    " \n",
    "            os.mkdir(model_dir + str(i))\n",
    "            torch.save(model, model_dir + str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "St2CrwjdQ5rP"
   },
   "outputs": [],
   "source": [
    "# testing\n",
    "test_dataset = POSDataset(test_data)\n",
    "\n",
    "test_data_loader = DataLoader(dataset=test_dataset,                                                         \n",
    "                              batch_size=BATCH_SIZE,                                            \n",
    "                              collate_fn=collate_fn,\n",
    "                              shuffle=False,\n",
    "                              num_workers=4)\n",
    "\n",
    "i = 2500\n",
    "model = BertForTokenClassification.from_pretrained(model_dir + str(i))\n",
    "\n",
    "model.to(device)\n",
    "model.evaluate()\n",
    "test_true_labels, test_pred_labels = [], []\n",
    "with torch.no_grad():\n",
    "    for x, y_true in test_data_loader:\n",
    "        y_true = y_true.to(device)\n",
    "\n",
    "        _, y_pred = model(x['input_ids'].to(device), x['attention_masks'].to(device))[0]\n",
    "\n",
    "        test_true_labels.extend(y_true.reshape(-1).cpu().numpy().tolist())\n",
    "        test_pred_labels.extend(torch.argmax(y_pred, -1).reshape(-1).cpu().numpy().tolist())\n",
    "\n",
    "test_true_labels_, test_pred_labels_ = np.array(test_true_labels), np.array(test_pred_labels)\n",
    "test_pad_mask_ids = np.where(test_true_labels_ == TAG_PAD_OR_MASK_TOKEN)[0]\n",
    "test_true_labels_ = np.delete(test_true_labels_, test_pad_mask_ids)\n",
    "test_pred_labels_ = np.delete(test_pred_labels_, test_pad_mask_ids)\n",
    "\n",
    "test_f1 = classification_report(test_true_labels_, test_pred_labels_, digits=4)\n",
    "\n",
    "summary = f'epoch: {epoch + 1} | step: {i}:\\n\\n{test_f1}\\n'\n",
    "print(summary)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "bert_pos_multi.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}