Fine-Tune-LanguageModel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import sqrt\n",
    "import regex as re\n",
    "import os\n",
    "from glob import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import transformers as ppb\n",
    "from tokenizers import ByteLevelBPETokenizer\n",
    "import warnings\n",
    "import xml.etree.ElementTree as ET\n",
    "import json\n",
    "\n",
    "import spacy\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine-tune using BNC Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prep-clean up documents\n",
    "# Feed into BERT / roBERTa to fine-tune language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "bnc_corpus_loc = '../data/bnc2014spoken-xml/spoken/untagged/*'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1251"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(glob(bnc_corpus_loc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-7-d27e133ea087>\", line 30, in <module>\n",
      "    text.append(ET.fromstring(s).text)\n",
      "  File \"/Users/tom/anaconda3/envs/distil_bert/lib/python3.7/xml/etree/ElementTree.py\", line 1315, in XML\n",
      "    parser.feed(text)\n",
      "  File \"<string>\", line None\n",
      "xml.etree.ElementTree.ParseError: mismatched tag: line 1, column 107\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S99Z --- <u n=\"343\" who=\"UNKMALE\" whoConfidence=\"low\">good good ... ... <foreign lang=\"fre\">quelle heure est que ?</u>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "texts = {}\n",
    "text_speakers = {}\n",
    "child_els = {}\n",
    "speakers = {}\n",
    "for file in glob(bnc_corpus_loc):\n",
    "    name = file.split('/')[-1].split('.')[0]\n",
    "    tree = ET.parse(file)\n",
    "    root = tree.getroot()\n",
    "    file_id = root.attrib['id']\n",
    "    speakers[name] = root.find('header').find('list_speakers').text.split(' ')\n",
    "    text_s = []\n",
    "    text = []\n",
    "    c_els = []\n",
    "    utterances = tree.getroot().find('body').findall('u')\n",
    "    for u in utterances:\n",
    "        c_els += u.getchildren()\n",
    "        text_s.append(u.attrib['who'])\n",
    "        s = ET.tostring(u, encoding='unicode')\n",
    "        s = re.sub('<pause dur=\"short\"\\s*/>', '...', s)\n",
    "        s = re.sub('<pause dur=\"long\"\\s*/>', '... ...', s)\n",
    "        s = re.sub('<anon .*\\/>', '', s)\n",
    "        s = re.sub('<shift.*\\/>', '', s)\n",
    "        s = re.sub('<trunc>(\\w*)</trunc>', '\\\\1', s)\n",
    "        s = re.sub('<unclear>(.*)</unclear>', '\\\\1', s)\n",
    "        s = re.sub('<\\/?unclear\\s*\\/?>', '', s)\n",
    "        s = re.sub('<vocal desc=\"laugh\"\\s*\\/>', '&amp;=laughs', s)\n",
    "        s = re.sub('<\\/?event\\s*\\/?>', '', s)\n",
    "        s = re.sub('<foreign.*>(.*)</foreign>', '\\\\1', s)\n",
    "        try:\n",
    "            text.append(ET.fromstring(s).text)\n",
    "        except Exception as e:\n",
    "            import traceback\n",
    "            traceback.print_exc()\n",
    "            print(f'{name} --- {s}')\n",
    "            text.append(s)\n",
    "    child_els[name] = c_els\n",
    "    texts[name] = text\n",
    "    text_speakers[name] = text_s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- pause tags become '...'\n",
    "- trunc tags get the text included\n",
    "- shift is ignored.\n",
    "- anon removed - should be replaced with a random place?? \n",
    "- event is ignored\n",
    "- anon is ignored\n",
    "- vocal (laugh), &=laughs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_speakers_clean = {k: [s for s, t in zip(v, texts[k]) if t is None or len(t) > 0] for k,v in text_speakers.items()}\n",
    "texts_clean = {k: [t for t in v if t is not None and len(t) > 0] for k,v in texts.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame([{'file': f, 'speakers': speakers[f], 'text_speaker': text_speakers[f], 'texts': texts[f]} for f in texts.keys()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['texts_joined'] = df.file.apply(lambda f: ', '.join(texts_clean[f]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle('bnc_corpus_df.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle and split corpus text 80/20, train / test,\n",
    "# join each text into a single doc "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle('bnc_corpus_df.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_md\")\n",
    "nlp.pipeline = []\n",
    "sbd = nlp.create_pipe('sentencizer')\n",
    "nlp.add_pipe(sbd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24.2 s, sys: 343 ms, total: 24.6 s\n",
      "Wall time: 24.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sliding_window_sents = []\n",
    "for doc in df.texts_joined:\n",
    "    output_doc = []\n",
    "    sent_idx = 0\n",
    "    doc_sents = nlp(doc).sents\n",
    "    split_sents = [re.sub('\\s+', ' ', sent.text).split(' ') for sent in doc_sents]\n",
    "    trunc_split_sents = []\n",
    "    # spoken speech has potentially very long sentences. Split arbitrarily...\n",
    "    for sent in split_sents:\n",
    "        if len(sent) < 250:\n",
    "            trunc_split_sents.append(' '.join(sent))\n",
    "        else:\n",
    "            for idx in range(0, len(sent), 250):\n",
    "                new_sent = sent[idx:idx+250 if idx+250 <= len(sent) else len(sent)]\n",
    "                trunc_split_sents.append(' '.join(new_sent))\n",
    "    # sliding window of 4 'sentences' per line\n",
    "    for i, sent in enumerate(trunc_split_sents):\n",
    "        end_idx = i+6 if i+6 <= len(trunc_split_sents) else len(trunc_split_sents)\n",
    "        sliding_window_sents.append(' '.join(trunc_split_sents[i:end_idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('../data/pre-train/all_text_raw', 'w')\n",
    "f.write('\\n'.join(sliding_window_sents))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train BPE / BERTWordPiece Tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a tokenizer\n",
    "tokenizer = ByteLevelBPETokenizer(lowercase=False)\n",
    "\n",
    "# For BERT\n",
    "# from tokenizers import BertWordPieceTokenizer, ByteLevelBPETokenizer\n",
    "# # Initialize a tokenizer\n",
    "# tokenizer = BertWordPieceTokenizer(lowercase=False, handle_chinese_chars=False)\n",
    "\n",
    "# vocab size must be the same to fine-tune BERT / RoBERTa??\n",
    "\n",
    "# Customize training\n",
    "tokenizer.train(files='../data/pre-train/all_text_raw', vocab_size=50265, min_frequency=3, special_tokens=[\n",
    "    \"<s>\",\n",
    "    \"<pad>\",\n",
    "    \"</s>\",\n",
    "    \"<unk>\",\n",
    "    \"<mask>\",\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs('bncRoBERTaConfig', exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split text into LM train / test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split \n",
    "random_state = 42\n",
    "with open('../data/pre-train/all_text_raw') as f:\n",
    "    all_text = f.readlines()\n",
    "train, test = train_test_split(all_text, test_size=0.2, random_state=random_state)\n",
    "f = open('train', 'w')\n",
    "f.write('\\n'.join(train))\n",
    "f.close()\n",
    "f = open('test', 'w')\n",
    "f.write('\\n'.join(test))\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bncRoBERTaConfig/vocab.json', 'bncRoBERTaConfig/merges.txt']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save('bncRoBERTaConfig')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!bash run_training.sh"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:distil_bert]",
   "language": "python",
   "name": "conda-env-distil_bert-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}