customize_bart_tokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modify BART tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from transformers import BartTokenize\n",
    "\n",
    "import logging\n",
    "logger = logging.getLogger(__name__)\n",
    "logger.setLevel(logging.INFO)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load default BART tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BartTokenizer.from_pretrained(\n",
    "    'bart-large',  \n",
    "    cache_dir='./cache',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./data/BartTokenizer/added_tokens.json\",'rt') as f:\n",
    "    added_tokens = json.load(f)\n",
    "with open(\"./data/BartTokenizer/vocab.json\",'rt') as f:\n",
    "    vocab = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modify"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add words to vocab, using the IDs from `'madeupword0000'`, `'madeupword0001'` and `'madeupword0002'`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "added_tokens['<newline>'] = 50261\n",
    "added_tokens['<bullet>'] = 50262\n",
    "added_tokens['<sep>'] = 50263"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete other words to make sure vocab size is the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "del vocab['madeupword0000']\n",
    "del vocab['madeupword0001']\n",
    "del vocab['madeupword0002']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the new vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"./data/BartTokenizer/vocab.json\",'wt') as f:\n",
    "    json.dump(vocab,f)\n",
    "with open(\"./data/BartTokenizer/added_tokens.json\",'wt') as f:\n",
    "    json.dump(added_tokens,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./data/BartTokenizer/vocab.json',\n",
       " './data/BartTokenizer/merges.txt',\n",
       " './data/BartTokenizer/special_tokens_map.json',\n",
       " './data/BartTokenizer/added_tokens.json')"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(\"./data/BartTokenizer\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[42891, 6, 99, 16, 70, 42, 14, 939, 50261, 192, 116]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(\"Hello, what is all this that i <newline> see?\".lower(), add_special_tokens=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Transformers BART",
   "language": "python",
   "name": "transformers-bart"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}