web_app.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import tensorflow as tf\n",
    "tf.get_logger().setLevel('ERROR')\n",
    "\n",
    "from utils.process_text import init_tokenizer\n",
    "\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import display\n",
    "\n",
    "\n",
    "TOKEN_CONFIG = \"configs/tokenizer_default_config.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# Define tokenization pattern:\n",
    "PUNCTUATION_PATTERN = re.compile(r\"[\\.\\!\\?;]+\\s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EUMT\n",
    "\n",
    "> English-Ukrainian bidirectional neural machine translator, based on [fastText](https://fasttext.cc/docs/en/support.html) word embeddings (*sisg-* model [1]) and default Transformer architecture [2] of the [OpenNMT framework](https://opennmt.net/).\n",
    "\n",
    "The сurrent version is trained on one of the [OPUS datasets](https://opus.nlpl.eu/) [3]: the [QED dataset](https://opus.nlpl.eu/QED-v2.0a.php) [4]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer_en = tf.saved_model.load(\"models_serve/QED_en/export/255000\")\n",
    "transformer_uk = tf.saved_model.load(\"models_serve/QED_uk/export/30000\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(tokenizer, data):\n",
    "    \"\"\"\n",
    "    Tokenize list of strings\n",
    "    \"\"\"\n",
    "\n",
    "    all_tokens = []\n",
    "    lengths = []\n",
    "\n",
    "    for text in data:\n",
    "        tokens = tokenizer._tokenize_string(text.lower(), False)\n",
    "        length = len(tokens)\n",
    "        all_tokens.append(tokens)\n",
    "        lengths.append(length)\n",
    "    max_length = max(lengths)\n",
    "\n",
    "    for tokens, length in zip(all_tokens, lengths):\n",
    "        if length < max_length:\n",
    "            tokens += [\"\"] * (max_length - length)\n",
    "\n",
    "    inputs = {\n",
    "        \"tokens\": tf.constant(all_tokens, dtype=tf.string),\n",
    "        \"length\": tf.constant(lengths, dtype=tf.int32),\n",
    "    }\n",
    "\n",
    "    return inputs\n",
    "\n",
    "def postprocess(tokenizer, outputs, punctuation_signs):\n",
    "    \"\"\"\n",
    "    Detokenize and merge list of tokens\n",
    "    \"\"\"\n",
    "    sent_tokens = outputs[\"tokens\"].numpy()\n",
    "    sent_lenghts = outputs[\"length\"].numpy()\n",
    "\n",
    "    assert len(punctuation_signs) <= sent_tokens.shape[0] <= len(punctuation_signs) + 1, \\\n",
    "                \"Tokenization error has occurred\"\n",
    "\n",
    "    translation = \"\"\n",
    "    for idx, (tokens, length) in enumerate(zip(sent_tokens, sent_lenghts)):\n",
    "        tokens = tokens[0][: length[0]].tolist()\n",
    "        translation += tokenizer._detokenize_string(tokens).replace(\"<unk>\", \"\").capitalize()\n",
    "        if len(punctuation_signs) == sent_tokens.shape[0] or idx < len(punctuation_signs):\n",
    "            translation += punctuation_signs[idx]\n",
    "\n",
    "    return translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "is_uk = False\n",
    "\n",
    "# Initialize tokenizer:\n",
    "with open(TOKEN_CONFIG, \"r\") as f:\n",
    "    tokenizer_config = json.load(f)\n",
    "\n",
    "tokenizer = init_tokenizer(tokenizer_config)\n",
    "\n",
    "labels = [\"English\", \"Ukrainian\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Buttons:\n",
    "switch_language = widgets.Button(description='Switch language')\n",
    "translate = widgets.Button(description='Translate')\n",
    "\n",
    "buttons = widgets.VBox([switch_language, translate])\n",
    "\n",
    "# Text fields:\n",
    "source_label = widgets.Label(labels[is_uk])\n",
    "source_text = widgets.Textarea()\n",
    "source = widgets.VBox([source_label, source_text])\n",
    "\n",
    "target_label = widgets.Label(labels[not is_uk])\n",
    "target_text = widgets.Output()\n",
    "target = widgets.VBox([target_label, target_text])\n",
    "\n",
    "dashboard = widgets.HBox([source, buttons, target])\n",
    "\n",
    "display(dashboard)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_click_translate(change):\n",
    "    \"\"\"\n",
    "    Translate user text\n",
    "    \"\"\"\n",
    "\n",
    "    # Get the input text and split it into phrases:\n",
    "    source_doc = PUNCTUATION_PATTERN.split(source_text.value)\n",
    "    punctuation_signs = PUNCTUATION_PATTERN.findall(source_text.value)\n",
    "\n",
    "    # Tokenize each phrase:\n",
    "    inputs = preprocess(tokenizer, source_doc)\n",
    "    \n",
    "    # Translate phrases:\n",
    "    translation = transformer_uk.signatures[\"serving_default\"](**inputs) \\\n",
    "        if is_uk else transformer_en.signatures[\"serving_default\"](**inputs)\n",
    "\n",
    "    # Refresh output:\n",
    "    target_text.clear_output()\n",
    "    \n",
    "    # Detokenize and merge translated phrases:\n",
    "    with target_text:\n",
    "        print(postprocess(tokenizer, translation, punctuation_signs))\n",
    "\n",
    "translate.on_click(on_click_translate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_click_switch(change):\n",
    "    \"\"\"\n",
    "    Swap source and target languages\n",
    "    \"\"\"\n",
    "\n",
    "    global is_uk\n",
    "    is_uk = not is_uk\n",
    "\n",
    "    source_label.value = labels[is_uk]\n",
    "    target_label.value = labels[not is_uk]\n",
    "    \n",
    "    source_text.value = \"\"\n",
    "    target_text.clear_output()\n",
    "\n",
    "switch_language.on_click(on_click_switch, is_uk)"
   ]
  },
  {
   "source": [
    "## References\n",
    "\n",
    "1. Bojanowski, P., Grave, E., Joulin, A., & Mikolov, T. (2017). [Enriching word vectors with subword information](https://www.mitpressjournals.org/doi/pdfplus/10.1162/tacl_a_00051?source=post_page---------------------------). Transactions of the Association for Computational Linguistics, 5, 135-146.\n",
    "2. Vaswani, A., Shazeer, N., Parmar, N., Uszkoreit, J., Jones, L., Gomez, A. N., ... & Polosukhin, I. (2017). [Attention is all you need](https://arxiv.org/pdf/1706.03762.pdf%EF%BC%89%E6%8F%8F%E8%BF%B0%E4%BA%86%E8%BF%99%E6%A0%B7%E5%81%9A%E7%9A%84%E5%8E%9F%E5%9B%A0%E3%80%82). arXiv preprint arXiv:1706.03762.\n",
    "3. Jörg Tiedemann, 2012, [Parallel Data, Tools and Interfaces in OPUS](http://www.lrec-conf.org/proceedings/lrec2012/pdf/463_Paper.pdf). In *Proceedings of the 8th International Conference on Language Resources and Evaluation (LREC'2012)*.\n",
    "4. A. Abdelali, F. Guzman, H. Sajjad and S. Vogel, \"[The AMARA Corpus: Building parallel language resources for the educational domain](https://www.aclweb.org/anthology/L14-1675/)\", The Proceedings of the 9th International Conference on Language Resources and Evaluation (LREC'14). Reykjavik, Iceland, 2014. Pp. 1856-1862. Isbn. 978-2-9517408-8-4."
   ],
   "cell_type": "markdown",
   "metadata": {}
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.7.6 64-bit ('base': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "metadata": {
   "interpreter": {
    "hash": "54bfe388838f029d69968b9e925c392f1790cd451ad44dba51ac4808fa59fdcd"
   }
  },
  "interpreter": {
   "hash": "54bfe388838f029d69968b9e925c392f1790cd451ad44dba51ac4808fa59fdcd"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}