notebooks/4_Score_consistency_test_set_CADec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In this notebook, you will learn how to score translations with CADec\n",
    "\n",
    "You can use this code to score consistency test sets with your trained model.\n",
    "\n",
    "(If you don't know how to load a model or how to operate with vocabularies yet, look in the notebook __1_Load_model_and_translate_baseline__.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.insert(0, 'path_to_good_translation_wrong_in_context') # insert your local path to the repo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load vocabularies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "DATA_PATH = # insert your path\n",
    "VOC_PATH =  # insert your path\n",
    "\n",
    "inp_voc = pickle.load(open(VOC_PATH + 'src.voc', 'rb'))\n",
    "out_voc = pickle.load(open(VOC_PATH + 'dst.voc', 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=1\n",
      "MODEL: use_dst_ctx  True\n",
      "reusing...\n",
      "DEC2: use_dst_ctx True\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0\n",
    "\n",
    "import tensorflow as tf\n",
    "import lib\n",
    "import lib.task.seq2seq.cadec.model as tr\n",
    "\n",
    "tf.reset_default_graph()\n",
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.99, allow_growth=True)\n",
    "sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))\n",
    "\n",
    "hp = {\n",
    "     \"num_layers\": 6,\n",
    "     \"num_heads\": 8,\n",
    "     \"ff_size\": 2048,\n",
    "     \"ffn_type\": \"conv_relu\",\n",
    "     \"hid_size\": 512,\n",
    "     \"emb_size\": 512,\n",
    "     \"res_steps\": \"nlda\", \n",
    "    \n",
    "     \"rescale_emb\": True,\n",
    "     \"inp_emb_bias\": True,\n",
    "     \"normalize_out\": True,\n",
    "     \"share_emb\": False,\n",
    "     \"replace\": 0,\n",
    "    \n",
    "     \"relu_dropout\": 0.1,\n",
    "     \"res_dropout\": 0.1,\n",
    "     \"attn_dropout\": 0.1,\n",
    "     \"label_smoothing\": 0.1,\n",
    "    \n",
    "     \"translator\": \"ingraph\",\n",
    "     \"beam_size\": 4,\n",
    "     \"beam_spread\": 3,\n",
    "     \"len_alpha\": 0.6,\n",
    "     \"attn_beta\": 0,\n",
    "    \n",
    "     \"dec1_attn_mode\": \"rdo_and_emb\",\n",
    "     \"share_loss\": False,\n",
    "     \"decoder2_name\": \"decoder2\",\n",
    "     \"max_ctx_sents\": 3,\n",
    "     \"use_dst_ctx\": True,\n",
    "}\n",
    "\n",
    "model = tr.CADecModel('mod', inp_voc, out_voc, inference_mode='lazy', **hp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_ckpt = # insert path to the final checkpoint\n",
    "var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "lib.train.saveload.load(path_to_ckpt, var_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Score <a name=\"translate\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load consistency test set (for example, we'll take deixis)\n",
    "path_to_testset = # path to your data\n",
    "test_src = open(path_to_testset + 'deixis_dev.src').readlines()\n",
    "test_dst = open(path_to_testset + 'deixis_dev.dst').readlines()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are groups of consecutive sentences and treir translations in our test set (sentences are separated with the `_eos` token):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\",\n",
       " \"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\",\n",
       " \"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\",\n",
       " \"just leave them outside the door . _eos the rooms need to be cleaned , once a week in minimum . _eos - that 's a policy ... _eos - didn 't i clear your policy ?\\n\"]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_src[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['просто оставьте их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не ваша политика ?\\n',\n",
       " 'просто оставьте их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не твоя политика ?\\n',\n",
       " 'просто оставь их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не твоя политика ?\\n',\n",
       " 'просто оставь их за дверью . _eos номера должны быть очищ `ены , раз в неделю минимум . _eos - это политика ... . _eos - разве я не ваша политика ?\\n']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dst[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To score sentences, we have to get loss values from the problem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INIT!!!\n",
      "PROBLEM INIT: dropout_teacher_forcing {'keep_first': False, 'method': 'random_word', 'dropout': 0}\n",
      "new batch sample\n"
     ]
    }
   ],
   "source": [
    "from lib.task.seq2seq.cadec.problem import CADecProblem\n",
    "from lib.task.seq2seq.data import make_batch_placeholder\n",
    "\n",
    "problem = CADecProblem({'mod': model},\n",
    "                        inference_flags={'mode':'beam_search'})\n",
    "batch_ph = make_batch_placeholder(model.make_feed_dict(model._get_batch_sample()))\n",
    "loss_values = problem.loss_values(batch=batch_ph, is_train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def num_sents(text):\n",
    "    return len(text.split(' _eos '))\n",
    "\n",
    "def make_cadec_batch_data(src_lines, dst_lines):\n",
    "    \"\"\"\n",
    "    src_lines contain groups of N sentences, last of which is to be translated (' _eos '-separated)\n",
    "    dst_lines contain translations of sentences in src_lines (' _eos '-separated)\n",
    "    \n",
    "    returns: list of pairs (src, dst) which one can give to a model\n",
    "    \"\"\"\n",
    "    assert len(src_lines) == len(dst_lines), \"Different number of text fragments\"\n",
    "    batch_src = []\n",
    "    batch_dst = []\n",
    "    for src, dst in zip(src_lines, dst_lines):\n",
    "        assert num_sents(src) == num_sents(dst)\n",
    "        src_elems = src.strip().split(' _eos ')\n",
    "        dst_elems = dst.strip().split(' _eos ')\n",
    "        batch_src.append(' _eos '.join([src_elems[-1]] + src_elems[:-1]) + ' _eos_eos ' + ' _eos '.join(dst_elems[:-1]))\n",
    "        batch_dst.append(dst_elems[-1])\n",
    "    return list(zip(batch_src, batch_dst))\n",
    "\n",
    "def score_batch(src_lines, dst_lines):\n",
    "    feed_dict = model.make_feed_dict(make_cadec_batch_data(src_lines, dst_lines))\n",
    "    feed = {batch_ph[k]: feed_dict[k] for k in batch_ph}\n",
    "    scores = sess.run(loss_values, feed)\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11.329386 , 14.755627 , 14.646226 , 11.4268875], dtype=float32)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_batch(test_src[:4], test_dst[:4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To score a test set, just do this for a sequence of batches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}