runDSKG.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model import *\n",
    "from reader import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts = Options()\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "\n",
    "#relation enhancement method\n",
    "use_jeval = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#parameters\n",
    "opts.hidden_size = 512\n",
    "opts.num_samples = 2048*3\n",
    "opts.keep_prob = 0.5\n",
    "opts.num_layers = 2\n",
    "opts.learning_rate=0.001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### You have to select one of the three datasets,\n",
    "#### the default dataset is FB15K-237 (datasets\\[0\\])\n",
    "#### you can also select another two datasets.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#select one dataset\n",
    "datasets = ['237', 'FB15K', 'WN18']\n",
    "used_dataset = datasets[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load file from local\n",
      "start gen filter mat\n",
      "WARNING:tensorflow:From /home/lingbing/Projects/kgcompletion/implementations/DSKG/model.py:379: get_or_create_global_step (from tensorflow.contrib.framework.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.train.get_or_create_global_step\n"
     ]
    }
   ],
   "source": [
    "if used_dataset == '237':\n",
    "    #set log filename and data path\n",
    "    file_name = '237-dskg-hs512'\n",
    "    opts.data_path = 'data/FB15k-237/'\n",
    "    \n",
    "    #different datasets use different data parser\n",
    "    model = FBRespective(opts, sess)\n",
    "if used_dataset == 'FB15K':\n",
    "    file_name = 'fb-dskg-hs512'\n",
    "    opts.data_path = 'data/FB15k-237/'\n",
    "    model = FBRespective(opts, sess)\n",
    "if used_dataset == 'WN18':\n",
    "    file_name = 'wn-dskg-hs512'\n",
    "    opts.data_path = 'data/wordnet-mlj12/'\n",
    "    model = WNRespective(opts, sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculate ranks\n",
    "def cal_ranks(probs, method, label):\n",
    "    if method == 'min':\n",
    "        probs = probs - probs[range(len(label)), label].reshape(len(probs), 1)\n",
    "        ranks = (probs > 0).sum(axis=1) + 1\n",
    "    else:\n",
    "        ranks = pd.DataFrame(probs).rank(axis=1, ascending=False, method=method)\n",
    "        ranks = ranks.values[range(len(label)), label]\n",
    "    return ranks\n",
    "\n",
    "#calculate performance\n",
    "def cal_performance(ranks, top=10):\n",
    "    m_r = sum(ranks) * 1.0 / len(ranks)\n",
    "    h_10 = sum(ranks <= top) * 1.0 / len(ranks)\n",
    "    mrr = (1. / ranks).sum() / len(ranks)\n",
    "    return m_r, h_10, mrr\n",
    "\n",
    "def eval_entity_prediction(model, data, filter_mat, method='min', return_ranks=False, return_probs=False, return_label_probs=False):\n",
    "    options = model._options\n",
    "    batch_size = options.batch_size\n",
    "    \n",
    "    label = data[:, 2]\n",
    "    \n",
    "    data, padding_num = model.padding_data(data)\n",
    "\n",
    "    num_batch = len(data) // batch_size \n",
    "    \n",
    "    e_placeholder, r_placeholder, fectch_entity_probs = model._eval_e, model._eval_r, model._entity_probs\n",
    "    \n",
    "    probs = []\n",
    "    for i in range(num_batch):\n",
    "        e = data[:, 0][i * batch_size:(i + 1) * batch_size]\n",
    "        r = data[:, 1][i * batch_size:(i + 1) * batch_size]\n",
    "        \n",
    "        feed_dict = {}\n",
    "        feed_dict[e_placeholder] = e\n",
    "        feed_dict[r_placeholder] = r\n",
    "        \n",
    "        probs.append(sess.run(fectch_entity_probs, feed_dict))\n",
    "    probs = np.concatenate(probs)[:len(data) - padding_num]\n",
    "\n",
    "    if return_label_probs:\n",
    "        return probs[range(len(label)), label]\n",
    "    \n",
    "    if return_probs:\n",
    "        return probs\n",
    "\n",
    "    filter_probs = probs * filter_mat\n",
    "    filter_probs[range(len(label)), label] = probs[range(len(label)), label]\n",
    "\n",
    "    filter_ranks = cal_ranks(filter_probs, method=method, label=label)\n",
    "    if return_ranks:\n",
    "        return filter_ranks\n",
    "    ranks = cal_ranks(probs, method=method, label=label)\n",
    "    m_r, h_10, mrr = cal_performance(ranks)\n",
    "    f_m_r, f_h_10, f_mrr = cal_performance(filter_ranks)\n",
    "    \n",
    "    return (m_r, h_10, mrr, f_m_r, f_h_10, f_mrr)\n",
    "\n",
    "def eval_relation_prediction(model, data, filter_mat, method='min', return_ranks=False, return_probs=False):\n",
    "    options = model._options\n",
    "    batch_size = options.batch_size\n",
    "    \n",
    "    #data[:, 0]-->e, data[:, 1]-->r, data[:, 2]-->e2\n",
    "    label = data[:, 1]\n",
    "    \n",
    "    data, padding_num = model.padding_data(data)\n",
    "\n",
    "    num_batch = len(data) // batch_size\n",
    "    \n",
    "    e_placeholder, fectch_relation_probs = model._eval_e, model._relation_probs\n",
    "    \n",
    "    probs = []\n",
    "    \n",
    "    for i in range(num_batch):\n",
    "        e = data[:, 0][i * batch_size:(i + 1) * batch_size]\n",
    "        \n",
    "        feed_dict = {}\n",
    "        feed_dict[e_placeholder] = e\n",
    "        \n",
    "        probs.append(sess.run(fectch_relation_probs, feed_dict))\n",
    "        \n",
    "    probs = np.concatenate(probs)[:len(data) - padding_num]\n",
    "    return probs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocess data\n",
    "\n",
    "test_data = np.array(model._test_data[['h_id', 'r_id', 't_id']].values)\n",
    "train_data = model._train_data[['h_id', 'r_id', 't_id']].values\n",
    "valid_data = model._valid_data[['h_id', 'r_id', 't_id']].values\n",
    "\n",
    "filter_mat = model._tail_test_filter_mat\n",
    "vfilter_mat = model._tail_valid_filter_mat\n",
    "\n",
    "all_data = np.concatenate([train_data, test_data,valid_data])\n",
    "p_data = np.concatenate([test_data,valid_data])\n",
    "\n",
    "def gen_rev_rel(test_data):\n",
    "    half = len(test_data)//2\n",
    "    forward = test_data[:half]\n",
    "    back = test_data[half:]\n",
    "    rev_rel_test_data = test_data[:]\n",
    "    rev_rel = np.concatenate([back[:,1], forward[:,1]])\n",
    "    return rev_rel\n",
    "\n",
    "rev_rel = gen_rev_rel(test_data)\n",
    "vrev_rel=  gen_rev_rel(valid_data)\n",
    "\n",
    "rev_rel_test_data = np.stack([np.arange(model._entity_num),np.arange(model._entity_num)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def cal_r(probs, label, filter_mat):\n",
    "    filter_probs = probs * filter_mat\n",
    "    \n",
    "    filter_probs[range(len(label)), label] = probs[range(len(label)), label]\n",
    "    filter_ranks = cal_ranks(filter_probs, method='min', label=label)\n",
    "    \n",
    "    return filter_ranks\n",
    "\n",
    "\n",
    "def joint_eval(test_data, filter_mat, rev_rel):\n",
    "    label=test_data[:, 2]\n",
    "\n",
    "    ep =  eval_entity_prediction(model, data=test_data, filter_mat=filter_mat, return_probs=True)\n",
    "    efr = cal_r(ep, label, filter_mat)\n",
    "    if use_jeval:\n",
    "        rp = eval_relation_prediction(model, rev_rel_test_data, filter_mat=None, return_probs=True).T\n",
    "        rp = rp**0.33\n",
    "        rp = rp[rev_rel]\n",
    "        joint_probs = ep * rp\n",
    "        joint_fr = cal_r(joint_probs, label, filter_mat)\n",
    "    else:\n",
    "        joint_fr = efr\n",
    "    return joint_fr, efr\n",
    "\n",
    "def joint_eval_raw(test_data, filter_mat, rev_rel):\n",
    "    label=test_data[:, 2]\n",
    "    \n",
    "    \n",
    "    ep =  eval_entity_prediction(model, data=test_data, filter_mat=filter_mat, return_probs=True)\n",
    "    efr = cal_ranks(ep, method='min', label=label)\n",
    "    if use_jeval:\n",
    "        rp = eval_relation_prediction(model, rev_rel_test_data, filter_mat=None, return_probs=True).T\n",
    "        rp = rp**0.33\n",
    "        rp = rp[rev_rel]\n",
    "        joint_probs = ep * rp\n",
    "        joint_fr = cal_ranks(joint_probs, method='min', label=label)\n",
    "    else:\n",
    "        joint_fr = efr\n",
    "    return joint_fr, efr\n",
    "\n",
    "def process_ranks(efr, i=0, last_mean_loss=1000, title=''):\n",
    "\n",
    "    MR, H1, MRR = cal_performance(efr[:len(efr)], top=1)\n",
    "    _, H10, _ = cal_performance(efr[:len(efr)], top=10)\n",
    "    msg = '%s epoch:%i, Hits@1:%.3f, Hits@10:%.3f, MR:%.3f, MRR:%.3f, mean_loss:%.3f' % (format(title,'<15'), i, H1, H10, MR, MRR, last_mean_loss)\n",
    "    print(msg)\n",
    "    return (i, H1, H10, MR, MRR, last_mean_loss)\n",
    "\n",
    "def handle_eval(i=0, last_mean_loss=1000, valid=True, test=False):\n",
    "    if valid:\n",
    "        jfr, efr = joint_eval(test_data=valid_data, filter_mat=vfilter_mat, rev_rel=vrev_rel)\n",
    "        jrr, rr = joint_eval_raw(test_data=valid_data, filter_mat=vfilter_mat, rev_rel=vrev_rel)\n",
    "        \n",
    "        process_ranks(rr, i, last_mean_loss, title='Valid-R')\n",
    "        process_ranks(jrr, i, last_mean_loss, title='Valid-R-RH')\n",
    "        \n",
    "        msg = process_ranks(efr, i, last_mean_loss, title='Valid-F')\n",
    "        jmsg = process_ranks(jfr, i, last_mean_loss, title='Valid-F-RH')\n",
    "        \n",
    "        #process early stop\n",
    "        global best_hits_1, dropped_time\n",
    "        current_hits_1 = jmsg[1]\n",
    "        if current_hits_1 > best_hits_1:\n",
    "            best_hits_1 = current_hits_1\n",
    "            dropped_time = 0\n",
    "        else:\n",
    "            dropped_time += 1\n",
    "        \n",
    "        \n",
    "        valid_results.append(msg)\n",
    "        valid_results.append(jmsg)\n",
    "        if i % 50 == 0:\n",
    "            pd.DataFrame(valid_results, columns=['epoch','Hits@1', 'Hits@10', 'MR', 'MRR', 'mean_loss']).to_csv('results/'+file_name+'valid')\n",
    "        \n",
    "    if test:\n",
    "        jfr, efr = joint_eval(test_data=test_data, filter_mat=filter_mat, rev_rel=rev_rel)\n",
    "        jrr, rr = joint_eval_raw(test_data=test_data, filter_mat=filter_mat, rev_rel=rev_rel)\n",
    "        \n",
    "        process_ranks(rr, i, last_mean_loss, title='Test-R')\n",
    "        process_ranks(jrr, i, last_mean_loss, title='Test-R-RH')\n",
    "        \n",
    "        \n",
    "        msg = process_ranks(efr, i, last_mean_loss, title='Test-F')\n",
    "        jmsg = process_ranks(jfr, i, last_mean_loss, title='Test-F-RH')\n",
    "        results.append(msg)\n",
    "        results.append(jmsg)\n",
    "        if i % 50 == 0:\n",
    "            pd.DataFrame(results, columns=['epoch','Hits@1', 'Hits@10', 'MR', 'MRR', 'mean_loss']).to_csv('results/'+file_name+'test')\n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch =0\n",
    "results = []\n",
    "valid_results = []\n",
    "last_mean_loss=1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#early stop setting\n",
    "global best_hits_1, dropped_time\n",
    "best_hits_1 = 0\n",
    "dropped_time = 0\n",
    "\n",
    "\n",
    "max_dropped_time = 3\n",
    "\n",
    "max_epoch = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## the function handle_eval(i=i, last_mean_loss=last_mean_loss, valid=True, test=True) returns the evaluation results:\n",
    "\n",
    "**Valid** and **Test** denote the datasets\n",
    "\n",
    "**R** denotes Raw results\n",
    "\n",
    "**F** denotes Filtered results\n",
    "\n",
    "**RH** denotes using relation enhancement method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid-R         epoch:0, Hits@1:0.000, Hits@10:0.001, MR:7244.412, MRR:0.001, mean_loss:1000.000\n",
      "Valid-R-RH      epoch:0, Hits@1:0.000, Hits@10:0.001, MR:7207.007, MRR:0.001, mean_loss:1000.000\n",
      "Valid-F         epoch:0, Hits@1:0.000, Hits@10:0.001, MR:7123.872, MRR:0.001, mean_loss:1000.000\n",
      "Valid-F-RH      epoch:0, Hits@1:0.000, Hits@10:0.001, MR:7086.761, MRR:0.001, mean_loss:1000.000\n",
      "2048 265 0.001 100000\n",
      "2048 265 0.001 9.09361203032\n",
      "2048 265 0.001 7.27079046537\n",
      "2048 265 0.001 6.80453052881\n",
      "2048 265 0.001 6.51655452116\n",
      "2048 265 0.001 6.30637829439\n",
      "2048 265 0.001 6.15042848767\n",
      "2048 265 0.001 6.03160143618\n",
      "2048 265 0.001 5.93442380113\n",
      "2048 265 0.001 5.85395814968\n",
      "2048 265 0.001 5.78373944804\n",
      "2048 265 0.001 5.72603296604\n",
      "2048 265 0.001 5.67308706068\n",
      "2048 265 0.001 5.63056590962\n",
      "2048 265 0.001 5.58961372016\n",
      "2048 265 0.001 5.554965676\n",
      "2048 265 0.001 5.52393407282\n",
      "2048 265 0.001 5.49358127162\n",
      "2048 265 0.001 5.46647464104\n",
      "2048 265 0.001 5.4425500096\n",
      "Valid-R         epoch:20, Hits@1:0.115, Hits@10:0.329, MR:370.658, MRR:0.183, mean_loss:5.421\n",
      "Valid-R-RH      epoch:20, Hits@1:0.115, Hits@10:0.331, MR:348.856, MRR:0.184, mean_loss:5.421\n",
      "Valid-F         epoch:20, Hits@1:0.235, Hits@10:0.507, MR:179.482, MRR:0.325, mean_loss:5.421\n",
      "Valid-F-RH      epoch:20, Hits@1:0.235, Hits@10:0.508, MR:157.079, MRR:0.326, mean_loss:5.421\n",
      "2048 265 0.001 5.42081977916\n",
      "2048 265 0.001 5.39593054034\n",
      "2048 265 0.001 5.37981551188\n",
      "2048 265 0.001 5.3586798434\n",
      "2048 265 0.001 5.34309368853\n",
      "2048 265 0.001 5.3283024572\n",
      "2048 265 0.001 5.31031187525\n",
      "2048 265 0.001 5.29884734424\n",
      "2048 265 0.001 5.2841843605\n",
      "2048 265 0.001 5.27037673626\n",
      "2048 265 0.001 5.25798344882\n",
      "2048 265 0.001 5.24785567769\n",
      "2048 265 0.001 5.23480470585\n",
      "2048 265 0.001 5.22333336776\n",
      "2048 265 0.001 5.21388682779\n",
      "2048 265 0.001 5.20277558813\n",
      "2048 265 0.001 5.19335983744\n",
      "2048 265 0.001 5.18649345974\n",
      "2048 265 0.001 5.17569714852\n",
      "2048 265 0.001 5.16858556316\n",
      "Valid-R         epoch:40, Hits@1:0.105, Hits@10:0.322, MR:387.854, MRR:0.173, mean_loss:5.161\n",
      "Valid-R-RH      epoch:40, Hits@1:0.106, Hits@10:0.324, MR:361.397, MRR:0.175, mean_loss:5.161\n",
      "Valid-F         epoch:40, Hits@1:0.237, Hits@10:0.513, MR:185.373, MRR:0.329, mean_loss:5.161\n",
      "Valid-F-RH      epoch:40, Hits@1:0.238, Hits@10:0.515, MR:158.577, MRR:0.330, mean_loss:5.161\n",
      "2048 265 0.001 5.16133787767\n",
      "2048 265 0.001 5.15325804476\n",
      "2048 265 0.001 5.14323685124\n",
      "2048 265 0.001 5.13592077111\n",
      "2048 265 0.001 5.12949021357\n",
      "2048 265 0.001 5.12579802747\n",
      "2048 265 0.001 5.11703923423\n",
      "2048 265 0.001 5.1090751486\n",
      "2048 265 0.001 5.10280488212\n",
      "2048 265 0.001 5.09822952882\n",
      "2048 265 0.001 5.09301392897\n",
      "2048 265 0.001 5.08479236747\n",
      "2048 265 0.001 5.08246331125\n",
      "2048 265 0.001 5.07293196084\n",
      "2048 265 0.001 5.07063253511\n",
      "2048 265 0.001 5.06519474533\n",
      "2048 265 0.001 5.05642882833\n",
      "2048 265 0.001 5.05317011599\n",
      "2048 265 0.001 5.05078372595\n",
      "2048 265 0.001 5.04445024886\n",
      "Valid-R         epoch:60, Hits@1:0.100, Hits@10:0.316, MR:391.935, MRR:0.168, mean_loss:5.040\n",
      "Valid-R-RH      epoch:60, Hits@1:0.100, Hits@10:0.319, MR:364.608, MRR:0.169, mean_loss:5.040\n",
      "Valid-F         epoch:60, Hits@1:0.241, Hits@10:0.515, MR:184.493, MRR:0.331, mean_loss:5.040\n",
      "Valid-F-RH      epoch:60, Hits@1:0.243, Hits@10:0.518, MR:157.207, MRR:0.333, mean_loss:5.040\n",
      "2048 265 0.001 5.04011414906\n",
      "2048 265 0.001 5.03484185777\n",
      "2048 265 0.001 5.02767049502\n",
      "2048 265 0.001 5.02488854966\n",
      "2048 265 0.001 5.02062058359\n",
      "2048 265 0.001 5.01676536776\n",
      "2048 265 0.001 5.01270985693\n",
      "2048 265 0.001 5.00808762604\n",
      "2048 265 0.001 5.00663186739\n",
      "2048 265 0.001 5.00035078121\n",
      "2048 265 0.001 4.99548447987\n",
      "2048 265 0.001 4.99327335897\n",
      "2048 265 0.001 4.98941845444\n",
      "2048 265 0.001 4.9866127374\n",
      "2048 265 0.001 4.98210557722\n",
      "2048 265 0.001 4.97855642787\n",
      "2048 265 0.001 4.97591094251\n",
      "2048 265 0.001 4.97283852235\n",
      "2048 265 0.001 4.96802916617\n",
      "2048 265 0.001 4.96560813976\n",
      "Valid-R         epoch:80, Hits@1:0.099, Hits@10:0.314, MR:395.851, MRR:0.166, mean_loss:4.962\n",
      "Valid-R-RH      epoch:80, Hits@1:0.099, Hits@10:0.316, MR:368.272, MRR:0.167, mean_loss:4.962\n",
      "Valid-F         epoch:80, Hits@1:0.244, Hits@10:0.520, MR:186.071, MRR:0.334, mean_loss:4.962\n",
      "Valid-F-RH      epoch:80, Hits@1:0.247, Hits@10:0.522, MR:158.645, MRR:0.337, mean_loss:4.962\n",
      "2048 265 0.001 4.9622341102\n",
      "2048 265 0.001 4.95989838546\n",
      "2048 265 0.001 4.95634431299\n",
      "2048 265 0.001 4.95208589086\n",
      "2048 265 0.001 4.95014158465\n",
      "2048 265 0.001 4.94587011517\n",
      "2048 265 0.001 4.94552612665\n",
      "2048 265 0.001 4.94327477149\n",
      "2048 265 0.001 4.93806362512\n",
      "2048 265 0.001 4.93702868516\n",
      "2048 265 0.001 4.93360422782\n",
      "2048 265 0.001 4.93267424601\n",
      "2048 265 0.001 4.92917868776\n",
      "2048 265 0.001 4.92621392664\n",
      "2048 265 0.001 4.92125320075\n",
      "2048 265 0.001 4.91761270559\n",
      "2048 265 0.001 4.9175652414\n",
      "2048 265 0.001 4.91613195527\n",
      "2048 265 0.001 4.91324371122\n",
      "2048 265 0.001 4.91059113269\n",
      "Valid-R         epoch:100, Hits@1:0.098, Hits@10:0.312, MR:398.310, MRR:0.165, mean_loss:4.907\n",
      "Valid-R-RH      epoch:100, Hits@1:0.098, Hits@10:0.314, MR:370.016, MRR:0.166, mean_loss:4.907\n",
      "Valid-F         epoch:100, Hits@1:0.246, Hits@10:0.517, MR:186.979, MRR:0.336, mean_loss:4.907\n",
      "Valid-F-RH      epoch:100, Hits@1:0.247, Hits@10:0.521, MR:158.819, MRR:0.338, mean_loss:4.907\n",
      "2048 265 0.001 4.9069460509\n",
      "2048 265 0.001 4.90718930442\n",
      "2048 265 0.001 4.9056848418\n",
      "2048 265 0.001 4.89896677665\n",
      "2048 265 0.001 4.89811030874\n",
      "2048 265 0.001 4.89929702687\n",
      "2048 265 0.001 4.89434393037\n",
      "2048 265 0.001 4.89171858194\n",
      "2048 265 0.001 4.89063142021\n",
      "2048 265 0.001 4.88877835544\n",
      "2048 265 0.001 4.88675947369\n",
      "2048 265 0.001 4.88341970804\n",
      "2048 265 0.001 4.88075021888\n",
      "2048 265 0.001 4.88285132174\n",
      "2048 265 0.001 4.87760300546\n",
      "2048 265 0.001 4.87337361462\n",
      "2048 265 0.001 4.87487774435\n",
      "2048 265 0.001 4.87110289988\n",
      "2048 265 0.001 4.8714802778\n",
      "2048 265 0.001 4.86843501037\n",
      "Valid-R         epoch:120, Hits@1:0.096, Hits@10:0.310, MR:398.049, MRR:0.163, mean_loss:4.867\n",
      "Valid-R-RH      epoch:120, Hits@1:0.097, Hits@10:0.313, MR:370.673, MRR:0.164, mean_loss:4.867\n",
      "Valid-F         epoch:120, Hits@1:0.247, Hits@10:0.521, MR:185.424, MRR:0.337, mean_loss:4.867\n",
      "Valid-F-RH      epoch:120, Hits@1:0.249, Hits@10:0.523, MR:158.243, MRR:0.340, mean_loss:4.867\n",
      "2048 265 0.001 4.86699851054\n",
      "2048 265 0.001 4.86251354397\n",
      "2048 265 0.001 4.86309245667\n",
      "2048 265 0.001 4.86383822819\n",
      "2048 265 0.001 4.85828735963\n",
      "2048 265 0.001 4.85878774535\n",
      "2048 265 0.001 4.8545981965\n",
      "2048 265 0.001 4.85508419433\n",
      "2048 265 0.001 4.85270731224\n",
      "2048 265 0.001 4.85138897986\n",
      "2048 265 0.001 4.84866700982\n",
      "2048 265 0.001 4.84962046101\n",
      "2048 265 0.001 4.8486354486\n",
      "2048 265 0.001 4.84577424211\n",
      "2048 265 0.001 4.84591508182\n",
      "2048 265 0.001 4.84180105677\n",
      "2048 265 0.001 4.83942394976\n",
      "2048 265 0.001 4.83940375886\n",
      "2048 265 0.001 4.83409057473\n",
      "2048 265 0.001 4.83423292232\n",
      "Valid-R         epoch:140, Hits@1:0.096, Hits@10:0.309, MR:398.772, MRR:0.162, mean_loss:4.836\n",
      "Valid-R-RH      epoch:140, Hits@1:0.096, Hits@10:0.312, MR:371.914, MRR:0.163, mean_loss:4.836\n",
      "Valid-F         epoch:140, Hits@1:0.246, Hits@10:0.520, MR:185.457, MRR:0.337, mean_loss:4.836\n",
      "Valid-F-RH      epoch:140, Hits@1:0.248, Hits@10:0.525, MR:158.743, MRR:0.339, mean_loss:4.836\n",
      "2048 265 0.001 4.83577785312\n",
      "2048 265 0.001 4.8314274482\n",
      "2048 265 0.001 4.83200995787\n",
      "2048 265 0.001 4.82987250382\n",
      "2048 265 0.001 4.83134417444\n",
      "2048 265 0.001 4.82584598469\n",
      "2048 265 0.001 4.82414369403\n",
      "2048 265 0.001 4.8247384971\n",
      "2048 265 0.001 4.82240642152\n",
      "2048 265 0.001 4.82036401821\n",
      "2048 265 0.001 4.82017103771\n",
      "2048 265 0.001 4.8165915975\n",
      "2048 265 0.001 4.81638835511\n",
      "2048 265 0.001 4.81693351494\n",
      "2048 265 0.001 4.81400967364\n",
      "2048 265 0.001 4.81374819594\n",
      "2048 265 0.001 4.81425301534\n",
      "2048 265 0.001 4.81075504771\n",
      "2048 265 0.001 4.80885947066\n",
      "2048 265 0.001 4.80741237604\n",
      "Valid-R         epoch:160, Hits@1:0.096, Hits@10:0.309, MR:401.672, MRR:0.162, mean_loss:4.808\n",
      "Valid-R-RH      epoch:160, Hits@1:0.096, Hits@10:0.311, MR:373.561, MRR:0.163, mean_loss:4.808\n",
      "Valid-F         epoch:160, Hits@1:0.248, Hits@10:0.521, MR:187.609, MRR:0.338, mean_loss:4.808\n",
      "Valid-F-RH      epoch:160, Hits@1:0.250, Hits@10:0.525, MR:159.663, MRR:0.340, mean_loss:4.808\n",
      "2048 265 0.001 4.80761614746\n",
      "2048 265 0.001 4.80463941502\n",
      "2048 265 0.001 4.80406578352\n",
      "2048 265 0.001 4.80095164461\n",
      "2048 265 0.001 4.80108212345\n",
      "2048 265 0.001 4.80114481044\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2048 265 0.001 4.80037028115\n",
      "2048 265 0.001 4.7982216979\n",
      "2048 265 0.001 4.79662321918\n",
      "2048 265 0.001 4.79763258268\n",
      "2048 265 0.001 4.79423569193\n",
      "2048 265 0.001 4.7947435397\n",
      "2048 265 0.001 4.79476264738\n",
      "2048 265 0.001 4.79317397531\n",
      "2048 265 0.001 4.79188492073\n",
      "2048 265 0.001 4.79028921487\n",
      "2048 265 0.001 4.78686061895\n",
      "2048 265 0.001 4.78787261135\n",
      "2048 265 0.001 4.78506594964\n",
      "2048 265 0.001 4.78369640494\n",
      "Valid-R         epoch:180, Hits@1:0.095, Hits@10:0.308, MR:402.373, MRR:0.161, mean_loss:4.785\n",
      "Valid-R-RH      epoch:180, Hits@1:0.096, Hits@10:0.312, MR:374.294, MRR:0.162, mean_loss:4.785\n",
      "Valid-F         epoch:180, Hits@1:0.247, Hits@10:0.524, MR:188.091, MRR:0.339, mean_loss:4.785\n",
      "Valid-F-RH      epoch:180, Hits@1:0.251, Hits@10:0.527, MR:160.151, MRR:0.342, mean_loss:4.785\n",
      "2048 265 0.001 4.78535209692\n",
      "2048 265 0.001 4.78461379285\n",
      "2048 265 0.001 4.78269881482\n",
      "2048 265 0.001 4.78196121612\n",
      "2048 265 0.001 4.77994404199\n",
      "2048 265 0.001 4.77947779062\n",
      "2048 265 0.001 4.77817325052\n",
      "2048 265 0.001 4.77762948162\n",
      "2048 265 0.001 4.77584115514\n",
      "2048 265 0.001 4.77680237968\n",
      "2048 265 0.001 4.77294660604\n",
      "2048 265 0.001 4.77391648742\n",
      "2048 265 0.001 4.77626388838\n",
      "2048 265 0.001 4.77167259432\n",
      "2048 265 0.001 4.76923933569\n",
      "2048 265 0.001 4.77130697538\n",
      "2048 265 0.001 4.77004152514\n",
      "2048 265 0.001 4.76980110384\n",
      "2048 265 0.001 4.76645916453\n",
      "2048 265 0.001 4.76657837382\n",
      "Valid-R         epoch:200, Hits@1:0.095, Hits@10:0.307, MR:404.376, MRR:0.161, mean_loss:4.766\n",
      "Valid-R-RH      epoch:200, Hits@1:0.096, Hits@10:0.309, MR:375.322, MRR:0.162, mean_loss:4.766\n",
      "Valid-F         epoch:200, Hits@1:0.250, Hits@10:0.521, MR:189.645, MRR:0.340, mean_loss:4.766\n",
      "Valid-F-RH      epoch:200, Hits@1:0.251, Hits@10:0.525, MR:160.771, MRR:0.342, mean_loss:4.766\n",
      "2048 265 0.001 4.76555655677\n",
      "2048 265 0.001 4.76438245953\n",
      "2048 265 0.001 4.76233232426\n",
      "2048 265 0.001 4.76223519523\n",
      "2048 265 0.001 4.76318649796\n",
      "2048 265 0.001 4.76017132705\n",
      "2048 265 0.001 4.7599837825\n",
      "2048 265 0.001 4.75924422786\n",
      "2048 265 0.001 4.75676840656\n",
      "2048 265 0.001 4.75652560468\n",
      "2048 265 0.001 4.75762795142\n",
      "2048 265 0.001 4.75407511333\n",
      "2048 265 0.001 4.75399253233\n",
      "2048 265 0.001 4.75186659975\n",
      "2048 265 0.001 4.75363727426\n",
      "2048 265 0.001 4.75227850248\n",
      "2048 265 0.001 4.75116153033\n",
      "2048 265 0.001 4.74879030911\n",
      "2048 265 0.001 4.74851984348\n",
      "2048 265 0.001 4.74694727772\n",
      "Valid-R         epoch:220, Hits@1:0.095, Hits@10:0.307, MR:402.833, MRR:0.161, mean_loss:4.749\n",
      "Valid-R-RH      epoch:220, Hits@1:0.096, Hits@10:0.309, MR:375.196, MRR:0.162, mean_loss:4.749\n",
      "Valid-F         epoch:220, Hits@1:0.248, Hits@10:0.524, MR:187.795, MRR:0.339, mean_loss:4.749\n",
      "Valid-F-RH      epoch:220, Hits@1:0.250, Hits@10:0.527, MR:160.341, MRR:0.342, mean_loss:4.749\n",
      "2048 265 0.001 4.74912105236\n",
      "2048 265 0.001 4.74772774678\n",
      "2048 265 0.001 4.74631114816\n",
      "2048 265 0.001 4.74694122278\n",
      "2048 265 0.001 4.74484395801\n",
      "2048 265 0.001 4.74422369903\n",
      "2048 265 0.001 4.74418475133\n",
      "2048 265 0.001 4.74045221221\n",
      "2048 265 0.001 4.74246766432\n",
      "2048 265 0.001 4.74204134131\n",
      "2048 265 0.001 4.742168162\n",
      "2048 265 0.001 4.74026751608\n",
      "2048 265 0.001 4.73884513963\n",
      "2048 265 0.001 4.73924898471\n",
      "2048 265 0.001 4.73960418161\n",
      "2048 265 0.001 4.7365594648\n",
      "2048 265 0.001 4.73732690811\n",
      "2048 265 0.001 4.7369084844\n",
      "2048 265 0.001 4.73437707109\n",
      "2048 265 0.001 4.73320778721\n",
      "Valid-R         epoch:240, Hits@1:0.095, Hits@10:0.307, MR:403.569, MRR:0.161, mean_loss:4.733\n",
      "Valid-R-RH      epoch:240, Hits@1:0.096, Hits@10:0.310, MR:375.678, MRR:0.162, mean_loss:4.733\n",
      "Valid-F         epoch:240, Hits@1:0.251, Hits@10:0.523, MR:188.286, MRR:0.341, mean_loss:4.733\n",
      "Valid-F-RH      epoch:240, Hits@1:0.253, Hits@10:0.526, MR:160.648, MRR:0.344, mean_loss:4.733\n",
      "2048 265 0.001 4.73263033921\n",
      "2048 265 0.001 4.73179915626\n",
      "2048 265 0.001 4.73121744552\n",
      "2048 265 0.001 4.73028619334\n",
      "2048 265 0.001 4.73138578163\n",
      "2048 265 0.001 4.73047043962\n",
      "2048 265 0.001 4.72978856249\n",
      "2048 265 0.001 4.72809771592\n",
      "2048 265 0.001 4.72680645349\n",
      "2048 265 0.001 4.72646858467\n",
      "2048 265 0.001 4.72769547408\n",
      "2048 265 0.001 4.72731340336\n",
      "2048 265 0.001 4.72597822873\n",
      "2048 265 0.001 4.72328133313\n",
      "2048 265 0.001 4.72272190958\n",
      "2048 265 0.001 4.7205227546\n",
      "2048 265 0.001 4.72306655848\n",
      "2048 265 0.001 4.722256261\n",
      "2048 265 0.001 4.72225400637\n",
      "2048 265 0.001 4.72088573024\n",
      "Valid-R         epoch:260, Hits@1:0.096, Hits@10:0.305, MR:407.011, MRR:0.161, mean_loss:4.719\n",
      "Valid-R-RH      epoch:260, Hits@1:0.096, Hits@10:0.307, MR:377.947, MRR:0.162, mean_loss:4.719\n",
      "Valid-F         epoch:260, Hits@1:0.250, Hits@10:0.520, MR:191.482, MRR:0.340, mean_loss:4.719\n",
      "Valid-F-RH      epoch:260, Hits@1:0.252, Hits@10:0.523, MR:162.654, MRR:0.343, mean_loss:4.719\n",
      "2048 265 0.001 4.71885499414\n",
      "2048 265 0.001 4.72025932636\n",
      "2048 265 0.001 4.71633123362\n",
      "2048 265 0.001 4.71682664403\n",
      "2048 265 0.001 4.7170311424\n",
      "2048 265 0.001 4.71714027693\n",
      "2048 265 0.001 4.71554190438\n",
      "2048 265 0.001 4.71433655361\n",
      "2048 265 0.001 4.71347163038\n",
      "2048 265 0.001 4.71304446346\n",
      "2048 265 0.001 4.71313359063\n",
      "2048 265 0.001 4.71049668654\n",
      "2048 265 0.001 4.71237689324\n",
      "2048 265 0.001 4.71302183079\n",
      "2048 265 0.001 4.71071549182\n",
      "2048 265 0.001 4.71085032517\n",
      "2048 265 0.001 4.71046395392\n",
      "2048 265 0.001 4.71079774353\n",
      "2048 265 0.001 4.70840640698\n",
      "2048 265 0.001 4.70924455895\n",
      "Valid-R         epoch:280, Hits@1:0.095, Hits@10:0.306, MR:407.008, MRR:0.160, mean_loss:4.709\n",
      "Valid-R-RH      epoch:280, Hits@1:0.096, Hits@10:0.309, MR:378.200, MRR:0.161, mean_loss:4.709\n",
      "Valid-F         epoch:280, Hits@1:0.250, Hits@10:0.525, MR:191.370, MRR:0.341, mean_loss:4.709\n",
      "Valid-F-RH      epoch:280, Hits@1:0.254, Hits@10:0.528, MR:162.758, MRR:0.344, mean_loss:4.709\n",
      "2048 265 0.001 4.70879183535\n",
      "2048 265 0.001 4.70807528406\n",
      "2048 265 0.001 4.70537684279\n",
      "2048 265 0.001 4.70576748218\n",
      "2048 265 0.001 4.70532958553\n",
      "2048 265 0.001 4.70691455445\n",
      "2048 265 0.001 4.70457881712\n",
      "2048 265 0.001 4.70080140492\n",
      "2048 265 0.001 4.70350259025\n",
      "2048 265 0.001 4.70188438128\n",
      "2048 265 0.001 4.70202291777\n",
      "2048 265 0.001 4.70287181566\n",
      "2048 265 0.001 4.70081415536\n",
      "2048 265 0.001 4.70013864085\n",
      "2048 265 0.001 4.69929245823\n",
      "2048 265 0.001 4.69979811075\n",
      "2048 265 0.001 4.70164606886\n",
      "2048 265 0.001 4.69853385889\n",
      "2048 265 0.001 4.69819307147\n",
      "2048 265 0.001 4.69593291373\n",
      "Valid-R         epoch:299, Hits@1:0.095, Hits@10:0.305, MR:407.313, MRR:0.161, mean_loss:4.696\n",
      "Valid-R-RH      epoch:299, Hits@1:0.096, Hits@10:0.308, MR:378.480, MRR:0.161, mean_loss:4.696\n",
      "Valid-F         epoch:299, Hits@1:0.252, Hits@10:0.524, MR:191.599, MRR:0.342, mean_loss:4.696\n",
      "Valid-F-RH      epoch:299, Hits@1:0.254, Hits@10:0.527, MR:162.963, MRR:0.344, mean_loss:4.696\n",
      "Test-R          epoch:299, Hits@1:0.093, Hits@10:0.300, MR:416.841, MRR:0.158, mean_loss:4.696\n",
      "Test-R-RH       epoch:299, Hits@1:0.094, Hits@10:0.303, MR:386.846, MRR:0.159, mean_loss:4.696\n",
      "Test-F          epoch:299, Hits@1:0.244, Hits@10:0.518, MR:203.712, MRR:0.335, mean_loss:4.696\n",
      "Test-F-RH       epoch:299, Hits@1:0.247, Hits@10:0.523, MR:173.972, MRR:0.338, mean_loss:4.696\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for i in range(epoch, max_epoch):\n",
    "    if i % 20 == 0:\n",
    "        handle_eval(i=i, last_mean_loss=last_mean_loss, valid=True, test=False)\n",
    "    last_mean_loss = model.train()\n",
    "    epoch += 1\n",
    "    \n",
    "    #early stop\n",
    "    if dropped_time >= max_dropped_time:\n",
    "        break\n",
    "    \n",
    "handle_eval(i=i, last_mean_loss=last_mean_loss, valid=True, test=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}