math_ds_sandbox.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install --user tensorboardX\n",
    "!pip install --user --force https://github.com/chengs/tqdm/archive/colab.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "0m8YROdDpRAk",
    "outputId": "e6bb48eb-1c2b-4a99-9558-be452345ba23"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Torch Version 1.0.0.dev20190402\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils import data\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from tensorboard import Tensorboard\n",
    "\n",
    "from math_dataset import (\n",
    "    LazyFileMathDataset,\n",
    "    MathDatasetManager\n",
    ")\n",
    "\n",
    "from transformer.Models import Transformer\n",
    "from math_dataset import (\n",
    "    random_split_dataset, MAX_ANSWER_SZ, MAX_QUESTION_SZ, VOCAB_SZ,\n",
    "    question_answer_to_position_batch_collate_fn, question_to_position_batch_collate_fn\n",
    ")\n",
    "import model_process\n",
    "from itertools import islice\n",
    "\n",
    "print(\"Torch Version\", torch.__version__)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'q': 'Let d = -25019/90 - -278. Let v(j) be the third derivative of 0 + 1/27*j**3 - d*j**5 + 1/54*j**4 + 3*j**2 + 0*j. Suppose v(o) = 0. What is o?', 'q_enc': array([ 2, 45, 70, 85,  1, 69,  1, 30,  1, 14, 19, 22, 17, 18, 26, 16, 26,\n",
      "       17,  1, 14,  1, 14, 19, 24, 25, 15,  1, 45, 70, 85,  1, 87,  9, 75,\n",
      "       10,  1, 67, 70,  1, 85, 73, 70,  1, 85, 73, 74, 83, 69,  1, 69, 70,\n",
      "       83, 74, 87, 66, 85, 74, 87, 70,  1, 80, 71,  1, 17,  1, 12,  1, 18,\n",
      "       16, 19, 24, 11, 75, 11, 11, 20,  1, 14,  1, 69, 11, 75, 11, 11, 22,\n",
      "        1, 12,  1, 18, 16, 22, 21, 11, 75, 11, 11, 21,  1, 12,  1, 20, 11,\n",
      "       75, 11, 11, 19,  1, 12,  1, 17, 11, 75, 15,  1, 52, 86, 81, 81, 80,\n",
      "       84, 70,  1, 87,  9, 80, 10,  1, 30,  1, 17, 15,  1, 56, 73, 66, 85,\n",
      "        1, 74, 84,  1, 80, 32,  3], dtype=uint8), 'a': '-1/3, 1', 'a_enc': array([ 2, 14, 18, 16, 20, 13,  1, 18,  3], dtype=uint8)}\n",
      "{'q': 'Let t(v) be the second derivative of 1/16*v**4 + 3/8*v**2 - 3*v + 1/4*v**3 + 0. Factor t(s).', 'q_enc': array([ 2, 45, 70, 85,  1, 85,  9, 87, 10,  1, 67, 70,  1, 85, 73, 70,  1,\n",
      "       84, 70, 68, 80, 79, 69,  1, 69, 70, 83, 74, 87, 66, 85, 74, 87, 70,\n",
      "        1, 80, 71,  1, 18, 16, 18, 23, 11, 87, 11, 11, 21,  1, 12,  1, 20,\n",
      "       16, 25, 11, 87, 11, 11, 19,  1, 14,  1, 20, 11, 87,  1, 12,  1, 18,\n",
      "       16, 21, 11, 87, 11, 11, 20,  1, 12,  1, 17, 15,  1, 39, 66, 68, 85,\n",
      "       80, 83,  1, 85,  9, 84, 10, 15,  3], dtype=uint8), 'a': '3*(s + 1)**2/4', 'a_enc': array([ 2, 20, 11,  9, 84,  1, 12,  1, 18, 10, 11, 11, 19, 16, 21,  3],\n",
      "      dtype=uint8)}\n",
      "50\n",
      "{'q': 'Factor 4/9 - 14/9*f**2 - 10/9*f.', 'q_enc': array([ 2, 39, 66, 68, 85, 80, 83,  1, 21, 16, 26,  1, 14,  1, 18, 21, 16,\n",
      "       26, 11, 71, 11, 11, 19,  1, 14,  1, 18, 17, 16, 26, 11, 71, 15,  3],\n",
      "      dtype=uint8), 'a': '-2*(f + 1)*(7*f - 2)/9', 'a_enc': array([ 2, 14, 19, 11,  9, 71,  1, 12,  1, 18, 10, 11,  9, 24, 11, 71,  1,\n",
      "       14,  1, 19, 10, 16, 26,  3], dtype=uint8)}\n"
     ]
    }
   ],
   "source": [
    "my_ds = LazyFileMathDataset(\n",
    "    \"/home/mandubian/datasets/mathematics_dataset/v1.0/train-easy/algebra__polynomial_roots_composed.txt\",\n",
    "    lazy_load=True\n",
    ")\n",
    "print(my_ds[0])\n",
    "print(my_ds[500000])\n",
    "my_ds.set_max_elements(50)\n",
    "print(len(my_ds))\n",
    "print(my_ds[49])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initialized MultiFilesMathDataset with categories ['algebra', 'numbers', 'polynomials', 'arithmetic', 'measurement', 'comparison', 'probability', 'calculus'] and types ['train-easy', 'train-medium', 'train-hard', 'interpolate', 'extrapolate']\n"
     ]
    }
   ],
   "source": [
    "mdsmgr = MathDatasetManager(\n",
    "  \"/home/mandubian/datasets/mathematics_dataset/v1.0\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "categories dict_keys(['algebra', 'numbers', 'polynomials', 'arithmetic', 'measurement', 'comparison', 'probability', 'calculus'])\n",
      "types dict_keys(['train-easy', 'train-medium', 'train-hard', 'interpolate', 'extrapolate'])\n",
      "modules of arithmetic dict_keys(['mul', 'add_or_sub_in_base', 'simplify_surd', 'mul_div_multiple', 'mixed', 'nearest_integer_root', 'div', 'add_or_sub', 'add_sub_multiple', 'add_sub_multiple_longer', 'mul_div_multiple_longer', 'div_big', 'mul_big', 'mixed_longer', 'add_or_sub_big'])\n"
     ]
    }
   ],
   "source": [
    "print(\"categories\", mdsmgr.get_categories())\n",
    "print(\"types\", mdsmgr.get_types())\n",
    "print(\"modules of arithmetic\", mdsmgr.get_modules_for_category('arithmetic'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "adding category algebra/../train-easy\n",
      "added module algebra/polynomial_roots_composed/train-easy\n",
      "added module algebra/polynomial_roots/train-easy\n",
      "added module algebra/linear_1d_composed/train-easy\n",
      "added module algebra/linear_2d_composed/train-easy\n",
      "added module algebra/linear_2d/train-easy\n",
      "added module algebra/sequence_nth_term/train-easy\n",
      "added module algebra/linear_1d/train-easy\n",
      "added module algebra/sequence_next_term/train-easy\n",
      "size 5333328\n"
     ]
    }
   ],
   "source": [
    "ds = mdsmgr.build_dataset_from_category('algebra', 'train-easy')\n",
    "print(\"size\", len(ds))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size 666666\n"
     ]
    }
   ],
   "source": [
    "ds_arithmetic_add_or_sub_easy = mdsmgr.build_dataset_from_module('arithmetic', 'add_or_sub', 'train-easy')\n",
    "print(\"size\", len(ds_arithmetic_add_or_sub_easy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size 666666\n"
     ]
    }
   ],
   "source": [
    "ds_arithmetic_add_or_sub_hard = mdsmgr.build_dataset_from_module('arithmetic', 'add_or_sub', 'train-hard')\n",
    "print(\"size\", len(ds_arithmetic_add_or_sub_hard))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size 10000\n"
     ]
    }
   ],
   "source": [
    "ds_arithmetic_add_or_sub_interpolate = mdsmgr.build_dataset_from_module('arithmetic', 'add_or_sub', 'interpolate')\n",
    "print(\"size\", len(ds_arithmetic_add_or_sub_interpolate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device cuda\n"
     ]
    }
   ],
   "source": [
    "seed = 1\n",
    "torch.manual_seed(seed)\n",
    "device = torch.device(\"cuda\")\n",
    "print(\"device\", device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Transformer(\n",
    "  n_src_vocab=VOCAB_SZ + 1, # add PAD in vocabulary\n",
    "  n_tgt_vocab=VOCAB_SZ + 1, # add PAD in vocabulary\n",
    "  len_max_seq_encoder=MAX_QUESTION_SZ,\n",
    "  len_max_seq_decoder=MAX_ANSWER_SZ,\n",
    ")\n",
    "\n",
    "exp_name = \"math_ds_arithmetic_add_or_sub_hard\"\n",
    "unique_id = \"20190423_1800\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(model.parameters(), lr=6e-6, betas=(0.9, 0.995), eps=1e-9)\n",
    "        #filter(lambda x: x.requires_grad, model.parameters()),\n",
    "        #lr=6e-4, betas=(0.9, 0.995), eps=1e-9)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_ds, val_ds = random_split_dataset(list(islice(ds_arithmetic_add_or_sub_easy, 128*100)), split_rate=0.9)\n",
    "train_ds, val_ds = math_dataset.random_split_dataset(ds_arithmetic_add_or_sub_hard, split_rate=0.9)\n",
    "train_loader = data.DataLoader(\n",
    "    train_ds, batch_size=128, shuffle=True, num_workers=4,\n",
    "    collate_fn=question_answer_to_position_batch_collate_fn)\n",
    "\n",
    "val_loader = data.DataLoader(\n",
    "    val_ds, batch_size=128, shuffle=False, num_workers=4,\n",
    "    collate_fn=question_answer_to_position_batch_collate_fn)\n",
    "\n",
    "interpolate_loader = data.DataLoader(\n",
    "    ds_arithmetic_add_or_sub_interpolate, batch_size=128, shuffle=False, num_workers=4,\n",
    "    collate_fn=question_answer_to_position_batch_collate_fn)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting state from checkpoints/math_ds_arithmetic_add_or_sub_easy_2019-04-22T13:32:24_validation_best.pth\n",
      "acc 0.9608616416082527\n",
      "loss 0.171765097012516\n",
      "exp_name math_ds_arithmetic_add_or_sub_easy\n",
      "unique_id 2019-04-22T13:32:24\n",
      "epoch 13\n",
      "best_acc 0.9608616416082527\n",
      "best_loss 0.171765097012516\n"
     ]
    }
   ],
   "source": [
    "from checkpoints import restore_checkpoint\n",
    "\n",
    "state = restore_checkpoint(model, optimizer, \"checkpoints/math_ds_arithmetic_add_or_sub_easy_2019-04-22T13:32:24_validation_best.pth\")\n",
    "exp_name = state[\"exp_name\"]\n",
    "unique_id = state[\"unique_id\"]\n",
    "model = state[\"model\"]\n",
    "optimizer = state[\"optimizer\"]\n",
    "epoch = state[\"epoch\"]\n",
    "best_acc = state[\"acc\"]\n",
    "best_loss = state[\"loss\"]\n",
    "\n",
    "print(\"exp_name\", exp_name)\n",
    "print(\"unique_id\", unique_id)\n",
    "print(\"epoch\", epoch)\n",
    "print(\"best_acc\", best_acc)\n",
    "print(\"best_loss\", best_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = model.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing TensorBoard events locally to runs/math_ds_arithmetic_add_or_sub_hard_20190423_1800\n"
     ]
    }
   ],
   "source": [
    "tb = Tensorboard(exp_name, unique_name=unique_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 0 ]\n",
      "Adding group train to writers (dict_keys([]))\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 2.6261098524396793, ppl:  13.81990, accuracy: 26.954 %, elapse: 1326301.921ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding group eval to writers (dict_keys(['train']))\n",
      "[Validation]  loss: 2.009183946017596,  ppl:  7.45723, accuracy: 32.786 %, elapse: 59049.417ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding group interpolate to writers (dict_keys(['train', 'eval']))\n",
      "[Interpolate]  loss: 1.9652058213868955,  ppl:  7.13638, accuracy: 34.904 %, elapse: 8521.743ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 1 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 2.4193844069497454, ppl:  11.23894, accuracy: 34.804 %, elapse: 1330191.011ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 1.6934560433494328,  ppl:  5.43824, accuracy: 42.877 %, elapse: 59087.733ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 1.629926490597652,  ppl:  5.10350, accuracy: 45.579 %, elapse: 8520.825ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 2 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 2.2095352993943713, ppl:  9.11148, accuracy: 43.102 %, elapse: 1329812.416ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 1.4383366662863577,  ppl:  4.21368, accuracy: 51.893 %, elapse: 59110.307ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 1.3729854025081536,  ppl:  3.94712, accuracy: 54.870 %, elapse: 8493.141ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 3 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 2.0523151830481385, ppl:  7.78591, accuracy: 49.178 %, elapse: 1329342.120ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 1.2768513429233892,  ppl:  3.58533, accuracy: 57.191 %, elapse: 59105.786ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 1.2048870047649358,  ppl:  3.33638, accuracy: 60.082 %, elapse: 8516.227ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 4 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 4295/4688 [20:16<01:51,  3.52it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 1.642367266533072, ppl:  5.16739, accuracy: 64.373 %, elapse: 1328794.714ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 0.8363425628310155,  ppl:  2.30791, accuracy: 71.465 %, elapse: 59025.042ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 0.7751030592908736,  ppl:  2.17082, accuracy: 73.799 %, elapse: 8507.756ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 9 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 1.5750303601423803, ppl:  4.83089, accuracy: 67.280 %, elapse: 1328182.366ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 0.7438201703372517,  ppl:  2.10396, accuracy: 75.002 %, elapse: 58996.402ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 0.6848689156538703,  ppl:  1.98351, accuracy: 77.031 %, elapse: 8509.625ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 10 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 1.5016073457229309, ppl:  4.48890, accuracy: 70.429 %, elapse: 1328137.709ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 0.681095479055235,  ppl:  1.97604, accuracy: 77.133 %, elapse: 58999.226ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 0.6350420471742464,  ppl:  1.88710, accuracy: 78.705 %, elapse: 8518.597ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 11 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 1.4369096315441388, ppl:  4.20767, accuracy: 73.165 %, elapse: 1328068.570ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 0.612573622797567,  ppl:  1.84517, accuracy: 79.411 %, elapse: 59014.692ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 0.5709221237510936,  ppl:  1.76990, accuracy: 80.800 %, elapse: 8514.115ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 12 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/521 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Training]  loss: 1.37111557850934, ppl:  3.93974, accuracy: 75.920 %, elapse: 1327493.639ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Validation]  loss: 0.5606734197226444,  ppl:  1.75185, accuracy: 81.278 %, elapse: 59029.054ms\n",
      "Checkpointing Validation Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Interpolate]  loss: 0.5205480797488986,  ppl:  1.68295, accuracy: 82.621 %, elapse: 8505.996ms\n",
      "Checkpointing Interpolate Model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/4688 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ Epoch 13 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▎  | 3454/4688 [16:19<05:50,  3.52it/s]"
     ]
    }
   ],
   "source": [
    "model_process.train(\n",
    "    exp_name, unique_id,\n",
    "    model, \n",
    "    train_loader, val_loader, interpolate_loader,\n",
    "    optimizer, device,\n",
    "    epochs=100, tb=tb, log_interval=100,\n",
    "    #start_epoch=epoch+1, best_valid_accu=best_acc, best_valid_loss=best_loss\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch_qs tensor([[ 2, 88, 73, 66, 85,  1, 74, 84,  1, 18, 17,  1, 12,  1, 20, 19, 32,  3]])\n",
      "batch_qs_pos tensor([[ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17, 18]])\n",
      "4\u0000\u0000\u0000\u0000\u0000\u0000\u00002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\"\u0000\u0000\u0000\u0000\u0000\u0000\u0000\n",
      "3\u0000\u0000\u0000\u0000\u0000\u0000\u00002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\"\u0000\u0000\u0000\u0000\u0000\u0000\u0000\n"
     ]
    }
   ],
   "source": [
    "from transformer.Generator import Generator\n",
    "from math_dataset import np_encode_string\n",
    "\n",
    "generator = Generator(model, device, beam_size=5, n_best=1, max_token_seq_len=MAX_QUESTION_SZ)\n",
    "\n",
    "#generator.generate_batch()\n",
    "pred = [\"what is 10 + 32?\"]\n",
    "pred = list(map(lambda q: np_encode_string(q), pred))\n",
    "pred_loader = data.DataLoader(pred, batch_size=1, shuffle=False, num_workers=1, collate_fn=question_to_position_batch_collate_fn)\n",
    "\n",
    "model_process.predict(generator, pred_loader, device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AxFzs2b8DCNw"
   },
   "outputs": [],
   "source": [
    "'''A wrapper class for optimizer '''\n",
    "import numpy as np\n",
    "\n",
    "class ScheduledOptim():\n",
    "    '''A simple wrapper class for learning rate scheduling'''\n",
    "\n",
    "    def __init__(self, optimizer, d_model, n_warmup_steps):\n",
    "        self._optimizer = optimizer\n",
    "        self.n_warmup_steps = n_warmup_steps\n",
    "        self.n_current_steps = 0\n",
    "        self.init_lr = np.power(d_model, -0.5)\n",
    "\n",
    "    def step_and_update_lr(self):\n",
    "        \"Step with the inner optimizer\"\n",
    "        self._update_learning_rate()\n",
    "        self._optimizer.step()\n",
    "\n",
    "    def zero_grad(self):\n",
    "        \"Zero out the gradients by the inner optimizer\"\n",
    "        self._optimizer.zero_grad()\n",
    "\n",
    "    def _get_lr_scale(self):\n",
    "        return np.min([\n",
    "            np.power(self.n_current_steps, -0.5),\n",
    "            np.power(self.n_warmup_steps, -1.5) * self.n_current_steps])\n",
    "\n",
    "    def _update_learning_rate(self):\n",
    "        ''' Learning rate scheduling per step '''\n",
    "\n",
    "        self.n_current_steps += 1\n",
    "        lr = self.init_lr * self._get_lr_scale()\n",
    "\n",
    "        for param_group in self._optimizer.param_groups:\n",
    "            param_group['lr'] = lr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = {\n",
    "    \"exp_name\": \"math_ds_arithmetic_add_or_sub_easy\",\n",
    "    \"unique_id\": \"2019-04-22T13:32:24\",\n",
    "    \"type\": \"train\",\n",
    "    \"model\": model.state_dict(),\n",
    "    \"optimizer\": optimizer.state_dict(),\n",
    "    \"acc\": 0.9319619128828253,\n",
    "    \"loss\": 0.2518192112851183,\n",
    "    \"epoch\": 10,\n",
    "}\n",
    "torch.save(state, \"checkpoints/math_ds_arithmetic_add_or_sub_easy_2019-04-22T13:32:24_validation_best.pth\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rotating_checkpoint import restore_checkpoint\n",
    "\n",
    "state = restore_checkpoint(model, optimizer, \"math_ds_arithmetic_add_or_sub_easy_2019-04-22T13:32:24.pth\")\n",
    "print(state)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "mathematics_dataset_transformer.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}