got-book-generator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Corpus\n",
    "##### Get book names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 5 books\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "book_filenames = sorted(glob.glob(\"/data/*.txt\"))\n",
    "\n",
    "print(\"Found {} books\".format(len(book_filenames)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Combine books into a string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus is 9719485 characters long\n"
     ]
    }
   ],
   "source": [
    "import codecs\n",
    "\n",
    "corpus_raw = u\"\"\n",
    "for filename in book_filenames:\n",
    "    with codecs.open(filename, 'r', 'utf-8') as book_file:\n",
    "        corpus_raw += book_file.read()\n",
    "\n",
    "print(\"Corpus is {} characters long\".format(len(corpus_raw)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Corpus\n",
    "##### Create lookup tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_lookup_tables(text):\n",
    "    \"\"\"\n",
    "    Create lookup tables for vocab\n",
    "    :param text: The GOT text split into words\n",
    "    :return: A tuple of dicts (vocab_to_int, int_to_vocab)\n",
    "    \"\"\"\n",
    "    vocab = set(text)\n",
    "    int_to_vocab = {key: word for key, word in enumerate(vocab)}\n",
    "    vocab_to_int = {word: key for key, word in enumerate(vocab)}\n",
    "    return vocab_to_int, int_to_vocab\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Tokenize punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def token_lookup():\n",
    "    \"\"\"\n",
    "    Generate a dict to map punctuation into a token\n",
    "    :return: dictionary mapping puncuation to token\n",
    "    \"\"\"\n",
    "    return {\n",
    "        '.': '||period||',\n",
    "        ',': '||comma||',\n",
    "        '\"': '||quotes||',\n",
    "        ';': '||semicolon||',\n",
    "        '!': '||exclamation-mark||',\n",
    "        '?': '||question-mark||',\n",
    "        '(': '||left-parentheses||',\n",
    "        ')': '||right-parentheses||',\n",
    "        '--': '||emm-dash||',\n",
    "        '\\n': '||return||'\n",
    "        \n",
    "    }\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Process and save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "token_dict = token_lookup()\n",
    "for token, replacement in token_dict.items():\n",
    "    corpus_raw = corpus_raw.replace(token, ' {} '.format(replacement))\n",
    "corpus_raw = corpus_raw.lower()\n",
    "corpus_raw = corpus_raw.split()\n",
    "\n",
    "vocab_to_int, int_to_vocab = create_lookup_tables(corpus_raw)\n",
    "corpus_int = [vocab_to_int[word] for word in corpus_raw]\n",
    "pickle.dump((corpus_int, vocab_to_int, int_to_vocab, token_dict), open('preprocess.p', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the Network\n",
    "### Batch the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batches(int_text, batch_size, seq_length):\n",
    "    \"\"\"\n",
    "    Return batches of input and target data\n",
    "    :param int_text: text with words replaced by their ids\n",
    "    :param batch_size: the size that each batch of data should be\n",
    "    :param seq_length: the length of each sequence\n",
    "    :return: batches of data as a numpy array\n",
    "    \"\"\"\n",
    "    words_per_batch = batch_size * seq_length\n",
    "    num_batches = len(int_text)//words_per_batch\n",
    "    int_text = int_text[:num_batches*words_per_batch]\n",
    "    y = np.array(int_text[1:] + [int_text[0]])\n",
    "    x = np.array(int_text)\n",
    "    \n",
    "    x_batches = np.split(x.reshape(batch_size, -1), num_batches, axis=1)\n",
    "    y_batches = np.split(y.reshape(batch_size, -1), num_batches, axis=1)\n",
    "    \n",
    "    batch_data = list(zip(x_batches, y_batches))\n",
    "    \n",
    "    return np.array(batch_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 10000\n",
    "batch_size = 512\n",
    "rnn_size = 512\n",
    "num_layers = 3\n",
    "keep_prob = 0.7\n",
    "embed_dim = 512\n",
    "seq_length = 30\n",
    "learning_rate = 0.001\n",
    "save_dir = './save'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build the Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_graph = tf.Graph()\n",
    "with train_graph.as_default():    \n",
    "    \n",
    "    # Initialize input placeholders\n",
    "    input_text = tf.placeholder(tf.int32, [None, None], name='input')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    lr = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    \n",
    "    # Calculate text attributes\n",
    "    vocab_size = len(int_to_vocab)\n",
    "    input_text_shape = tf.shape(input_text)\n",
    "    \n",
    "    # Build the RNN cell\n",
    "    lstm = tf.contrib.rnn.BasicLSTMCell(num_units=rnn_size)\n",
    "    drop_cell = tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=keep_prob)\n",
    "    cell = tf.contrib.rnn.MultiRNNCell([drop_cell] * num_layers)\n",
    "    \n",
    "    # Set the initial state\n",
    "    initial_state = cell.zero_state(input_text_shape[0], tf.float32)\n",
    "    initial_state = tf.identity(initial_state, name='initial_state')\n",
    "    \n",
    "    # Create word embedding as input to RNN\n",
    "    embed = tf.contrib.layers.embed_sequence(input_text, vocab_size, embed_dim)\n",
    "    \n",
    "    # Build RNN\n",
    "    outputs, final_state = tf.nn.dynamic_rnn(cell, embed, dtype=tf.float32)\n",
    "    final_state = tf.identity(final_state, name='final_state')\n",
    "    \n",
    "    # Take RNN output and make logits\n",
    "    logits = tf.contrib.layers.fully_connected(outputs, vocab_size, activation_fn=None)\n",
    "    \n",
    "    # Calculate the probability of generating each word\n",
    "    probs = tf.nn.softmax(logits, name='probs')\n",
    "    \n",
    "    # Define loss function\n",
    "    cost = tf.contrib.seq2seq.sequence_loss(\n",
    "        logits,\n",
    "        targets,\n",
    "        tf.ones([input_text_shape[0], input_text_shape[1]])\n",
    "    )\n",
    "    \n",
    "    # Learning rate optimizer\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    \n",
    "    # Gradient clipping to avoid exploding gradients\n",
    "    gradients = optimizer.compute_gradients(cost)\n",
    "    capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]\n",
    "    train_op = optimizer.apply_gradients(capped_gradients)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   1 Batch  144/144   train_loss = 6.483   time_elapsed = 208.531   time_remaining = 2085101\n",
      "Model Trained and Saved\n",
      "Epoch   2 Batch  144/144   train_loss = 6.469   time_elapsed = 420.586   time_remaining = 2102508\n",
      "Epoch   3 Batch  144/144   train_loss = 6.465   time_elapsed = 628.625   time_remaining = 2094788\n",
      "Epoch   4 Batch  144/144   train_loss = 6.466   time_elapsed = 836.495   time_remaining = 2090401\n",
      "Epoch   5 Batch  144/144   train_loss = 6.466   time_elapsed = 1044.271   time_remaining = 2087497\n",
      "Epoch   6 Batch  144/144   train_loss = 6.466   time_elapsed = 1251.556   time_remaining = 2084675\n",
      "Epoch   7 Batch  144/144   train_loss = 6.464   time_elapsed = 1458.474   time_remaining = 2082076\n",
      "Epoch   8 Batch  144/144   train_loss = 6.462   time_elapsed = 1665.455   time_remaining = 2080153\n",
      "Epoch   9 Batch  144/144   train_loss = 6.338   time_elapsed = 1872.648   time_remaining = 2078847\n",
      "Epoch  10 Batch  144/144   train_loss = 6.278   time_elapsed = 2080.062   time_remaining = 2077982\n",
      "Epoch  11 Batch  144/144   train_loss = 6.246   time_elapsed = 2287.501   time_remaining = 2077259\n",
      "Model Trained and Saved\n",
      "Epoch  12 Batch  144/144   train_loss = 6.218   time_elapsed = 2499.131   time_remaining = 2080110\n",
      "Epoch  13 Batch  144/144   train_loss = 6.153   time_elapsed = 2706.696   time_remaining = 2079368\n",
      "Epoch  14 Batch  144/144   train_loss = 6.024   time_elapsed = 2914.182   time_remaining = 2078644\n",
      "Epoch  15 Batch  144/144   train_loss = 5.915   time_elapsed = 3122.004   time_remaining = 2078214\n",
      "Epoch  16 Batch  144/144   train_loss = 5.836   time_elapsed = 3329.697   time_remaining = 2077731\n",
      "Epoch  17 Batch  144/144   train_loss = 5.755   time_elapsed = 3537.443   time_remaining = 2077312\n",
      "Epoch  18 Batch  144/144   train_loss = 5.649   time_elapsed = 3745.209   time_remaining = 2076926\n",
      "Epoch  19 Batch  144/144   train_loss = 5.577   time_elapsed = 3953.085   time_remaining = 2076618\n",
      "Epoch  20 Batch  144/144   train_loss = 5.514   time_elapsed = 4160.767   time_remaining = 2076223\n",
      "Epoch  21 Batch  144/144   train_loss = 5.435   time_elapsed = 4368.896   time_remaining = 2076058\n",
      "Model Trained and Saved\n",
      "Epoch  22 Batch  144/144   train_loss = 5.339   time_elapsed = 4581.231   time_remaining = 2077797\n",
      "Epoch  23 Batch  144/144   train_loss = 5.216   time_elapsed = 4790.114   time_remaining = 2077868\n",
      "Epoch  24 Batch  144/144   train_loss = 5.102   time_elapsed = 4999.074   time_remaining = 2077948\n",
      "Epoch  25 Batch  144/144   train_loss = 4.989   time_elapsed = 5208.321   time_remaining = 2078120\n",
      "Epoch  26 Batch  144/144   train_loss = 4.900   time_elapsed = 5417.591   time_remaining = 2078271\n",
      "Epoch  27 Batch  144/144   train_loss = 4.816   time_elapsed = 5626.879   time_remaining = 2078402\n",
      "Epoch  28 Batch  144/144   train_loss = 4.739   time_elapsed = 5836.327   time_remaining = 2078566\n",
      "Epoch  29 Batch  144/144   train_loss = 4.688   time_elapsed = 6045.897   time_remaining = 2078746\n",
      "Epoch  30 Batch  144/144   train_loss = 4.641   time_elapsed = 6255.412   time_remaining = 2078882\n",
      "Epoch  31 Batch  144/144   train_loss = 4.583   time_elapsed = 6464.829   time_remaining = 2078964\n",
      "Model Trained and Saved\n",
      "Epoch  32 Batch  144/144   train_loss = 4.538   time_elapsed = 6677.942   time_remaining = 2080179\n",
      "Epoch  33 Batch  144/144   train_loss = 4.492   time_elapsed = 6887.259   time_remaining = 2080161\n",
      "Epoch  34 Batch  144/144   train_loss = 4.464   time_elapsed = 7096.374   time_remaining = 2080072\n",
      "Epoch  35 Batch  144/144   train_loss = 4.424   time_elapsed = 7305.310   time_remaining = 2079926\n",
      "Epoch  36 Batch  144/144   train_loss = 4.389   time_elapsed = 7514.219   time_remaining = 2079769\n",
      "Epoch  37 Batch  144/144   train_loss = 4.353   time_elapsed = 7723.138   time_remaining = 2079612\n",
      "Epoch  38 Batch  144/144   train_loss = 4.316   time_elapsed = 7931.790   time_remaining = 2079381\n",
      "Epoch  39 Batch  144/144   train_loss = 4.287   time_elapsed = 8140.435   time_remaining = 2079151\n",
      "Epoch  40 Batch  144/144   train_loss = 4.255   time_elapsed = 8348.987   time_remaining = 2078898\n",
      "Epoch  41 Batch  144/144   train_loss = 4.236   time_elapsed = 8557.384   time_remaining = 2078610\n",
      "Model Trained and Saved\n",
      "Epoch  42 Batch  144/144   train_loss = 4.200   time_elapsed = 8769.655   time_remaining = 2079243\n",
      "Epoch  43 Batch  144/144   train_loss = 4.175   time_elapsed = 8978.079   time_remaining = 2078947\n",
      "Epoch  44 Batch  144/144   train_loss = 4.152   time_elapsed = 9186.577   time_remaining = 2078672\n",
      "Epoch  45 Batch  144/144   train_loss = 4.125   time_elapsed = 9394.589   time_remaining = 2078292\n",
      "Epoch  46 Batch  144/144   train_loss = 4.108   time_elapsed = 9602.828   time_remaining = 2077968\n",
      "Epoch  47 Batch  144/144   train_loss = 4.091   time_elapsed = 9811.073   time_remaining = 2077651\n",
      "Epoch  48 Batch  144/144   train_loss = 4.064   time_elapsed = 10019.210   time_remaining = 2077316\n",
      "Epoch  49 Batch  144/144   train_loss = 4.050   time_elapsed = 10227.407   time_remaining = 2076999\n",
      "Epoch  50 Batch  144/144   train_loss = 4.018   time_elapsed = 10435.675   time_remaining = 2076699\n",
      "Epoch  51 Batch  144/144   train_loss = 3.999   time_elapsed = 10643.610   time_remaining = 2076339\n",
      "Model Trained and Saved\n",
      "Epoch  52 Batch  144/144   train_loss = 3.984   time_elapsed = 10855.407   time_remaining = 2076723\n",
      "Epoch  53 Batch  144/144   train_loss = 3.957   time_elapsed = 11063.425   time_remaining = 2076375\n",
      "Epoch  54 Batch  144/144   train_loss = 3.948   time_elapsed = 11271.448   time_remaining = 2076034\n",
      "Epoch  55 Batch  144/144   train_loss = 3.918   time_elapsed = 11479.563   time_remaining = 2075714\n",
      "Epoch  56 Batch  144/144   train_loss = 3.904   time_elapsed = 11687.248   time_remaining = 2075321\n",
      "Epoch  57 Batch  144/144   train_loss = 3.878   time_elapsed = 11895.205   time_remaining = 2074983\n",
      "Epoch  58 Batch  144/144   train_loss = 3.864   time_elapsed = 12103.136   time_remaining = 2074644\n",
      "Epoch  59 Batch  144/144   train_loss = 3.849   time_elapsed = 12310.929   time_remaining = 2074287\n",
      "Epoch  60 Batch  144/144   train_loss = 3.828   time_elapsed = 12518.848   time_remaining = 2073956\n",
      "Epoch  61 Batch  144/144   train_loss = 3.803   time_elapsed = 12726.708   time_remaining = 2073619\n",
      "Model Trained and Saved\n",
      "Epoch  62 Batch  144/144   train_loss = 3.801   time_elapsed = 12938.315   time_remaining = 2073887\n",
      "Epoch  63 Batch  144/144   train_loss = 3.770   time_elapsed = 13146.131   time_remaining = 2073541\n",
      "Epoch  64 Batch  144/144   train_loss = 3.753   time_elapsed = 13353.953   time_remaining = 2073201\n",
      "Epoch  65 Batch  144/144   train_loss = 3.739   time_elapsed = 13561.733   time_remaining = 2072859\n",
      "Epoch  66 Batch  144/144   train_loss = 3.731   time_elapsed = 13769.380   time_remaining = 2072500\n",
      "Epoch  67 Batch  144/144   train_loss = 3.716   time_elapsed = 13976.714   time_remaining = 2072100\n",
      "Epoch  68 Batch  144/144   train_loss = 3.705   time_elapsed = 14184.263   time_remaining = 2071737\n",
      "Epoch  69 Batch  144/144   train_loss = 3.726   time_elapsed = 14391.824   time_remaining = 2071380\n",
      "Epoch  70 Batch  144/144   train_loss = 3.686   time_elapsed = 14599.426   time_remaining = 2071033\n",
      "Epoch  71 Batch  144/144   train_loss = 3.662   time_elapsed = 14807.215   time_remaining = 2070716\n",
      "Model Trained and Saved\n",
      "Epoch  72 Batch  144/144   train_loss = 3.659   time_elapsed = 15018.714   time_remaining = 2070914\n",
      "Epoch  73 Batch  144/144   train_loss = 3.651   time_elapsed = 15226.117   time_remaining = 2070543\n",
      "Epoch  74 Batch  144/144   train_loss = 3.623   time_elapsed = 15433.349   time_remaining = 2070154\n",
      "Epoch  75 Batch  144/144   train_loss = 3.614   time_elapsed = 15640.606   time_remaining = 2069774\n",
      "Epoch  76 Batch  144/144   train_loss = 3.597   time_elapsed = 15847.847   time_remaining = 2069395\n",
      "Epoch  77 Batch  144/144   train_loss = 3.597   time_elapsed = 16055.219   time_remaining = 2069038\n",
      "Epoch  78 Batch  144/144   train_loss = 3.587   time_elapsed = 16262.589   time_remaining = 2068685\n",
      "Epoch  79 Batch  144/144   train_loss = 3.575   time_elapsed = 16469.955   time_remaining = 2068334\n",
      "Epoch  80 Batch  144/144   train_loss = 3.559   time_elapsed = 16677.201   time_remaining = 2067973\n",
      "Epoch  81 Batch  144/144   train_loss = 3.581   time_elapsed = 16884.212   time_remaining = 2067586\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Trained and Saved\n",
      "Epoch  82 Batch  144/144   train_loss = 3.545   time_elapsed = 17095.380   time_remaining = 2067707\n",
      "Epoch  83 Batch  144/144   train_loss = 3.523   time_elapsed = 17302.661   time_remaining = 2067355\n",
      "Epoch  84 Batch  144/144   train_loss = 3.510   time_elapsed = 17510.039   time_remaining = 2067018\n",
      "Epoch  85 Batch  144/144   train_loss = 3.507   time_elapsed = 17717.511   time_remaining = 2066696\n",
      "Epoch  86 Batch  144/144   train_loss = 3.487   time_elapsed = 17924.611   time_remaining = 2066333\n",
      "Epoch  87 Batch  144/144   train_loss = 3.482   time_elapsed = 18131.600   time_remaining = 2065960\n",
      "Epoch  88 Batch  144/144   train_loss = 3.465   time_elapsed = 18338.684   time_remaining = 2065603\n",
      "Epoch  89 Batch  144/144   train_loss = 3.454   time_elapsed = 18545.614   time_remaining = 2065231\n",
      "Epoch  90 Batch  144/144   train_loss = 3.443   time_elapsed = 18752.711   time_remaining = 2064882\n",
      "Epoch  91 Batch  144/144   train_loss = 3.439   time_elapsed = 18959.752   time_remaining = 2064529\n",
      "Model Trained and Saved\n",
      "Epoch  92 Batch  144/144   train_loss = 3.456   time_elapsed = 19170.922   time_remaining = 2064625\n",
      "Epoch  93 Batch  144/144   train_loss = 3.471   time_elapsed = 19377.979   time_remaining = 2064276\n",
      "Epoch  94 Batch  144/144   train_loss = 3.437   time_elapsed = 19585.009   time_remaining = 2063927\n",
      "Epoch  95 Batch  144/144   train_loss = 3.413   time_elapsed = 19792.066   time_remaining = 2063583\n",
      "Epoch  96 Batch  144/144   train_loss = 3.411   time_elapsed = 19999.016   time_remaining = 2063232\n",
      "Epoch  97 Batch  144/144   train_loss = 3.393   time_elapsed = 20206.030   time_remaining = 2062890\n",
      "Epoch  98 Batch  144/144   train_loss = 3.378   time_elapsed = 20413.042   time_remaining = 2062550\n",
      "Epoch  99 Batch  144/144   train_loss = 3.357   time_elapsed = 20619.997   time_remaining = 2062208\n",
      "Epoch 100 Batch  144/144   train_loss = 3.353   time_elapsed = 20827.073   time_remaining = 2061880\n",
      "Epoch 101 Batch  144/144   train_loss = 3.350   time_elapsed = 21034.111   time_remaining = 2061551\n",
      "Model Trained and Saved\n",
      "Epoch 102 Batch  144/144   train_loss = 3.337   time_elapsed = 21244.963   time_remaining = 2061595\n",
      "Epoch 103 Batch  144/144   train_loss = 3.330   time_elapsed = 21451.982   time_remaining = 2061265\n",
      "Epoch 104 Batch  144/144   train_loss = 3.334   time_elapsed = 21658.994   time_remaining = 2060937\n",
      "Epoch 105 Batch  144/144   train_loss = 3.328   time_elapsed = 21865.921   time_remaining = 2060603\n",
      "Epoch 106 Batch  144/144   train_loss = 3.322   time_elapsed = 22072.877   time_remaining = 2060274\n",
      "Epoch 107 Batch  144/144   train_loss = 3.305   time_elapsed = 22279.647   time_remaining = 2059930\n",
      "Epoch 108 Batch  144/144   train_loss = 3.315   time_elapsed = 22486.519   time_remaining = 2059599\n",
      "Epoch 109 Batch  144/144   train_loss = 3.295   time_elapsed = 22693.535   time_remaining = 2059282\n",
      "Epoch 110 Batch  144/144   train_loss = 3.269   time_elapsed = 22900.605   time_remaining = 2058973\n",
      "Epoch 111 Batch  144/144   train_loss = 3.254   time_elapsed = 23107.580   time_remaining = 2058656\n",
      "Model Trained and Saved\n",
      "Epoch 112 Batch  144/144   train_loss = 3.246   time_elapsed = 23318.148   time_remaining = 2058659\n",
      "Epoch 113 Batch  144/144   train_loss = 3.243   time_elapsed = 23524.802   time_remaining = 2058316\n",
      "Epoch 114 Batch  144/144   train_loss = 3.227   time_elapsed = 23731.551   time_remaining = 2057983\n",
      "Epoch 115 Batch  144/144   train_loss = 3.220   time_elapsed = 23938.114   time_remaining = 2057637\n",
      "Epoch 116 Batch  144/144   train_loss = 3.227   time_elapsed = 24144.743   time_remaining = 2057299\n",
      "Epoch 117 Batch  144/144   train_loss = 3.224   time_elapsed = 24351.610   time_remaining = 2056983\n",
      "Epoch 118 Batch  144/144   train_loss = 3.206   time_elapsed = 24558.415   time_remaining = 2056663\n",
      "Epoch 119 Batch  144/144   train_loss = 3.205   time_elapsed = 24765.222   time_remaining = 2056346\n",
      "Epoch 120 Batch  144/144   train_loss = 3.196   time_elapsed = 24971.733   time_remaining = 2056006\n",
      "Epoch 121 Batch  144/144   train_loss = 3.207   time_elapsed = 25178.341   time_remaining = 2055676\n",
      "Model Trained and Saved\n",
      "Epoch 122 Batch  144/144   train_loss = 3.196   time_elapsed = 25389.218   time_remaining = 2055694\n",
      "Epoch 123 Batch  144/144   train_loss = 3.183   time_elapsed = 25596.201   time_remaining = 2055396\n",
      "Epoch 124 Batch  144/144   train_loss = 3.194   time_elapsed = 25803.147   time_remaining = 2055096\n",
      "Epoch 125 Batch  144/144   train_loss = 3.189   time_elapsed = 26009.737   time_remaining = 2054769\n",
      "Epoch 126 Batch  144/144   train_loss = 3.180   time_elapsed = 26216.853   time_remaining = 2054486\n",
      "Epoch 127 Batch  144/144   train_loss = 3.174   time_elapsed = 26423.665   time_remaining = 2054180\n",
      "Epoch 128 Batch  144/144   train_loss = 3.164   time_elapsed = 26630.344   time_remaining = 2053865\n",
      "Epoch 129 Batch  144/144   train_loss = 3.154   time_elapsed = 26837.301   time_remaining = 2053574\n",
      "Epoch 130 Batch  144/144   train_loss = 3.147   time_elapsed = 27043.873   time_remaining = 2053254\n",
      "Epoch 131 Batch  144/144   train_loss = 3.145   time_elapsed = 27250.573   time_remaining = 2052946\n",
      "Model Trained and Saved\n",
      "Epoch 132 Batch  144/144   train_loss = 3.148   time_elapsed = 27461.507   time_remaining = 2052956\n",
      "Epoch 133 Batch  144/144   train_loss = 3.134   time_elapsed = 27668.121   time_remaining = 2052642\n",
      "Epoch 134 Batch  144/144   train_loss = 3.152   time_elapsed = 27874.597   time_remaining = 2052319\n",
      "Epoch 135 Batch  144/144   train_loss = 3.134   time_elapsed = 28081.228   time_remaining = 2052010\n",
      "Epoch 136 Batch  144/144   train_loss = 3.131   time_elapsed = 28287.923   time_remaining = 2051706\n",
      "Epoch 137 Batch  144/144   train_loss = 3.126   time_elapsed = 28494.570   time_remaining = 2051401\n",
      "Epoch 138 Batch  144/144   train_loss = 3.099   time_elapsed = 28701.484   time_remaining = 2051116\n",
      "Epoch 139 Batch  144/144   train_loss = 3.111   time_elapsed = 28908.196   time_remaining = 2050818\n",
      "Epoch 140 Batch  144/144   train_loss = 3.090   time_elapsed = 29114.786   time_remaining = 2050513\n",
      "Epoch 141 Batch  144/144   train_loss = 3.079   time_elapsed = 29321.640   time_remaining = 2050227\n",
      "Model Trained and Saved\n",
      "Epoch 142 Batch  144/144   train_loss = 3.086   time_elapsed = 29532.426   time_remaining = 2050216\n",
      "Epoch 143 Batch  144/144   train_loss = 3.070   time_elapsed = 29739.063   time_remaining = 2049916\n",
      "Epoch 144 Batch  144/144   train_loss = 3.064   time_elapsed = 29945.849   time_remaining = 2049627\n",
      "Epoch 145 Batch  144/144   train_loss = 3.046   time_elapsed = 30152.645   time_remaining = 2049340\n",
      "Epoch 146 Batch  144/144   train_loss = 3.049   time_elapsed = 30359.442   time_remaining = 2049054\n",
      "Epoch 147 Batch  144/144   train_loss = 3.038   time_elapsed = 30566.116   time_remaining = 2048761\n",
      "Epoch 148 Batch  144/144   train_loss = 3.025   time_elapsed = 30772.667   time_remaining = 2048462\n",
      "Epoch 149 Batch  144/144   train_loss = 3.039   time_elapsed = 30979.358   time_remaining = 2048172\n",
      "Epoch 150 Batch  144/144   train_loss = 3.031   time_elapsed = 31186.024   time_remaining = 2047882\n",
      "Epoch 151 Batch  144/144   train_loss = 3.023   time_elapsed = 31392.737   time_remaining = 2047596\n",
      "Model Trained and Saved\n",
      "Epoch 152 Batch  144/144   train_loss = 3.039   time_elapsed = 31603.296   time_remaining = 2047561\n",
      "Epoch 153 Batch  144/144   train_loss = 3.080   time_elapsed = 31810.263   time_remaining = 2047292\n",
      "Epoch 154 Batch  144/144   train_loss = 3.084   time_elapsed = 32016.947   time_remaining = 2047006\n",
      "Epoch 155 Batch  144/144   train_loss = 3.064   time_elapsed = 32223.679   time_remaining = 2046723\n",
      "Epoch 156 Batch  144/144   train_loss = 3.052   time_elapsed = 32430.322   time_remaining = 2046436\n",
      "Epoch 157 Batch  144/144   train_loss = 3.051   time_elapsed = 32636.938   time_remaining = 2046149\n",
      "Epoch 158 Batch  144/144   train_loss = 3.039   time_elapsed = 32843.474   time_remaining = 2045857\n",
      "Epoch 159 Batch  144/144   train_loss = 3.036   time_elapsed = 33049.997   time_remaining = 2045566\n",
      "Epoch 160 Batch  144/144   train_loss = 3.003   time_elapsed = 33256.567   time_remaining = 2045279\n",
      "Epoch 161 Batch  144/144   train_loss = 3.001   time_elapsed = 33463.223   time_remaining = 2044998\n",
      "Model Trained and Saved\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 162 Batch  144/144   train_loss = 2.987   time_elapsed = 33673.942   time_remaining = 2044964\n",
      "Epoch 163 Batch  144/144   train_loss = 2.978   time_elapsed = 33880.433   time_remaining = 2044674\n",
      "Epoch 164 Batch  144/144   train_loss = 2.962   time_elapsed = 34086.995   time_remaining = 2044388\n",
      "Epoch 165 Batch  144/144   train_loss = 2.985   time_elapsed = 34293.618   time_remaining = 2044107\n",
      "Epoch 166 Batch  144/144   train_loss = 2.985   time_elapsed = 34500.275   time_remaining = 2043830\n",
      "Epoch 167 Batch  144/144   train_loss = 2.983   time_elapsed = 34707.086   time_remaining = 2043562\n",
      "Epoch 168 Batch  144/144   train_loss = 2.971   time_elapsed = 34913.906   time_remaining = 2043295\n",
      "Epoch 169 Batch  144/144   train_loss = 2.953   time_elapsed = 35120.742   time_remaining = 2043030\n",
      "Epoch 170 Batch  144/144   train_loss = 2.949   time_elapsed = 35327.330   time_remaining = 2042751\n",
      "Epoch 171 Batch  144/144   train_loss = 2.971   time_elapsed = 35534.208   time_remaining = 2042490\n",
      "Model Trained and Saved\n",
      "Epoch 172 Batch  144/144   train_loss = 2.983   time_elapsed = 35745.271   time_remaining = 2042468\n",
      "Epoch 173 Batch  144/144   train_loss = 2.975   time_elapsed = 35951.792   time_remaining = 2042186\n",
      "Epoch 174 Batch  144/144   train_loss = 2.952   time_elapsed = 36158.353   time_remaining = 2041908\n",
      "Epoch 175 Batch  144/144   train_loss = 2.928   time_elapsed = 36364.848   time_remaining = 2041626\n",
      "Epoch 176 Batch  144/144   train_loss = 2.910   time_elapsed = 36571.522   time_remaining = 2041356\n",
      "Epoch 177 Batch  144/144   train_loss = 2.895   time_elapsed = 36778.145   time_remaining = 2041083\n",
      "Epoch 178 Batch  144/144   train_loss = 2.885   time_elapsed = 36984.735   time_remaining = 2040809\n",
      "Epoch 179 Batch  144/144   train_loss = 2.895   time_elapsed = 37191.038   time_remaining = 2040521\n",
      "Epoch 180 Batch  144/144   train_loss = 2.875   time_elapsed = 37397.419   time_remaining = 2040237\n",
      "Epoch 181 Batch  144/144   train_loss = 2.884   time_elapsed = 37603.848   time_remaining = 2039957\n",
      "Model Trained and Saved\n",
      "Epoch 182 Batch  144/144   train_loss = 2.870   time_elapsed = 37814.293   time_remaining = 2039894\n",
      "Epoch 183 Batch  144/144   train_loss = 2.867   time_elapsed = 38020.641   time_remaining = 2039610\n",
      "Epoch 184 Batch  144/144   train_loss = 2.866   time_elapsed = 38227.039   time_remaining = 2039329\n",
      "Epoch 185 Batch  144/144   train_loss = 2.858   time_elapsed = 38433.384   time_remaining = 2039047\n",
      "Epoch 186 Batch  144/144   train_loss = 2.850   time_elapsed = 38639.503   time_remaining = 2038753\n",
      "Epoch 187 Batch  144/144   train_loss = 2.859   time_elapsed = 38845.728   time_remaining = 2038466\n",
      "Epoch 188 Batch  144/144   train_loss = 2.833   time_elapsed = 39051.686   time_remaining = 2038166\n",
      "Epoch 189 Batch  144/144   train_loss = 2.841   time_elapsed = 39258.092   time_remaining = 2037890\n",
      "Epoch 190 Batch  144/144   train_loss = 2.847   time_elapsed = 39464.112   time_remaining = 2037594\n",
      "Epoch 191 Batch  144/144   train_loss = 2.861   time_elapsed = 39670.014   time_remaining = 2037294\n",
      "Model Trained and Saved\n",
      "Epoch 192 Batch  144/144   train_loss = 2.848   time_elapsed = 39880.251   time_remaining = 2037216\n",
      "Epoch 193 Batch  144/144   train_loss = 2.830   time_elapsed = 40086.679   time_remaining = 2036943\n",
      "Epoch 194 Batch  144/144   train_loss = 2.826   time_elapsed = 40293.062   time_remaining = 2036669\n",
      "Epoch 195 Batch  144/144   train_loss = 2.839   time_elapsed = 40499.529   time_remaining = 2036399\n",
      "Epoch 196 Batch  144/144   train_loss = 2.826   time_elapsed = 40705.861   time_remaining = 2036124\n",
      "Epoch 197 Batch  144/144   train_loss = 2.824   time_elapsed = 40912.294   time_remaining = 2035854\n",
      "Epoch 198 Batch  144/144   train_loss = 2.845   time_elapsed = 41119.084   time_remaining = 2035602\n",
      "Epoch 199 Batch  144/144   train_loss = 2.841   time_elapsed = 41326.015   time_remaining = 2035358\n",
      "Epoch 200 Batch  144/144   train_loss = 2.833   time_elapsed = 41532.981   time_remaining = 2035116\n",
      "Epoch 201 Batch  144/144   train_loss = 2.836   time_elapsed = 41739.915   time_remaining = 2034873\n",
      "Model Trained and Saved\n",
      "Epoch 202 Batch  144/144   train_loss = 2.815   time_elapsed = 41951.024   time_remaining = 2034832\n",
      "Epoch 203 Batch  144/144   train_loss = 2.815   time_elapsed = 42158.076   time_remaining = 2034594\n",
      "Epoch 204 Batch  144/144   train_loss = 2.813   time_elapsed = 42365.186   time_remaining = 2034360\n",
      "Epoch 205 Batch  144/144   train_loss = 2.814   time_elapsed = 42572.534   time_remaining = 2034136\n",
      "Epoch 206 Batch  144/144   train_loss = 2.846   time_elapsed = 42779.772   time_remaining = 2033908\n",
      "Epoch 207 Batch  144/144   train_loss = 2.829   time_elapsed = 42987.105   time_remaining = 2033685\n",
      "Epoch 208 Batch  144/144   train_loss = 2.819   time_elapsed = 43194.083   time_remaining = 2033445\n",
      "Epoch 209 Batch  144/144   train_loss = 2.819   time_elapsed = 43401.209   time_remaining = 2033212\n",
      "Epoch 210 Batch  144/144   train_loss = 2.821   time_elapsed = 43608.052   time_remaining = 2032966\n",
      "Epoch 211 Batch  144/144   train_loss = 2.808   time_elapsed = 43815.116   time_remaining = 2032731\n",
      "Model Trained and Saved\n",
      "Epoch 212 Batch  144/144   train_loss = 2.780   time_elapsed = 44025.965   time_remaining = 2032670\n",
      "Epoch 213 Batch  144/144   train_loss = 2.787   time_elapsed = 44232.888   time_remaining = 2032429\n",
      "Epoch 214 Batch  144/144   train_loss = 2.783   time_elapsed = 44439.906   time_remaining = 2032191\n",
      "Epoch 215 Batch  144/144   train_loss = 2.778   time_elapsed = 44646.424   time_remaining = 2031931\n",
      "Epoch 216 Batch  144/144   train_loss = 2.794   time_elapsed = 44853.175   time_remaining = 2031683\n",
      "Epoch 217 Batch  144/144   train_loss = 2.777   time_elapsed = 45059.811   time_remaining = 2031429\n",
      "Epoch 218 Batch  144/144   train_loss = 2.758   time_elapsed = 45266.109   time_remaining = 2031161\n",
      "Epoch 219 Batch  144/144   train_loss = 2.766   time_elapsed = 45472.542   time_remaining = 2030899\n",
      "Epoch 220 Batch  144/144   train_loss = 2.761   time_elapsed = 45678.841   time_remaining = 2030632\n",
      "Epoch 221 Batch  144/144   train_loss = 2.745   time_elapsed = 45884.869   time_remaining = 2030354\n",
      "Model Trained and Saved\n",
      "Epoch 222 Batch  144/144   train_loss = 2.746   time_elapsed = 46095.401   time_remaining = 2030274\n",
      "Epoch 223 Batch  144/144   train_loss = 2.738   time_elapsed = 46301.928   time_remaining = 2030018\n",
      "Epoch 224 Batch  144/144   train_loss = 2.729   time_elapsed = 46508.225   time_remaining = 2029752\n",
      "Epoch 225 Batch  144/144   train_loss = 2.729   time_elapsed = 46714.707   time_remaining = 2029495\n",
      "Epoch 226 Batch  144/144   train_loss = 2.727   time_elapsed = 46920.899   time_remaining = 2029225\n",
      "Epoch 227 Batch  144/144   train_loss = 2.722   time_elapsed = 47126.976   time_remaining = 2028951\n",
      "Epoch 228 Batch  144/144   train_loss = 2.739   time_elapsed = 47333.305   time_remaining = 2028689\n",
      "Epoch 229 Batch  144/144   train_loss = 2.716   time_elapsed = 47539.632   time_remaining = 2028427\n",
      "Epoch 230 Batch  144/144   train_loss = 2.721   time_elapsed = 47745.771   time_remaining = 2028157\n",
      "Epoch 231 Batch  144/144   train_loss = 2.722   time_elapsed = 47952.098   time_remaining = 2027896\n",
      "Model Trained and Saved\n",
      "Epoch 232 Batch  144/144   train_loss = 2.716   time_elapsed = 48162.855   time_remaining = 2027822\n",
      "Epoch 233 Batch  144/144   train_loss = 2.717   time_elapsed = 48369.079   time_remaining = 2027557\n",
      "Epoch 234 Batch  144/144   train_loss = 2.723   time_elapsed = 48575.594   time_remaining = 2027305\n",
      "Epoch 235 Batch  144/144   train_loss = 2.729   time_elapsed = 48782.110   time_remaining = 2027052\n",
      "Epoch 236 Batch  144/144   train_loss = 2.746   time_elapsed = 48988.611   time_remaining = 2026800\n",
      "Epoch 237 Batch  144/144   train_loss = 2.768   time_elapsed = 49195.187   time_remaining = 2026551\n",
      "Epoch 238 Batch  144/144   train_loss = 2.786   time_elapsed = 49401.864   time_remaining = 2026307\n",
      "Epoch 239 Batch  144/144   train_loss = 2.768   time_elapsed = 49608.426   time_remaining = 2026058\n",
      "Epoch 240 Batch  144/144   train_loss = 2.753   time_elapsed = 49814.895   time_remaining = 2025806\n",
      "Epoch 241 Batch  144/144   train_loss = 2.719   time_elapsed = 50021.677   time_remaining = 2025567\n",
      "Model Trained and Saved\n",
      "Epoch 242 Batch  144/144   train_loss = 2.716   time_elapsed = 50233.167   time_remaining = 2025518\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 243 Batch  144/144   train_loss = 2.727   time_elapsed = 50440.176   time_remaining = 2025287\n",
      "Epoch 244 Batch  144/144   train_loss = 2.706   time_elapsed = 50647.217   time_remaining = 2025058\n",
      "Epoch 245 Batch  144/144   train_loss = 2.703   time_elapsed = 50854.223   time_remaining = 2024828\n",
      "Epoch 246 Batch  144/144   train_loss = 2.703   time_elapsed = 51061.237   time_remaining = 2024599\n",
      "Epoch 247 Batch  144/144   train_loss = 2.686   time_elapsed = 51268.173   time_remaining = 2024366\n",
      "Epoch 248 Batch  144/144   train_loss = 2.688   time_elapsed = 51475.259   time_remaining = 2024140\n",
      "Epoch 249 Batch  144/144   train_loss = 2.679   time_elapsed = 51682.328   time_remaining = 2023913\n",
      "Epoch 250 Batch  144/144   train_loss = 2.686   time_elapsed = 51889.375   time_remaining = 2023686\n",
      "Epoch 251 Batch  144/144   train_loss = 2.689   time_elapsed = 52096.357   time_remaining = 2023456\n",
      "Model Trained and Saved\n",
      "Epoch 252 Batch  144/144   train_loss = 2.672   time_elapsed = 52307.264   time_remaining = 2023378\n",
      "Epoch 253 Batch  144/144   train_loss = 2.675   time_elapsed = 52514.259   time_remaining = 2023148\n",
      "Epoch 254 Batch  144/144   train_loss = 2.670   time_elapsed = 52721.287   time_remaining = 2022920\n",
      "Epoch 255 Batch  144/144   train_loss = 2.664   time_elapsed = 52928.299   time_remaining = 2022691\n",
      "Epoch 256 Batch  144/144   train_loss = 2.680   time_elapsed = 53135.359   time_remaining = 2022465\n",
      "Epoch 257 Batch  144/144   train_loss = 2.671   time_elapsed = 53342.348   time_remaining = 2022235\n",
      "Epoch 258 Batch  144/144   train_loss = 2.672   time_elapsed = 53548.988   time_remaining = 2021993\n",
      "Epoch 259 Batch  144/144   train_loss = 2.676   time_elapsed = 53755.920   time_remaining = 2021762\n",
      "Epoch 260 Batch  144/144   train_loss = 2.669   time_elapsed = 53963.085   time_remaining = 2021540\n",
      "Epoch 261 Batch  144/144   train_loss = 2.667   time_elapsed = 54170.377   time_remaining = 2021323\n",
      "Model Trained and Saved\n",
      "Epoch 262 Batch  144/144   train_loss = 2.686   time_elapsed = 54382.625   time_remaining = 2021290\n",
      "Epoch 263 Batch  144/144   train_loss = 2.698   time_elapsed = 54590.919   time_remaining = 2021109\n",
      "Epoch 264 Batch  144/144   train_loss = 2.690   time_elapsed = 54799.125   time_remaining = 2020925\n",
      "Epoch 265 Batch  144/144   train_loss = 2.688   time_elapsed = 55007.441   time_remaining = 2020745\n",
      "Epoch 266 Batch  144/144   train_loss = 2.664   time_elapsed = 55215.886   time_remaining = 2020569\n",
      "Epoch 267 Batch  144/144   train_loss = 2.648   time_elapsed = 55424.029   time_remaining = 2020382\n",
      "Epoch 268 Batch  144/144   train_loss = 2.652   time_elapsed = 55631.741   time_remaining = 2020179\n",
      "Epoch 269 Batch  144/144   train_loss = 2.655   time_elapsed = 55839.178   time_remaining = 2019967\n",
      "Epoch 270 Batch  144/144   train_loss = 2.660   time_elapsed = 56046.703   time_remaining = 2019757\n",
      "Epoch 271 Batch  144/144   train_loss = 2.654   time_elapsed = 56253.962   time_remaining = 2019538\n",
      "Model Trained and Saved\n",
      "Epoch 272 Batch  144/144   train_loss = 2.651   time_elapsed = 56465.401   time_remaining = 2019468\n",
      "Epoch 273 Batch  144/144   train_loss = 2.640   time_elapsed = 56672.408   time_remaining = 2019240\n",
      "Epoch 274 Batch  144/144   train_loss = 2.638   time_elapsed = 56879.398   time_remaining = 2019011\n",
      "Epoch 275 Batch  144/144   train_loss = 2.646   time_elapsed = 57086.413   time_remaining = 2018783\n",
      "Epoch 276 Batch  144/144   train_loss = 2.630   time_elapsed = 57293.464   time_remaining = 2018557\n",
      "Epoch 277 Batch  144/144   train_loss = 2.640   time_elapsed = 57500.485   time_remaining = 2018329\n",
      "Epoch 278 Batch  144/144   train_loss = 2.619   time_elapsed = 57707.579   time_remaining = 2018105\n",
      "Epoch 279 Batch  144/144   train_loss = 2.617   time_elapsed = 57914.468   time_remaining = 2017873\n",
      "Epoch 280 Batch  144/144   train_loss = 2.616   time_elapsed = 58121.512   time_remaining = 2017647\n",
      "Epoch 281 Batch  144/144   train_loss = 2.600   time_elapsed = 58328.542   time_remaining = 2017420\n",
      "Model Trained and Saved\n",
      "Epoch 282 Batch  144/144   train_loss = 2.612   time_elapsed = 58539.836   time_remaining = 2017341\n",
      "Epoch 283 Batch  144/144   train_loss = 2.611   time_elapsed = 58746.810   time_remaining = 2017112\n",
      "Epoch 284 Batch  144/144   train_loss = 2.621   time_elapsed = 58953.804   time_remaining = 2016884\n",
      "Epoch 285 Batch  144/144   train_loss = 2.595   time_elapsed = 59160.910   time_remaining = 2016660\n",
      "Epoch 286 Batch  144/144   train_loss = 2.596   time_elapsed = 59367.938   time_remaining = 2016434\n",
      "Epoch 287 Batch  144/144   train_loss = 2.595   time_elapsed = 59574.919   time_remaining = 2016206\n",
      "Epoch 288 Batch  144/144   train_loss = 2.599   time_elapsed = 59781.837   time_remaining = 2015976\n",
      "Epoch 289 Batch  144/144   train_loss = 2.618   time_elapsed = 59988.654   time_remaining = 2015743\n",
      "Epoch 290 Batch  144/144   train_loss = 2.601   time_elapsed = 60195.583   time_remaining = 2015514\n",
      "Epoch 291 Batch  144/144   train_loss = 2.606   time_elapsed = 60402.756   time_remaining = 2015293\n",
      "Model Trained and Saved\n",
      "Epoch 292 Batch  144/144   train_loss = 2.604   time_elapsed = 60613.950   time_remaining = 2015206\n",
      "Epoch 293 Batch  144/144   train_loss = 2.618   time_elapsed = 60821.157   time_remaining = 2014986\n",
      "Epoch 294 Batch  144/144   train_loss = 2.625   time_elapsed = 61028.076   time_remaining = 2014757\n",
      "Epoch 295 Batch  144/144   train_loss = 2.617   time_elapsed = 61234.978   time_remaining = 2014527\n",
      "Epoch 296 Batch  144/144   train_loss = 2.613   time_elapsed = 61441.844   time_remaining = 2014296\n",
      "Epoch 297 Batch  144/144   train_loss = 2.612   time_elapsed = 61648.863   time_remaining = 2014070\n",
      "Epoch 298 Batch  144/144   train_loss = 2.620   time_elapsed = 61855.972   time_remaining = 2013848\n",
      "Epoch 299 Batch  144/144   train_loss = 2.597   time_elapsed = 62063.078   time_remaining = 2013625\n",
      "Epoch 300 Batch  144/144   train_loss = 2.606   time_elapsed = 62270.108   time_remaining = 2013400\n",
      "Epoch 301 Batch  144/144   train_loss = 2.590   time_elapsed = 62477.141   time_remaining = 2013175\n",
      "Model Trained and Saved\n",
      "Epoch 302 Batch  144/144   train_loss = 2.608   time_elapsed = 62688.380   time_remaining = 2013086\n",
      "Epoch 303 Batch  144/144   train_loss = 2.602   time_elapsed = 62895.375   time_remaining = 2012860\n",
      "Epoch 304 Batch  144/144   train_loss = 2.591   time_elapsed = 63102.469   time_remaining = 2012637\n",
      "Epoch 305 Batch  144/144   train_loss = 2.605   time_elapsed = 63309.436   time_remaining = 2012410\n",
      "Epoch 306 Batch  144/144   train_loss = 2.619   time_elapsed = 63516.451   time_remaining = 2012185\n",
      "Epoch 307 Batch  144/144   train_loss = 2.619   time_elapsed = 63723.476   time_remaining = 2011960\n",
      "Epoch 308 Batch  144/144   train_loss = 2.611   time_elapsed = 63930.568   time_remaining = 2011737\n",
      "Epoch 309 Batch  144/144   train_loss = 2.610   time_elapsed = 64137.546   time_remaining = 2011511\n",
      "Epoch 310 Batch  144/144   train_loss = 2.602   time_elapsed = 64344.516   time_remaining = 2011285\n",
      "Epoch 311 Batch  144/144   train_loss = 2.589   time_elapsed = 64551.552   time_remaining = 2011061\n",
      "Model Trained and Saved\n",
      "Epoch 312 Batch  144/144   train_loss = 2.583   time_elapsed = 64762.997   time_remaining = 2010974\n",
      "Epoch 313 Batch  144/144   train_loss = 2.566   time_elapsed = 64969.917   time_remaining = 2010746\n",
      "Epoch 314 Batch  144/144   train_loss = 2.564   time_elapsed = 65177.081   time_remaining = 2010526\n",
      "Epoch 315 Batch  144/144   train_loss = 2.559   time_elapsed = 65384.085   time_remaining = 2010301\n",
      "Epoch 316 Batch  144/144   train_loss = 2.550   time_elapsed = 65591.475   time_remaining = 2010088\n",
      "Epoch 317 Batch  144/144   train_loss = 2.559   time_elapsed = 65798.604   time_remaining = 2009867\n",
      "Epoch 318 Batch  144/144   train_loss = 2.555   time_elapsed = 66005.506   time_remaining = 2009639\n",
      "Epoch 319 Batch  144/144   train_loss = 2.573   time_elapsed = 66212.582   time_remaining = 2009417\n",
      "Epoch 320 Batch  144/144   train_loss = 2.539   time_elapsed = 66419.626   time_remaining = 2009194\n",
      "Epoch 321 Batch  144/144   train_loss = 2.538   time_elapsed = 66626.618   time_remaining = 2008969\n",
      "Model Trained and Saved\n",
      "Epoch 322 Batch  144/144   train_loss = 2.515   time_elapsed = 66837.976   time_remaining = 2008876\n",
      "Epoch 323 Batch  144/144   train_loss = 2.542   time_elapsed = 67045.025   time_remaining = 2008652\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 324 Batch  144/144   train_loss = 2.534   time_elapsed = 67252.079   time_remaining = 2008429\n",
      "Epoch 325 Batch  144/144   train_loss = 2.530   time_elapsed = 67459.233   time_remaining = 2008209\n",
      "Epoch 326 Batch  144/144   train_loss = 2.543   time_elapsed = 67666.477   time_remaining = 2007992\n",
      "Epoch 327 Batch  144/144   train_loss = 2.519   time_elapsed = 67873.447   time_remaining = 2007767\n",
      "Epoch 328 Batch  144/144   train_loss = 2.519   time_elapsed = 68080.579   time_remaining = 2007547\n",
      "Epoch 329 Batch  144/144   train_loss = 2.511   time_elapsed = 68287.609   time_remaining = 2007324\n",
      "Epoch 330 Batch  144/144   train_loss = 2.510   time_elapsed = 68494.624   time_remaining = 2007100\n",
      "Epoch 331 Batch  144/144   train_loss = 2.510   time_elapsed = 68701.541   time_remaining = 2006874\n",
      "Model Trained and Saved\n",
      "Epoch 332 Batch  144/144   train_loss = 2.501   time_elapsed = 68913.019   time_remaining = 2006780\n",
      "Epoch 333 Batch  144/144   train_loss = 2.503   time_elapsed = 69119.954   time_remaining = 2006554\n",
      "Epoch 334 Batch  144/144   train_loss = 2.501   time_elapsed = 69327.032   time_remaining = 2006333\n",
      "Epoch 335 Batch  144/144   train_loss = 2.492   time_elapsed = 69534.013   time_remaining = 2006108\n",
      "Epoch 336 Batch  144/144   train_loss = 2.502   time_elapsed = 69741.082   time_remaining = 2005886\n",
      "Epoch 337 Batch  144/144   train_loss = 2.489   time_elapsed = 69948.098   time_remaining = 2005663\n",
      "Epoch 338 Batch  144/144   train_loss = 2.479   time_elapsed = 70155.299   time_remaining = 2005445\n",
      "Epoch 339 Batch  144/144   train_loss = 2.496   time_elapsed = 70362.466   time_remaining = 2005226\n",
      "Epoch 340 Batch  144/144   train_loss = 2.496   time_elapsed = 70569.479   time_remaining = 2005003\n",
      "Epoch 341 Batch  144/144   train_loss = 2.510   time_elapsed = 70776.653   time_remaining = 2004785\n",
      "Model Trained and Saved\n",
      "Epoch 342 Batch  144/144   train_loss = 2.503   time_elapsed = 70988.032   time_remaining = 2004685\n",
      "Epoch 343 Batch  144/144   train_loss = 2.498   time_elapsed = 71194.934   time_remaining = 2004459\n",
      "Epoch 344 Batch  144/144   train_loss = 2.490   time_elapsed = 71401.826   time_remaining = 2004233\n",
      "Epoch 345 Batch  144/144   train_loss = 2.490   time_elapsed = 71608.897   time_remaining = 2004011\n",
      "Epoch 346 Batch  144/144   train_loss = 2.483   time_elapsed = 71815.968   time_remaining = 2003790\n",
      "Epoch 347 Batch  144/144   train_loss = 2.496   time_elapsed = 72023.014   time_remaining = 2003568\n",
      "Epoch 348 Batch  144/144   train_loss = 2.489   time_elapsed = 72229.975   time_remaining = 2003344\n",
      "Epoch 349 Batch  144/144   train_loss = 2.496   time_elapsed = 72437.163   time_remaining = 2003126\n",
      "Epoch 350 Batch  144/144   train_loss = 2.500   time_elapsed = 72644.298   time_remaining = 2002907\n",
      "Epoch 351 Batch  144/144   train_loss = 2.499   time_elapsed = 72851.487   time_remaining = 2002689\n",
      "Model Trained and Saved\n",
      "Epoch 352 Batch  144/144   train_loss = 2.483   time_elapsed = 73062.925   time_remaining = 2002588\n",
      "Epoch 353 Batch  144/144   train_loss = 2.499   time_elapsed = 73269.866   time_remaining = 2002364\n",
      "Epoch 354 Batch  144/144   train_loss = 2.487   time_elapsed = 73476.805   time_remaining = 2002139\n",
      "Epoch 355 Batch  144/144   train_loss = 2.508   time_elapsed = 73683.894   time_remaining = 2001919\n",
      "Epoch 356 Batch  144/144   train_loss = 2.501   time_elapsed = 73890.833   time_remaining = 2001694\n",
      "Epoch 357 Batch  144/144   train_loss = 2.489   time_elapsed = 74097.822   time_remaining = 2001471\n",
      "Epoch 358 Batch  144/144   train_loss = 2.477   time_elapsed = 74304.914   time_remaining = 2001251\n",
      "Epoch 359 Batch  144/144   train_loss = 2.474   time_elapsed = 74512.021   time_remaining = 2001032\n",
      "Epoch 360 Batch  144/144   train_loss = 2.474   time_elapsed = 74719.268   time_remaining = 2000816\n",
      "Epoch 361 Batch  144/144   train_loss = 2.474   time_elapsed = 74926.424   time_remaining = 2000598\n",
      "Model Trained and Saved\n",
      "Epoch 362 Batch  144/144   train_loss = 2.479   time_elapsed = 75137.665   time_remaining = 2000488\n",
      "Epoch 363 Batch  144/144   train_loss = 2.477   time_elapsed = 75344.841   time_remaining = 2000271\n",
      "Epoch 364 Batch  144/144   train_loss = 2.459   time_elapsed = 75551.761   time_remaining = 2000046\n",
      "Epoch 365 Batch  144/144   train_loss = 2.472   time_elapsed = 75759.101   time_remaining = 1999833\n",
      "Epoch 366 Batch  144/144   train_loss = 2.472   time_elapsed = 75966.281   time_remaining = 1999615\n",
      "Epoch 367 Batch  144/144   train_loss = 2.487   time_elapsed = 76173.361   time_remaining = 1999395\n",
      "Epoch 368 Batch  144/144   train_loss = 2.468   time_elapsed = 76380.278   time_remaining = 1999171\n",
      "Epoch 369 Batch  144/144   train_loss = 2.467   time_elapsed = 76587.445   time_remaining = 1998953\n",
      "Epoch 370 Batch  144/144   train_loss = 2.453   time_elapsed = 76794.676   time_remaining = 1998737\n",
      "Epoch 371 Batch  144/144   train_loss = 2.465   time_elapsed = 77001.730   time_remaining = 1998517\n",
      "Model Trained and Saved\n",
      "Epoch 372 Batch  144/144   train_loss = 2.471   time_elapsed = 77213.324   time_remaining = 1998414\n",
      "Epoch 373 Batch  144/144   train_loss = 2.462   time_elapsed = 77420.594   time_remaining = 1998199\n",
      "Epoch 374 Batch  144/144   train_loss = 2.462   time_elapsed = 77627.651   time_remaining = 1997978\n",
      "Epoch 375 Batch  144/144   train_loss = 2.442   time_elapsed = 77834.772   time_remaining = 1997759\n",
      "Epoch 376 Batch  144/144   train_loss = 2.439   time_elapsed = 78041.819   time_remaining = 1997538\n",
      "Epoch 377 Batch  144/144   train_loss = 2.447   time_elapsed = 78249.242   time_remaining = 1997327\n",
      "Epoch 378 Batch  144/144   train_loss = 2.451   time_elapsed = 78456.290   time_remaining = 1997107\n",
      "Epoch 379 Batch  144/144   train_loss = 2.446   time_elapsed = 78663.193   time_remaining = 1996883\n",
      "Epoch 380 Batch  144/144   train_loss = 2.431   time_elapsed = 78870.207   time_remaining = 1996662\n",
      "Epoch 381 Batch  144/144   train_loss = 2.444   time_elapsed = 79077.356   time_remaining = 1996444\n",
      "Model Trained and Saved\n",
      "Epoch 382 Batch  144/144   train_loss = 2.435   time_elapsed = 79289.502   time_remaining = 1996352\n",
      "Epoch 383 Batch  144/144   train_loss = 2.419   time_elapsed = 79496.639   time_remaining = 1996134\n",
      "Epoch 384 Batch  144/144   train_loss = 2.434   time_elapsed = 79703.861   time_remaining = 1995918\n",
      "Epoch 385 Batch  144/144   train_loss = 2.451   time_elapsed = 79911.091   time_remaining = 1995702\n",
      "Epoch 386 Batch  144/144   train_loss = 2.439   time_elapsed = 80118.206   time_remaining = 1995483\n",
      "Epoch 387 Batch  144/144   train_loss = 2.447   time_elapsed = 80325.250   time_remaining = 1995263\n",
      "Epoch 388 Batch  144/144   train_loss = 2.437   time_elapsed = 80532.589   time_remaining = 1995050\n",
      "Epoch 389 Batch  144/144   train_loss = 2.429   time_elapsed = 80739.607   time_remaining = 1994829\n",
      "Epoch 390 Batch  144/144   train_loss = 2.435   time_elapsed = 80946.902   time_remaining = 1994615\n",
      "Epoch 391 Batch  144/144   train_loss = 2.429   time_elapsed = 81154.067   time_remaining = 1994398\n",
      "Model Trained and Saved\n",
      "Epoch 392 Batch  144/144   train_loss = 2.418   time_elapsed = 81365.942   time_remaining = 1994296\n",
      "Epoch 393 Batch  144/144   train_loss = 2.413   time_elapsed = 81572.971   time_remaining = 1994075\n",
      "Epoch 394 Batch  144/144   train_loss = 2.431   time_elapsed = 81780.108   time_remaining = 1993857\n",
      "Epoch 395 Batch  144/144   train_loss = 2.433   time_elapsed = 81987.346   time_remaining = 1993642\n",
      "Epoch 396 Batch  144/144   train_loss = 2.433   time_elapsed = 82194.530   time_remaining = 1993425\n",
      "Epoch 397 Batch  144/144   train_loss = 2.424   time_elapsed = 82401.942   time_remaining = 1993214\n",
      "Epoch 398 Batch  144/144   train_loss = 2.419   time_elapsed = 82608.975   time_remaining = 1992993\n",
      "Epoch 399 Batch  144/144   train_loss = 2.425   time_elapsed = 82815.996   time_remaining = 1992773\n",
      "Epoch 400 Batch  144/144   train_loss = 2.424   time_elapsed = 83023.169   time_remaining = 1992556\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "pickle.dump((seq_length, save_dir), open('params.p', 'wb'))\n",
    "batches = get_batches(corpus_int, batch_size, seq_length)\n",
    "num_batches = len(batches)\n",
    "start_time = time.time()\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        state = sess.run(initial_state, {input_text: batches[0][0]})\n",
    "        \n",
    "        for batch_index, (x, y) in enumerate(batches):\n",
    "            feed_dict = {\n",
    "                input_text: x,\n",
    "                targets: y,\n",
    "                initial_state: state,\n",
    "                lr: learning_rate\n",
    "            }\n",
    "            train_loss, state, _ = sess.run([cost, final_state, train_op], feed_dict)\n",
    "            \n",
    "        time_elapsed = time.time() - start_time\n",
    "        print('Epoch {:>3} Batch {:>4}/{}   train_loss = {:.3f}   time_elapsed = {:.3f}   time_remaining = {:.0f}'.format(\n",
    "            epoch + 1,\n",
    "            batch_index + 1,\n",
    "            len(batches),\n",
    "            train_loss,\n",
    "            time_elapsed,\n",
    "            ((num_batches * num_epochs)/((epoch + 1) * (batch_index + 1))) * time_elapsed - time_elapsed))\n",
    "\n",
    "        # save model every 10 epochs\n",
    "        if epoch % 10 == 0:\n",
    "            saver = tf.train.Saver()\n",
    "            saver.save(sess, save_dir)\n",
    "            print('Model Trained and Saved')\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "corpus_int, vocab_to_int, int_to_vocab, token_dict = pickle.load(open('preprocess.p', mode='rb'))\n",
    "seq_length, save_dir = pickle.load(open('params.p', mode='rb'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate GOT Text\n",
    "### Pick a Random Word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pick_word(probabilities, int_to_vocab):\n",
    "    \"\"\"\n",
    "    Pick the next word with some randomness\n",
    "    :param probabilities: Probabilites of the next word\n",
    "    :param int_to_vocab: Dictionary of word ids as the keys and words as the values\n",
    "    :return: String of the predicted word\n",
    "    \"\"\"\n",
    "    return np.random.choice(list(int_to_vocab.values()), 1, p=probabilities)[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Graph and Generate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen_length = 1000\n",
    "prime_words = 'daenerys'\n",
    "\n",
    "loaded_graph = tf.Graph()\n",
    "with tf.Session(graph=loaded_graph) as sess:\n",
    "    # Load the saved model\n",
    "    loader = tf.train.import_meta_graph(save_dir + '.meta')\n",
    "    loader.restore(sess, save_dir)\n",
    "    \n",
    "    # Get tensors from loaded graph\n",
    "    input_text = loaded_graph.get_tensor_by_name('input:0')\n",
    "    initial_state = loaded_graph.get_tensor_by_name('initial_state:0')\n",
    "    final_state = loaded_graph.get_tensor_by_name('final_state:0')\n",
    "    probs = loaded_graph.get_tensor_by_name('probs:0')\n",
    "    \n",
    "    # Sentences generation setup\n",
    "    gen_sentences = prime_words.split()\n",
    "    prev_state = sess.run(initial_state, {input_text: np.array([[1 for word in gen_sentences]])})\n",
    "    \n",
    "    # Generate sentences\n",
    "    for n in range(gen_length):\n",
    "        # Dynamic Input\n",
    "        dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]\n",
    "        dyn_seq_length = len(dyn_input[0])\n",
    "\n",
    "        # Get Prediction\n",
    "        probabilities, prev_state = sess.run(\n",
    "            [probs, final_state],\n",
    "            {input_text: dyn_input, initial_state: prev_state})\n",
    "\n",
    "        pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab)\n",
    "\n",
    "        gen_sentences.append(pred_word)\n",
    "        \n",
    "    # Remove tokens\n",
    "    chapter_text = ' '.join(gen_sentences)\n",
    "    for key, token in token_dict.items():\n",
    "        chapter_text = chapter_text.replace(' ' + token.lower(), key)\n",
    "        \n",
    "    print(chapter_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save a Chapter\n",
    "### Cleanup Data a Bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "chapter_text = ' '.join(gen_sentences)\n",
    "for key, token in token_dict.items():\n",
    "    chapter_text = chapter_text.replace(' ' + token.lower(), key)\n",
    "chapter_text = chapter_text.replace('\\n ', '\\n')\n",
    "chapter_text = chapter_text.replace('( ', '(')\n",
    "chapter_text = chapter_text.replace(' ”', '”')\n",
    "\n",
    "capitalize_words = ['lannister', 'stark', 'lord', 'ser', 'tyrion', 'jon', 'john snow', 'daenerys', 'targaryen', 'cersei', 'jaime', 'arya', 'sansa', 'bran', 'rikkon', 'joffrey', \n",
    "                    'khal', 'drogo', 'gregor', 'clegane', 'kings landing', 'winterfell', 'the mountain', 'the hound', 'ramsay', 'bolton', 'melisandre', 'shae', 'tyrell',\n",
    "                   'margaery', 'sandor', 'hodor', 'ygritte', 'brienne', 'tarth', 'petyr', 'baelish', 'eddard', 'greyjoy', 'theon', 'gendry', 'baratheon', 'baraTheon',\n",
    "                   'varys', 'stannis', 'bronn', 'jorah', 'mormont', 'martell', 'oberyn', 'catelyn', 'robb', 'loras', 'missandei', 'tommen', 'robert', 'lady', 'donella', 'redwyne'\n",
    "                   'myrcella', 'samwell', 'tarly', 'grey worm', 'podrick', 'osha', 'davos', 'seaworth', 'jared', 'jeyne poole', 'rickard', 'yoren', 'meryn', 'trant', 'king', 'queen',\n",
    "                   'aemon']\n",
    "\n",
    "for word in capitalize_words:\n",
    "    chapter_text = chapter_text.replace(word, word.lower().title())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "version_dir = './generated-book-v1'\n",
    "if not os.path.exists(version_dir):\n",
    "    os.makedirs(version_dir)\n",
    "\n",
    "num_chapters = len([name for name in os.listdir(version_dir) if os.path.isfile(os.path.join(version_dir, name))])\n",
    "next_chapter = version_dir + '/chapter-' + str(num_chapters + 1) + '.md'\n",
    "with open(next_chapter, \"w\") as text_file:\n",
    "    text_file.write(chapter_text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}