Deep_Learning_Project_On_TV_Script_Generation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Deep Learning Project On TV Script Generation.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "aG8EV4LMWPA3",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Problem Statement"
      ]
    },
    {
      "metadata": {
        "id": "joBCSa7jWTL5",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Language Translation\n",
        "<p>In this project, I am going to take a peek into the realm of neural network machine\n",
        "translation. I’ll be training a sequence to sequence model on a dataset of English and French\n",
        "sentences that can translate new sentences from English to French. </p>"
      ]
    },
    {
      "metadata": {
        "id": "8oAJq2W-WkS8",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "##Get the Data\n",
        "<p>Since translating the whole language of English to French will take lots of time to train, I'll be using a small portion of the English corpus. </p>"
      ]
    },
    {
      "metadata": {
        "id": "52I2e21BYJms",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Following are some helper functions:"
      ]
    },
    {
      "metadata": {
        "id": "kdmc3t8wBk1s",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "import pickle\n",
        "import copy\n",
        "import numpy as np\n",
        "\n",
        "\n",
        "CODES = {'<PAD>': 0, '<EOS>': 1, '<UNK>': 2, '<GO>': 3 }\n",
        "\n",
        "\n",
        "def load_data(path):\n",
        "    \"\"\"\n",
        "    Load Dataset from File\n",
        "    \"\"\"\n",
        "    input_file = os.path.join(path)\n",
        "    with open(input_file, 'r', encoding='utf-8') as f:\n",
        "        data = f.read()\n",
        "\n",
        "    return data\n",
        "\n",
        "\n",
        "def preprocess_and_save_data(source_path, target_path, text_to_ids):\n",
        "    \"\"\"\n",
        "    Preprocess Text Data.  Save to to file.\n",
        "    \"\"\"\n",
        "    # Preprocess\n",
        "    source_text = load_data(source_path)\n",
        "    target_text = load_data(target_path)\n",
        "\n",
        "    source_text = source_text.lower()\n",
        "    target_text = target_text.lower()\n",
        "\n",
        "    source_vocab_to_int, source_int_to_vocab = create_lookup_tables(source_text)\n",
        "    target_vocab_to_int, target_int_to_vocab = create_lookup_tables(target_text)\n",
        "\n",
        "    source_text, target_text = text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int)\n",
        "\n",
        "    # Save Data\n",
        "    pickle.dump((\n",
        "        (source_text, target_text),\n",
        "        (source_vocab_to_int, target_vocab_to_int),\n",
        "        (source_int_to_vocab, target_int_to_vocab)), open('preprocess.p', 'wb'))\n",
        "\n",
        "\n",
        "def load_preprocess():\n",
        "    \"\"\"\n",
        "    Load the Preprocessed Training data and return them in batches of <batch_size> or less\n",
        "    \"\"\"\n",
        "    return pickle.load(open('preprocess.p', mode='rb'))\n",
        "\n",
        "\n",
        "def create_lookup_tables(text):\n",
        "    \"\"\"\n",
        "    Create lookup tables for vocabulary\n",
        "    \"\"\"\n",
        "    vocab = set(text.split())\n",
        "    vocab_to_int = copy.copy(CODES)\n",
        "\n",
        "    for v_i, v in enumerate(vocab, len(CODES)):\n",
        "        vocab_to_int[v] = v_i\n",
        "\n",
        "    int_to_vocab = {v_i: v for v, v_i in vocab_to_int.items()}\n",
        "\n",
        "    return vocab_to_int, int_to_vocab\n",
        "\n",
        "\n",
        "def save_params(params):\n",
        "    \"\"\"\n",
        "    Save parameters to file\n",
        "    \"\"\"\n",
        "    pickle.dump(params, open('params.p', 'wb'))\n",
        "\n",
        "\n",
        "def load_params():\n",
        "    \"\"\"\n",
        "    Load parameters from file\n",
        "    \"\"\"\n",
        "    return pickle.load(open('params.p', mode='rb'))\n",
        "\n",
        "\n",
        "def batch_data(source, target, batch_size):\n",
        "    \"\"\"\n",
        "    Batch source and target together\n",
        "    \"\"\"\n",
        "    for batch_i in range(0, len(source)//batch_size):\n",
        "        start_i = batch_i * batch_size\n",
        "        source_batch = source[start_i:start_i + batch_size]\n",
        "        target_batch = target[start_i:start_i + batch_size]\n",
        "        yield np.array(pad_sentence_batch(source_batch)), np.array(pad_sentence_batch(target_batch))\n",
        "\n",
        "\n",
        "def pad_sentence_batch(sentence_batch):\n",
        "    \"\"\"\n",
        "    Pad sentence with <PAD> id\n",
        "    \"\"\"\n",
        "    max_sentence = max([len(sentence) for sentence in sentence_batch])\n",
        "    return [sentence + [CODES['<PAD>']] * (max_sentence - len(sentence))\n",
        "            for sentence in sentence_batch]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2S9qyNSJBk19",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import itertools\n",
        "import collections"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hrODZENEBrYB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "outputId": "9348e810-707e-4fef-dc7b-ffcb8b054c65"
      },
      "cell_type": "code",
      "source": [
        "!pip install pydrive --upgrade\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "from pydrive.drive import GoogleDrive\n",
        "from pydrive.auth import GoogleAuth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)\n",
        "\n",
        "myfile1 = drive.CreateFile({'id': '0B7nc62eNjfwcTkZuNVZIVXZzdnpud0xLZUpmQm42WGhMYWZz'})\n",
        "myfile1.GetContentFile('small_vocab_en.txt')\n",
        "\n",
        "myfile2 = drive.CreateFile({'id': '0B7nc62eNjfwcWkU1R0dyc0pFTzJ3T09peXhNNXdjYld1eWhJ'})\n",
        "myfile2.GetContentFile('small_vocab_fr.txt')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pydrive\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/e0/0e64788e5dd58ce2d6934549676243dc69d982f198524be9b99e9c2a4fd5/PyDrive-1.3.1.tar.gz (987kB)\n",
            "\u001b[K    100% |████████████████████████████████| 993kB 10.2MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from pydrive) (1.6.7)\n",
            "Requirement already satisfied, skipping upgrade: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pydrive) (4.1.2)\n",
            "Requirement already satisfied, skipping upgrade: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from pydrive) (3.13)\n",
            "Requirement already satisfied, skipping upgrade: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->pydrive) (3.0.0)\n",
            "Requirement already satisfied, skipping upgrade: six<2dev,>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->pydrive) (1.11.0)\n",
            "Requirement already satisfied, skipping upgrade: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->pydrive) (0.11.3)\n",
            "Requirement already satisfied, skipping upgrade: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->pydrive) (3.4.2)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->pydrive) (0.4.4)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->pydrive) (0.2.2)\n",
            "Building wheels for collected packages: pydrive\n",
            "  Running setup.py bdist_wheel for pydrive ... \u001b[?25l-\b \b\\\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/fa/d2/9a/d3b6b506c2da98289e5d417215ce34b696db856643bad779f4\n",
            "Successfully built pydrive\n",
            "Installing collected packages: pydrive\n",
            "Successfully installed pydrive-1.3.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LSBBUNxmBk2I",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#Load data\n",
        "\n",
        "source_path = 'small_vocab_en.txt'\n",
        "target_path = 'small_vocab_fr.txt'\n",
        "source_text = load_data(source_path)\n",
        "target_text = load_data(target_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "H1cepJXEXECC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Explore the Data\n",
        "<p>Play around with view_sentence_range to view different parts of the data.<p>\n",
        "<p>Dataset Stats: </p>\n",
        "<ul><li>Roughly the number of unique words: 227</li>\n",
        " <li>Number of sentences: 137861</li>\n",
        "<li>Average number of words in a sentence: 13.225277634719028 </li></ul>"
      ]
    },
    {
      "metadata": {
        "id": "qS02rZRBBk2T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        },
        "outputId": "160a3a80-d800-41e6-f993-e16dbfa6bed2"
      },
      "cell_type": "code",
      "source": [
        "view_sentence_range = (0, 10)\n",
        "\n",
        "\"\"\"\n",
        "DON'T MODIFY ANYTHING IN THIS CELL\n",
        "\"\"\"\n",
        "import numpy as np\n",
        "\n",
        "print('Dataset Stats')\n",
        "print('Roughly the number of unique words: {}'.format(len({word: None for word in source_text.split()})))\n",
        "\n",
        "sentences = source_text.split('\\n')\n",
        "word_counts = [len(sentence.split()) for sentence in sentences]\n",
        "print('Number of sentences: {}'.format(len(sentences)))\n",
        "print('Average number of words in a sentence: {}'.format(np.average(word_counts)))\n",
        "\n",
        "print()\n",
        "print('English sentences {} to {}:'.format(*view_sentence_range))\n",
        "print('\\n'.join(source_text.split('\\n')[view_sentence_range[0]:view_sentence_range[1]]))\n",
        "print()\n",
        "print('French sentences {} to {}:'.format(*view_sentence_range))\n",
        "print('\\n'.join(target_text.split('\\n')[view_sentence_range[0]:view_sentence_range[1]]))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dataset Stats\n",
            "Roughly the number of unique words: 227\n",
            "Number of sentences: 137861\n",
            "Average number of words in a sentence: 13.225277634719028\n",
            "\n",
            "English sentences 0 to 10:\n",
            "new jersey is sometimes quiet during autumn , and it is snowy in april .\n",
            "the united states is usually chilly during july , and it is usually freezing in november .\n",
            "california is usually quiet during march , and it is usually hot in june .\n",
            "the united states is sometimes mild during june , and it is cold in september .\n",
            "your least liked fruit is the grape , but my least liked is the apple .\n",
            "his favorite fruit is the orange , but my favorite is the grape .\n",
            "paris is relaxing during december , but it is usually chilly in july .\n",
            "new jersey is busy during spring , and it is never hot in march .\n",
            "our least liked fruit is the lemon , but my least liked is the grape .\n",
            "the united states is sometimes busy during january , and it is sometimes warm in november .\n",
            "\n",
            "French sentences 0 to 10:\n",
            "new jersey est parfois calme pendant l' automne , et il est neigeux en avril .\n",
            "les états-unis est généralement froid en juillet , et il gèle habituellement en novembre .\n",
            "california est généralement calme en mars , et il est généralement chaud en juin .\n",
            "les états-unis est parfois légère en juin , et il fait froid en septembre .\n",
            "votre moins aimé fruit est le raisin , mais mon moins aimé est la pomme .\n",
            "son fruit préféré est l'orange , mais mon préféré est le raisin .\n",
            "paris est relaxant en décembre , mais il est généralement froid en juillet .\n",
            "new jersey est occupé au printemps , et il est jamais chaude en mars .\n",
            "notre fruit est moins aimé le citron , mais mon moins aimé est le raisin .\n",
            "les états-unis est parfois occupé en janvier , et il est parfois chaud en novembre .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ALmsiPk-Yaps",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Implements Preprocessing Function"
      ]
    },
    {
      "metadata": {
        "id": "Pe4wAgGAYekv",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Text to Word Ids\n",
        "\n",
        "I must turn the text into a number so the computer can\n",
        "understand it. In the function text_to_ids(), I'll turn source_textand target_text from words\n",
        "to ids. However, I need to add the < EOS > word id at the end of each sentence from\n",
        "target_text. This will help the neural network predict when the sentence should end.\n",
        "I can get the \"< EOS >\" word id by doing: target_vocab_to_int[ '< EOS >' ] I can get other word ids using source_vocab_to_int and target_vocab_to_int. "
      ]
    },
    {
      "metadata": {
        "id": "H48Hpm5rBk2p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int):\n",
        "    \"\"\"\n",
        "    Convert source and target text to proper word ids\n",
        "    :param source_text: String that contains all the source text.\n",
        "    :param target_text: String that contains all the target text.\n",
        "    :param source_vocab_to_int: Dictionary to go from the source words to an id\n",
        "    :param target_vocab_to_int: Dictionary to go from the target words to an id\n",
        "    :return: A tuple of lists (source_id_text, target_id_text)\n",
        "    \"\"\"\n",
        "    source_sentences = source_text.split('\\n')\n",
        "    target_sentences = [sentence + ' <EOS>' for sentence in target_text.split('\\n')]\n",
        "    source_ids = list(map(lambda x: [source_vocab_to_int[word] for word in x.split()], source_sentences))\n",
        "    target_ids = list(map(lambda x: [target_vocab_to_int[word] for word in x.split()], target_sentences))\n",
        "    return source_ids, target_ids"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XaNfSdMMa6g9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Preprocess all the data and save it"
      ]
    },
    {
      "metadata": {
        "id": "txZeX2BIBk20",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "preprocess_and_save_data(source_path, target_path, text_to_ids)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vpNnDbPPBk2-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "(source_int_text, target_int_text), (source_vocab_to_int, target_vocab_to_int), _ = load_preprocess()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Tkont98RbL1W",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Build the Neural Network\n",
        "<p>I'll build the components necessary to build a Sequence-to-Sequence model by\n",
        "implementing the following functions below:</p>\n",
        "<ul><li> model_inputs</li>\n",
        "<li>process_decoding_input</li>\n",
        "<li> encoding_layer</li>\n",
        "<li>decoding_layer_train</li>\n",
        "<li>decoding_layer_infer</li>\n",
        "<li> decoding_layer</li>\n",
        "<li> seq2seq_model</li></ul>"
      ]
    },
    {
      "metadata": {
        "id": "XW-QEPYwbww1",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Input\n",
        "<p>Implementing the model_inputs() function to create TF Placeholders for the Neural Network. It will\n",
        "create the following placeholders:</p>\n",
        "<ul><li> Input text placeholder named \"input\" using the TF Placeholder name parameter\n",
        "  with rank 2. </li>\n",
        "<li> Targets placeholder with rank 2.</li>\n",
        "<li> Learning rate placeholder with rank 0.</li>\n",
        "<li> Keep probability placeholder named \"keep_prob\" using the TF Placeholder name\n",
        "parameter with rank 0.</li>\n",
        "<p>Return the placeholders in the following the tuple (Input, Targets, Learning Rate, Keep Probability)</p>"
      ]
    },
    {
      "metadata": {
        "id": "PQTZV1UTBk3J",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def model_inputs():\n",
        "    \"\"\"\n",
        "    Create TF Placeholders for input, targets, learning rate, and lengths of source and target sequences.\n",
        "    :return: Tuple (input, targets, learning rate, keep probability, target sequence length,\n",
        "    max target sequence length, source sequence length)\n",
        "    \"\"\"\n",
        "    \n",
        "    input = tf.placeholder(tf.int32,shape=[None,None],name='input')\n",
        "    targets = tf.placeholder(tf.int32,shape=[None,None],name='targets')\n",
        "    learningrate = tf.placeholder(tf.float32,shape=[],name='learningrate')\n",
        "    keep_prob = tf.placeholder(tf.float32,shape=[],name='keep_prob')\n",
        "    target_sequence_length = tf.placeholder(tf.int32,[None,],name='target_sequence_length')\n",
        "    max_target_length = tf.reduce_max(target_sequence_length)\n",
        "    source_sequence_length = tf.placeholder(tf.int32,[None,],name='source_sequence_length')\n",
        "    \n",
        "    return (input,targets,learningrate,keep_prob,target_sequence_length,max_target_length,source_sequence_length)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oMNmZp6scXp6",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Process Decoding Input\n",
        "<p> Implementing process_decoding_input using TensorFlow to remove the last word id from each batch in\n",
        "target_data and concat the GO ID to the beginning of each batch. </p>"
      ]
    },
    {
      "metadata": {
        "id": "dTnBVRdkBk3U",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def process_decoder_input(target_data, target_vocab_to_int, batch_size):\n",
        "    \"\"\"\n",
        "    Preprocess target data for encoding\n",
        "    :param target_data: Target Placehoder\n",
        "    :param target_vocab_to_int: Dictionary to go from the target words to an id\n",
        "    :param batch_size: Batch Size\n",
        "    :return: Preprocessed target data\n",
        "    \"\"\"\n",
        "    ending = tf.strided_slice(target_data, [0,0], [batch_size, -1], [1,1])\n",
        "    decoder_input = tf.concat([tf.fill([batch_size,1], target_vocab_to_int['<GO>']), ending], 1)\n",
        "    return decoder_input"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zMRGDAAocis2",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Encoding\n",
        "<p>Implementing encoding_layer() to create a Encoder RNN layer using tf.nn.dynamic_rnn() . </p>"
      ]
    },
    {
      "metadata": {
        "id": "kFfBr6rwBk3i",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def encoding_layer(rnn_inputs, rnn_size, num_layers, keep_prob, \n",
        "                   source_sequence_length, source_vocab_size, \n",
        "                   encoding_embedding_size):\n",
        "    \"\"\"\n",
        "    Create encoding layer\n",
        "    :param rnn_inputs: Inputs for the RNN\n",
        "    :param rnn_size: RNN Size\n",
        "    :param num_layers: Number of layers\n",
        "    :param keep_prob: Dropout keep probability\n",
        "    :param source_sequence_length: a list of the lengths of each sequence in the batch\n",
        "    :param source_vocab_size: vocabulary size of source data\n",
        "    :param encoding_embedding_size: embedding size of source data\n",
        "    :return: tuple (RNN output, RNN state)\n",
        "    \"\"\"\n",
        "    # Encoder embedding\n",
        "    enc_embed = tf.contrib.layers.embed_sequence(rnn_inputs, source_vocab_size, encoding_embedding_size)\n",
        "    \n",
        "    def create_cell(rnn_size):\n",
        "        lstm_cell = tf.contrib.rnn.LSTMCell(rnn_size,initializer=tf.random_uniform_initializer(-0.1,0.1,seed=2))\n",
        "        drop = tf.contrib.rnn.DropoutWrapper(lstm_cell, output_keep_prob=keep_prob)\n",
        "        return drop\n",
        "    \n",
        "    enc_cell = tf.contrib.rnn.MultiRNNCell([create_cell(rnn_size) for _ in range(num_layers)])\n",
        "    encoding_output, encoding_state = tf.nn.dynamic_rnn(enc_cell, enc_embed, \n",
        "                                                        sequence_length=source_sequence_length,dtype=tf.float32)\n",
        "    \n",
        "    return encoding_output, encoding_state"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "M88bAy64cr7-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Decoding - Training\n",
        "<p>Creating training logits using tf.contrib.seq2seq.simple_decoder_fn_train() and\n",
        "tf.contrib.seq2seq.dynamic_rnn_decoder() . Applying the output_fnto the\n",
        "tf.contrib.seq2seq.dynamic_rnn_decoder() outputs. </p>"
      ]
    },
    {
      "metadata": {
        "id": "7SGe8NrvBk3u",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def decoding_layer_train(encoder_state, dec_cell, dec_embed_input, \n",
        "                         target_sequence_length, max_summary_length, \n",
        "                         output_layer, keep_prob):\n",
        "    \"\"\"\n",
        "    Create a decoding layer for training\n",
        "    :param encoder_state: Encoder State\n",
        "    :param dec_cell: Decoder RNN Cell\n",
        "    :param dec_embed_input: Decoder embedded input\n",
        "    :param target_sequence_length: The lengths of each sequence in the target batch\n",
        "    :param max_summary_length: The length of the longest sequence in the batch\n",
        "    :param output_layer: Function to apply the output layer\n",
        "    :param keep_prob: Dropout keep probability\n",
        "    :return: BasicDecoderOutput containing training logits and sample_id\n",
        "    \"\"\"\n",
        "\n",
        "    training_helper = tf.contrib.seq2seq.TrainingHelper(inputs=dec_embed_input,\n",
        "                                                        sequence_length=target_sequence_length,\n",
        "                                                        time_major=False)\n",
        "    \n",
        "    training_decoder = tf.contrib.seq2seq.BasicDecoder(dec_cell, training_helper, encoder_state, output_layer)\n",
        "    \n",
        "    training_decoder_output = tf.contrib.seq2seq.dynamic_decode(training_decoder,\n",
        "                                                                impute_finished=True,\n",
        "                                                                maximum_iterations=max_summary_length)[0]\n",
        "    return training_decoder_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WsbCUcnac3-o",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Decoding - Inference\n",
        "<p>Creating inference logits using tf.contrib.seq2seq.simple_decoder_fn_inference() and\n",
        "tf.contrib.seq2seq.dynamic_rnn_decoder() .</p>"
      ]
    },
    {
      "metadata": {
        "id": "_SC7EsQnBk35",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, start_of_sequence_id,\n",
        "                         end_of_sequence_id, max_target_sequence_length,\n",
        "                         vocab_size, output_layer, batch_size, keep_prob):\n",
        "    \"\"\"\n",
        "    Create a decoding layer for inference\n",
        "    :param encoder_state: Encoder state\n",
        "    :param dec_cell: Decoder RNN Cell\n",
        "    :param dec_embeddings: Decoder embeddings\n",
        "    :param start_of_sequence_id: GO ID\n",
        "    :param end_of_sequence_id: EOS Id\n",
        "    :param max_target_sequence_length: Maximum length of target sequences\n",
        "    :param vocab_size: Size of decoder/target vocabulary\n",
        "    :param decoding_scope: TenorFlow Variable Scope for decoding\n",
        "    :param output_layer: Function to apply the output layer\n",
        "    :param batch_size: Batch size\n",
        "    :param keep_prob: Dropout keep probability\n",
        "    :return: BasicDecoderOutput containing inference logits and sample_id\n",
        "    \"\"\"\n",
        "\n",
        "    start_tokens = tf.tile(tf.constant([start_of_sequence_id], dtype=tf.int32), [batch_size], name='start_tokens')\n",
        "\n",
        "    inference_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(dec_embeddings,\n",
        "                                                                start_tokens,\n",
        "                                                                end_of_sequence_id)\n",
        "    \n",
        "    inference_decoder = tf.contrib.seq2seq.BasicDecoder(dec_cell,\n",
        "                                                        inference_helper,\n",
        "                                                        encoder_state,\n",
        "                                                        output_layer)\n",
        "    \n",
        "    inference_decoder_output = tf.contrib.seq2seq.dynamic_decode(inference_decoder,\n",
        "                                                            impute_finished=True,\n",
        "                                                            maximum_iterations=max_target_sequence_length)[0]\n",
        "    return inference_decoder_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aYGmkGapdAtB",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Build the Decoding Layer\n",
        "<p>Implementing decoding_layer() to create a Decoder RNN layer.</p>\n",
        "<ul><li> Creates RNN cell for decoding using rnn_size and num_layers.</li>\n",
        "<li>Creates the output fuction using lambda to transform it's input, logits, to class logits.</li>\n",
        "<li>Uses the decoding_layer_train(encoder_state, dec_cell, dec_embed_input,\n",
        "sequence_length, decoding_scope, output_fn, keep_prob) function to get the\n",
        "training logits.</li>\n",
        "<li> Uses the decoding_layer_infer(encoder_state, dec_cell, dec_embeddings,\n",
        "start_of_sequence_id, end_of_sequence_id, maximum_length, vocab_size,\n",
        "decoding_scope, output_fn, keep_prob) function to get the inference logits. </li></ul>\n",
        "<p>Note: I'll need to use tf.variable_scope to share variables between training and inference.</p>"
      ]
    },
    {
      "metadata": {
        "id": "8UAZZMIiBk4D",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def decoding_layer(dec_input, encoder_state,\n",
        "                   target_sequence_length, max_target_sequence_length,\n",
        "                   rnn_size,\n",
        "                   num_layers, target_vocab_to_int, target_vocab_size,\n",
        "                   batch_size, keep_prob, decoding_embedding_size):\n",
        "    \"\"\"\n",
        "    Create decoding layer\n",
        "    :param dec_input: Decoder input\n",
        "    :param encoder_state: Encoder state\n",
        "    :param target_sequence_length: The lengths of each sequence in the target batch\n",
        "    :param max_target_sequence_length: Maximum length of target sequences\n",
        "    :param rnn_size: RNN Size\n",
        "    :param num_layers: Number of layers\n",
        "    :param target_vocab_to_int: Dictionary to go from the target words to an id\n",
        "    :param target_vocab_size: Size of target vocabulary\n",
        "    :param batch_size: The size of the batch\n",
        "    :param keep_prob: Dropout keep probability\n",
        "    :param decoding_embedding_size: Decoding embedding size\n",
        "    :return: Tuple of (Training BasicDecoderOutput, Inference BasicDecoderOutput)\n",
        "    \"\"\"\n",
        "    # 1. Decoder Embedding\n",
        "    dec_embeddings = tf.Variable(tf.random_uniform([target_vocab_size, decoding_embedding_size]))\n",
        "    dec_embed_input = tf.nn.embedding_lookup(dec_embeddings, dec_input)\n",
        "    \n",
        "    # 2. Construct the decoder cell\n",
        "    def create_cell(rnn_size):\n",
        "        lstm_cell = tf.contrib.rnn.LSTMCell(rnn_size,\n",
        "                                            initializer=tf.random_uniform_initializer(-0.1,0.1,seed=2))\n",
        "        drop = tf.contrib.rnn.DropoutWrapper(lstm_cell, output_keep_prob=keep_prob)\n",
        "        return drop\n",
        "\n",
        "    dec_cell = tf.contrib.rnn.MultiRNNCell([create_cell(rnn_size) for _ in range(num_layers)])\n",
        "    \n",
        "    output_layer = Dense(target_vocab_size,\n",
        "                         kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.1))\n",
        "\n",
        "    with tf.variable_scope(\"decode\"):\n",
        "        train_decoder_out = decoding_layer_train(encoder_state, dec_cell, dec_embed_input, \n",
        "                         target_sequence_length, max_target_sequence_length, output_layer, keep_prob)\n",
        "        \n",
        "    with tf.variable_scope(\"decode\", reuse=True):\n",
        "        infer_decoder_out = decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, \n",
        "                             target_vocab_to_int['<GO>'], target_vocab_to_int['<EOS>'], max_target_sequence_length, \n",
        "                             target_vocab_size, output_layer, batch_size, keep_prob)\n",
        "        \n",
        "    return (train_decoder_out, infer_decoder_out)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TeEikyMPd4nP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Build the Neural Network\n",
        "<p>Applying the functions I have implemented above too: </p>\n",
        "<ul><li> Apply embedding to the input data for the encoder.</li>\n",
        "<li>Encode the input using the encoding_layer(rnn_inputs, rnn_size, num_layers,\n",
        "keep_prob).</li>\n",
        "<li> Process target data using the process_decoding_input(target_data,\n",
        "target_vocab_to_int, batch_size) function.</li>\n",
        "<li> Apply embedding to the target data for the decoder.</li>\n",
        "<li>Decode the encoded input using the decoding_layer(dec_embed_input,\n",
        "dec_embeddings, encoder_state, vocab_size, sequence_length, rnn_size,\n",
        "num_layers, target_vocab_to_int, keep_prob).</li></ul>"
      ]
    },
    {
      "metadata": {
        "id": "gHF1yw7bBk4P",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def seq2seq_model(input_data, target_data, keep_prob, batch_size,\n",
        "                  source_sequence_length, target_sequence_length,\n",
        "                  max_target_sentence_length,\n",
        "                  source_vocab_size, target_vocab_size,\n",
        "                  enc_embedding_size, dec_embedding_size,\n",
        "                  rnn_size, num_layers, target_vocab_to_int):\n",
        "    \"\"\"\n",
        "    Build the Sequence-to-Sequence part of the neural network\n",
        "    :param input_data: Input placeholder\n",
        "    :param target_data: Target placeholder\n",
        "    :param keep_prob: Dropout keep probability placeholder\n",
        "    :param batch_size: Batch Size\n",
        "    :param source_sequence_length: Sequence Lengths of source sequences in the batch\n",
        "    :param target_sequence_length: Sequence Lengths of target sequences in the batch\n",
        "    :param source_vocab_size: Source vocabulary size\n",
        "    :param target_vocab_size: Target vocabulary size\n",
        "    :param enc_embedding_size: Decoder embedding size\n",
        "    :param dec_embedding_size: Encoder embedding size\n",
        "    :param rnn_size: RNN Size\n",
        "    :param num_layers: Number of layers\n",
        "    :param target_vocab_to_int: Dictionary to go from the target words to an id\n",
        "    :return: Tuple of (Training BasicDecoderOutput, Inference BasicDecoderOutput)\n",
        "    \"\"\"\n",
        "    _, enc_state = encoding_layer(input_data, rnn_size, num_layers, keep_prob, \n",
        "                   source_sequence_length, source_vocab_size, \n",
        "                   enc_embedding_size)\n",
        "    \n",
        "    dec_input = process_decoder_input(target_data, target_vocab_to_int, batch_size)\n",
        "    \n",
        "    training_decoder_output, inference_decoder_output = decoding_layer(dec_input, enc_state,\n",
        "                   target_sequence_length, max_target_sentence_length,\n",
        "                   rnn_size,\n",
        "                   num_layers, target_vocab_to_int, target_vocab_size,\n",
        "                   batch_size, keep_prob, dec_embedding_size)\n",
        "    \n",
        "    return training_decoder_output, inference_decoder_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0zYaEy6xeXQ2",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Neural Network Training\n",
        "<p>Hyperparameters</p>\n",
        "<p>Tune the following parameters:</p>\n",
        "<ul><li> Set epochs to the number of epochs.</li>\n",
        "<li> Set batch_size to the batch size.</li>\n",
        "<li> Set rnn_size to the size of the RNNs.</li>\n",
        "<li>Set num_layers to the number of layers.</li>\n",
        "<li> Set encoding_embedding_size to the size of the embedding for the encoder.</li>\n",
        "<li> Set decoding_embedding_size to the size of the embedding for the decoder.</li>\n",
        "<li> Set learning_rate to the learning rate.</li>\n",
        "<li> Set keep_probability to the Dropout keep probability </li></ul>"
      ]
    },
    {
      "metadata": {
        "id": "7qHy0F02Bk4f",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Number of Epochs\n",
        "epochs = 10\n",
        "# Batch Size\n",
        "batch_size = 256\n",
        "# RNN Size\n",
        "rnn_size = 512\n",
        "# Number of Layers\n",
        "num_layers = 2\n",
        "# Embedding Size\n",
        "encoding_embedding_size = 256\n",
        "decoding_embedding_size = 256\n",
        "# Learning Rate\n",
        "learning_rate = 0.001\n",
        "# Dropout Keep Probability\n",
        "keep_probability = 0.75\n",
        "display_step = 10\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "G9wzygNAe9y_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Build the Graph\n",
        "<p>Building the graph using the neural network I have implemented.</p>"
      ]
    },
    {
      "metadata": {
        "id": "I1IyzGnIBk4p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.python.layers.core import Dense"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7z6D_vq2Bk4z",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "save_path = 'checkpoints/dev'\n",
        "(source_int_text, target_int_text), (source_vocab_to_int, target_vocab_to_int), _ = load_preprocess()\n",
        "max_target_sentence_length = max([len(sentence) for sentence in source_int_text])\n",
        "\n",
        "train_graph = tf.Graph()\n",
        "with train_graph.as_default():\n",
        "    input_data, targets, lr, keep_prob, target_sequence_length, max_target_sequence_length, source_sequence_length = model_inputs()\n",
        "\n",
        "    #sequence_length = tf.placeholder_with_default(max_target_sentence_length, None, name='sequence_length')\n",
        "    input_shape = tf.shape(input_data)\n",
        "\n",
        "    train_logits, inference_logits = seq2seq_model(tf.reverse(input_data, [-1]),\n",
        "                                                   targets,\n",
        "                                                   keep_prob,\n",
        "                                                   batch_size,\n",
        "                                                   source_sequence_length,\n",
        "                                                   target_sequence_length,\n",
        "                                                   max_target_sequence_length,\n",
        "                                                   len(source_vocab_to_int),\n",
        "                                                   len(target_vocab_to_int),\n",
        "                                                   encoding_embedding_size,\n",
        "                                                   decoding_embedding_size,\n",
        "                                                   rnn_size,\n",
        "                                                   num_layers,\n",
        "                                                   target_vocab_to_int)\n",
        "\n",
        "\n",
        "    training_logits = tf.identity(train_logits.rnn_output, name='logits')\n",
        "    inference_logits = tf.identity(inference_logits.sample_id, name='predictions')\n",
        "\n",
        "    masks = tf.sequence_mask(target_sequence_length, max_target_sequence_length, dtype=tf.float32, name='masks')\n",
        "\n",
        "    with tf.name_scope(\"optimization\"):\n",
        "        # Loss function\n",
        "        cost = tf.contrib.seq2seq.sequence_loss(\n",
        "            training_logits,\n",
        "            targets,\n",
        "            masks)\n",
        "\n",
        "        # Optimizer\n",
        "        optimizer = tf.train.AdamOptimizer(lr)\n",
        "\n",
        "        # Gradient Clipping\n",
        "        gradients = optimizer.compute_gradients(cost)\n",
        "        capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]\n",
        "        train_op = optimizer.apply_gradients(capped_gradients)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6IVYDmFEfSi4",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Train\n",
        "<p>Train the neural network on the preprocessed data. </p>"
      ]
    },
    {
      "metadata": {
        "id": "HetoefEIBk5C",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def pad_sentence_batch(sentence_batch, pad_int):\n",
        "    \"\"\"Pad sentences with <PAD> so that each sentence of a batch has the same length\"\"\"\n",
        "    max_sentence = max([len(sentence) for sentence in sentence_batch])\n",
        "    return [sentence + [pad_int] * (max_sentence - len(sentence)) for sentence in sentence_batch]\n",
        "\n",
        "\n",
        "def get_batches(sources, targets, batch_size, source_pad_int, target_pad_int):\n",
        "    \"\"\"Batch targets, sources, and the lengths of their sentences together\"\"\"\n",
        "    for batch_i in range(0, len(sources)//batch_size):\n",
        "        start_i = batch_i * batch_size\n",
        "\n",
        "        # Slice the right amount for the batch\n",
        "        sources_batch = sources[start_i:start_i + batch_size]\n",
        "        targets_batch = targets[start_i:start_i + batch_size]\n",
        "\n",
        "        # Pad\n",
        "        pad_sources_batch = np.array(pad_sentence_batch(sources_batch, source_pad_int))\n",
        "        pad_targets_batch = np.array(pad_sentence_batch(targets_batch, target_pad_int))\n",
        "\n",
        "        # Need the lengths for the _lengths parameters\n",
        "        pad_targets_lengths = []\n",
        "        for target in pad_targets_batch:\n",
        "            pad_targets_lengths.append(len(target))\n",
        "\n",
        "        pad_source_lengths = []\n",
        "        for source in pad_sources_batch:\n",
        "            pad_source_lengths.append(len(source))\n",
        "\n",
        "        yield pad_sources_batch, pad_targets_batch, pad_source_lengths, pad_targets_lengths"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Oe28MHVeBk5P",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 9456
        },
        "outputId": "42ce5979-5506-4e12-ae13-973b48291718"
      },
      "cell_type": "code",
      "source": [
        "def get_accuracy(target, logits):\n",
        "    \"\"\"\n",
        "    Calculate accuracy\n",
        "    \"\"\"\n",
        "    max_seq = max(target.shape[1], logits.shape[1])\n",
        "    if max_seq - target.shape[1]:\n",
        "        target = np.pad(\n",
        "            target,\n",
        "            [(0,0),(0,max_seq - target.shape[1])],\n",
        "            'constant')\n",
        "    if max_seq - logits.shape[1]:\n",
        "        logits = np.pad(\n",
        "            logits,\n",
        "            [(0,0),(0,max_seq - logits.shape[1])],\n",
        "            'constant')\n",
        "\n",
        "    return np.mean(np.equal(target, logits))\n",
        "\n",
        "# Split data to training and validation sets\n",
        "train_source = source_int_text[batch_size:]\n",
        "train_target = target_int_text[batch_size:]\n",
        "valid_source = source_int_text[:batch_size]\n",
        "valid_target = target_int_text[:batch_size]\n",
        "(valid_sources_batch, valid_targets_batch, valid_sources_lengths, valid_targets_lengths ) = next(get_batches(valid_source,\n",
        "                                                                                                             valid_target,\n",
        "                                                                                                             batch_size,\n",
        "                                                                                                             source_vocab_to_int['<PAD>'],\n",
        "                                                                                                             target_vocab_to_int['<PAD>']))                                                                                                  \n",
        "with tf.Session(graph=train_graph) as sess:\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "\n",
        "    for epoch_i in range(epochs):\n",
        "        for batch_i, (source_batch, target_batch, sources_lengths, targets_lengths) in enumerate(\n",
        "                get_batches(train_source, train_target, batch_size,\n",
        "                            source_vocab_to_int['<PAD>'],\n",
        "                            target_vocab_to_int['<PAD>'])):\n",
        "\n",
        "            _, loss = sess.run(\n",
        "                [train_op, cost],\n",
        "                {input_data: source_batch,\n",
        "                 targets: target_batch,\n",
        "                 lr: learning_rate,\n",
        "                 target_sequence_length: targets_lengths,\n",
        "                 source_sequence_length: sources_lengths,\n",
        "                 keep_prob: keep_probability})\n",
        "\n",
        "\n",
        "            if batch_i % display_step == 0 and batch_i > 0:\n",
        "\n",
        "\n",
        "                batch_train_logits = sess.run(\n",
        "                    inference_logits,\n",
        "                    {input_data: source_batch,\n",
        "                     source_sequence_length: sources_lengths,\n",
        "                     target_sequence_length: targets_lengths,\n",
        "                     keep_prob: 1.0})\n",
        "\n",
        "\n",
        "                batch_valid_logits = sess.run(\n",
        "                    inference_logits,\n",
        "                    {input_data: valid_sources_batch,\n",
        "                     source_sequence_length: valid_sources_lengths,\n",
        "                     target_sequence_length: valid_targets_lengths,\n",
        "                     keep_prob: 1.0})\n",
        "\n",
        "                train_acc = get_accuracy(target_batch, batch_train_logits)\n",
        "\n",
        "                valid_acc = get_accuracy(valid_targets_batch, batch_valid_logits)\n",
        "\n",
        "                print('Epoch {:>3} Batch {:>4}/{} - Train Accuracy: {:>6.4f}, Validation Accuracy: {:>6.4f}, Loss: {:>6.4f}'\n",
        "                      .format(epoch_i, batch_i, len(source_int_text) // batch_size, train_acc, valid_acc, loss))\n",
        "\n",
        "    # Save Model\n",
        "    saver = tf.train.Saver()\n",
        "    saver.save(sess, save_path)\n",
        "    print('Model Trained and Saved')"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch   0 Batch   10/538 - Train Accuracy: 0.3246, Validation Accuracy: 0.4110, Loss: 3.2427\n",
            "Epoch   0 Batch   20/538 - Train Accuracy: 0.4250, Validation Accuracy: 0.4657, Loss: 2.5849\n",
            "Epoch   0 Batch   30/538 - Train Accuracy: 0.4420, Validation Accuracy: 0.5096, Loss: 2.3628\n",
            "Epoch   0 Batch   40/538 - Train Accuracy: 0.4991, Validation Accuracy: 0.4959, Loss: 1.9260\n",
            "Epoch   0 Batch   50/538 - Train Accuracy: 0.4598, Validation Accuracy: 0.5034, Loss: 1.8367\n",
            "Epoch   0 Batch   60/538 - Train Accuracy: 0.4008, Validation Accuracy: 0.4600, Loss: 1.6491\n",
            "Epoch   0 Batch   70/538 - Train Accuracy: 0.4829, Validation Accuracy: 0.5055, Loss: 1.4523\n",
            "Epoch   0 Batch   80/538 - Train Accuracy: 0.4098, Validation Accuracy: 0.4801, Loss: 1.4065\n",
            "Epoch   0 Batch   90/538 - Train Accuracy: 0.5221, Validation Accuracy: 0.5194, Loss: 1.2515\n",
            "Epoch   0 Batch  100/538 - Train Accuracy: 0.5012, Validation Accuracy: 0.5286, Loss: 1.1526\n",
            "Epoch   0 Batch  110/538 - Train Accuracy: 0.4779, Validation Accuracy: 0.5204, Loss: 1.1128\n",
            "Epoch   0 Batch  120/538 - Train Accuracy: 0.5480, Validation Accuracy: 0.5579, Loss: 0.9768\n",
            "Epoch   0 Batch  130/538 - Train Accuracy: 0.5638, Validation Accuracy: 0.5625, Loss: 0.8876\n",
            "Epoch   0 Batch  140/538 - Train Accuracy: 0.5426, Validation Accuracy: 0.5795, Loss: 0.9418\n",
            "Epoch   0 Batch  150/538 - Train Accuracy: 0.5840, Validation Accuracy: 0.5771, Loss: 0.8279\n",
            "Epoch   0 Batch  160/538 - Train Accuracy: 0.5891, Validation Accuracy: 0.5964, Loss: 0.7567\n",
            "Epoch   0 Batch  170/538 - Train Accuracy: 0.6177, Validation Accuracy: 0.6032, Loss: 0.7431\n",
            "Epoch   0 Batch  180/538 - Train Accuracy: 0.6492, Validation Accuracy: 0.6154, Loss: 0.7011\n",
            "Epoch   0 Batch  190/538 - Train Accuracy: 0.5915, Validation Accuracy: 0.6033, Loss: 0.7148\n",
            "Epoch   0 Batch  200/538 - Train Accuracy: 0.6107, Validation Accuracy: 0.6046, Loss: 0.6789\n",
            "Epoch   0 Batch  210/538 - Train Accuracy: 0.5928, Validation Accuracy: 0.6060, Loss: 0.6471\n",
            "Epoch   0 Batch  220/538 - Train Accuracy: 0.5850, Validation Accuracy: 0.6181, Loss: 0.6293\n",
            "Epoch   0 Batch  230/538 - Train Accuracy: 0.6129, Validation Accuracy: 0.6332, Loss: 0.6257\n",
            "Epoch   0 Batch  240/538 - Train Accuracy: 0.6156, Validation Accuracy: 0.6112, Loss: 0.6224\n",
            "Epoch   0 Batch  250/538 - Train Accuracy: 0.6521, Validation Accuracy: 0.6445, Loss: 0.5895\n",
            "Epoch   0 Batch  260/538 - Train Accuracy: 0.6250, Validation Accuracy: 0.6497, Loss: 0.5632\n",
            "Epoch   0 Batch  270/538 - Train Accuracy: 0.6037, Validation Accuracy: 0.6275, Loss: 0.5606\n",
            "Epoch   0 Batch  280/538 - Train Accuracy: 0.6354, Validation Accuracy: 0.6314, Loss: 0.5079\n",
            "Epoch   0 Batch  290/538 - Train Accuracy: 0.6623, Validation Accuracy: 0.6689, Loss: 0.5054\n",
            "Epoch   0 Batch  300/538 - Train Accuracy: 0.6957, Validation Accuracy: 0.6942, Loss: 0.4707\n",
            "Epoch   0 Batch  310/538 - Train Accuracy: 0.7152, Validation Accuracy: 0.7001, Loss: 0.4732\n",
            "Epoch   0 Batch  320/538 - Train Accuracy: 0.7156, Validation Accuracy: 0.7262, Loss: 0.4330\n",
            "Epoch   0 Batch  330/538 - Train Accuracy: 0.7474, Validation Accuracy: 0.7402, Loss: 0.3974\n",
            "Epoch   0 Batch  340/538 - Train Accuracy: 0.7387, Validation Accuracy: 0.7259, Loss: 0.4042\n",
            "Epoch   0 Batch  350/538 - Train Accuracy: 0.7500, Validation Accuracy: 0.7459, Loss: 0.3883\n",
            "Epoch   0 Batch  360/538 - Train Accuracy: 0.7635, Validation Accuracy: 0.7422, Loss: 0.3579\n",
            "Epoch   0 Batch  370/538 - Train Accuracy: 0.7721, Validation Accuracy: 0.7775, Loss: 0.3337\n",
            "Epoch   0 Batch  380/538 - Train Accuracy: 0.7859, Validation Accuracy: 0.7853, Loss: 0.2957\n",
            "Epoch   0 Batch  390/538 - Train Accuracy: 0.8309, Validation Accuracy: 0.8093, Loss: 0.2689\n",
            "Epoch   0 Batch  400/538 - Train Accuracy: 0.8188, Validation Accuracy: 0.7981, Loss: 0.2648\n",
            "Epoch   0 Batch  410/538 - Train Accuracy: 0.8184, Validation Accuracy: 0.8043, Loss: 0.2530\n",
            "Epoch   0 Batch  420/538 - Train Accuracy: 0.8639, Validation Accuracy: 0.8086, Loss: 0.2194\n",
            "Epoch   0 Batch  430/538 - Train Accuracy: 0.8572, Validation Accuracy: 0.8269, Loss: 0.2062\n",
            "Epoch   0 Batch  440/538 - Train Accuracy: 0.8467, Validation Accuracy: 0.8395, Loss: 0.2172\n",
            "Epoch   0 Batch  450/538 - Train Accuracy: 0.8635, Validation Accuracy: 0.8420, Loss: 0.1945\n",
            "Epoch   0 Batch  460/538 - Train Accuracy: 0.8603, Validation Accuracy: 0.8368, Loss: 0.1832\n",
            "Epoch   0 Batch  470/538 - Train Accuracy: 0.8904, Validation Accuracy: 0.8512, Loss: 0.1521\n",
            "Epoch   0 Batch  480/538 - Train Accuracy: 0.9103, Validation Accuracy: 0.8654, Loss: 0.1456\n",
            "Epoch   0 Batch  490/538 - Train Accuracy: 0.8774, Validation Accuracy: 0.8777, Loss: 0.1323\n",
            "Epoch   0 Batch  500/538 - Train Accuracy: 0.9192, Validation Accuracy: 0.8716, Loss: 0.1168\n",
            "Epoch   0 Batch  510/538 - Train Accuracy: 0.9055, Validation Accuracy: 0.8810, Loss: 0.1144\n",
            "Epoch   0 Batch  520/538 - Train Accuracy: 0.8855, Validation Accuracy: 0.8734, Loss: 0.1299\n",
            "Epoch   0 Batch  530/538 - Train Accuracy: 0.8941, Validation Accuracy: 0.8805, Loss: 0.1275\n",
            "Epoch   1 Batch   10/538 - Train Accuracy: 0.9215, Validation Accuracy: 0.8691, Loss: 0.1177\n",
            "Epoch   1 Batch   20/538 - Train Accuracy: 0.9115, Validation Accuracy: 0.8901, Loss: 0.1009\n",
            "Epoch   1 Batch   30/538 - Train Accuracy: 0.9109, Validation Accuracy: 0.8844, Loss: 0.1151\n",
            "Epoch   1 Batch   40/538 - Train Accuracy: 0.9146, Validation Accuracy: 0.9087, Loss: 0.0762\n",
            "Epoch   1 Batch   50/538 - Train Accuracy: 0.9191, Validation Accuracy: 0.8933, Loss: 0.0927\n",
            "Epoch   1 Batch   60/538 - Train Accuracy: 0.9287, Validation Accuracy: 0.9100, Loss: 0.0839\n",
            "Epoch   1 Batch   70/538 - Train Accuracy: 0.8986, Validation Accuracy: 0.9007, Loss: 0.0809\n",
            "Epoch   1 Batch   80/538 - Train Accuracy: 0.9023, Validation Accuracy: 0.9148, Loss: 0.0821\n",
            "Epoch   1 Batch   90/538 - Train Accuracy: 0.9124, Validation Accuracy: 0.8901, Loss: 0.0855\n",
            "Epoch   1 Batch  100/538 - Train Accuracy: 0.9088, Validation Accuracy: 0.9176, Loss: 0.0704\n",
            "Epoch   1 Batch  110/538 - Train Accuracy: 0.9139, Validation Accuracy: 0.9141, Loss: 0.0780\n",
            "Epoch   1 Batch  120/538 - Train Accuracy: 0.9395, Validation Accuracy: 0.9102, Loss: 0.0603\n",
            "Epoch   1 Batch  130/538 - Train Accuracy: 0.9273, Validation Accuracy: 0.9016, Loss: 0.0656\n",
            "Epoch   1 Batch  140/538 - Train Accuracy: 0.9158, Validation Accuracy: 0.9057, Loss: 0.0866\n",
            "Epoch   1 Batch  150/538 - Train Accuracy: 0.9260, Validation Accuracy: 0.9116, Loss: 0.0626\n",
            "Epoch   1 Batch  160/538 - Train Accuracy: 0.9280, Validation Accuracy: 0.8968, Loss: 0.0583\n",
            "Epoch   1 Batch  170/538 - Train Accuracy: 0.9306, Validation Accuracy: 0.9290, Loss: 0.0709\n",
            "Epoch   1 Batch  180/538 - Train Accuracy: 0.9390, Validation Accuracy: 0.9196, Loss: 0.0697\n",
            "Epoch   1 Batch  190/538 - Train Accuracy: 0.9269, Validation Accuracy: 0.9118, Loss: 0.0757\n",
            "Epoch   1 Batch  200/538 - Train Accuracy: 0.9473, Validation Accuracy: 0.9096, Loss: 0.0506\n",
            "Epoch   1 Batch  210/538 - Train Accuracy: 0.9157, Validation Accuracy: 0.9178, Loss: 0.0565\n",
            "Epoch   1 Batch  220/538 - Train Accuracy: 0.9230, Validation Accuracy: 0.9315, Loss: 0.0625\n",
            "Epoch   1 Batch  230/538 - Train Accuracy: 0.9270, Validation Accuracy: 0.9272, Loss: 0.0560\n",
            "Epoch   1 Batch  240/538 - Train Accuracy: 0.9381, Validation Accuracy: 0.9354, Loss: 0.0599\n",
            "Epoch   1 Batch  250/538 - Train Accuracy: 0.9424, Validation Accuracy: 0.9228, Loss: 0.0534\n",
            "Epoch   1 Batch  260/538 - Train Accuracy: 0.9170, Validation Accuracy: 0.9453, Loss: 0.0579\n",
            "Epoch   1 Batch  270/538 - Train Accuracy: 0.9365, Validation Accuracy: 0.9268, Loss: 0.0508\n",
            "Epoch   1 Batch  280/538 - Train Accuracy: 0.9418, Validation Accuracy: 0.9292, Loss: 0.0476\n",
            "Epoch   1 Batch  290/538 - Train Accuracy: 0.9549, Validation Accuracy: 0.9286, Loss: 0.0488\n",
            "Epoch   1 Batch  300/538 - Train Accuracy: 0.9418, Validation Accuracy: 0.9371, Loss: 0.0513\n",
            "Epoch   1 Batch  310/538 - Train Accuracy: 0.9604, Validation Accuracy: 0.9515, Loss: 0.0575\n",
            "Epoch   1 Batch  320/538 - Train Accuracy: 0.9505, Validation Accuracy: 0.9421, Loss: 0.0444\n",
            "Epoch   1 Batch  330/538 - Train Accuracy: 0.9466, Validation Accuracy: 0.9277, Loss: 0.0490\n",
            "Epoch   1 Batch  340/538 - Train Accuracy: 0.9293, Validation Accuracy: 0.9418, Loss: 0.0519\n",
            "Epoch   1 Batch  350/538 - Train Accuracy: 0.9403, Validation Accuracy: 0.9375, Loss: 0.0540\n",
            "Epoch   1 Batch  360/538 - Train Accuracy: 0.9471, Validation Accuracy: 0.9444, Loss: 0.0452\n",
            "Epoch   1 Batch  370/538 - Train Accuracy: 0.9465, Validation Accuracy: 0.9411, Loss: 0.0473\n",
            "Epoch   1 Batch  380/538 - Train Accuracy: 0.9527, Validation Accuracy: 0.9284, Loss: 0.0410\n",
            "Epoch   1 Batch  390/538 - Train Accuracy: 0.9470, Validation Accuracy: 0.9366, Loss: 0.0359\n",
            "Epoch   1 Batch  400/538 - Train Accuracy: 0.9568, Validation Accuracy: 0.9400, Loss: 0.0432\n",
            "Epoch   1 Batch  410/538 - Train Accuracy: 0.9523, Validation Accuracy: 0.9384, Loss: 0.0460\n",
            "Epoch   1 Batch  420/538 - Train Accuracy: 0.9592, Validation Accuracy: 0.9391, Loss: 0.0419\n",
            "Epoch   1 Batch  430/538 - Train Accuracy: 0.9422, Validation Accuracy: 0.9403, Loss: 0.0400\n",
            "Epoch   1 Batch  440/538 - Train Accuracy: 0.9557, Validation Accuracy: 0.9441, Loss: 0.0434\n",
            "Epoch   1 Batch  450/538 - Train Accuracy: 0.9371, Validation Accuracy: 0.9318, Loss: 0.0561\n",
            "Epoch   1 Batch  460/538 - Train Accuracy: 0.9343, Validation Accuracy: 0.9411, Loss: 0.0449\n",
            "Epoch   1 Batch  470/538 - Train Accuracy: 0.9487, Validation Accuracy: 0.9528, Loss: 0.0384\n",
            "Epoch   1 Batch  480/538 - Train Accuracy: 0.9650, Validation Accuracy: 0.9499, Loss: 0.0368\n",
            "Epoch   1 Batch  490/538 - Train Accuracy: 0.9440, Validation Accuracy: 0.9528, Loss: 0.0383\n",
            "Epoch   1 Batch  500/538 - Train Accuracy: 0.9677, Validation Accuracy: 0.9352, Loss: 0.0284\n",
            "Epoch   1 Batch  510/538 - Train Accuracy: 0.9587, Validation Accuracy: 0.9556, Loss: 0.0367\n",
            "Epoch   1 Batch  520/538 - Train Accuracy: 0.9615, Validation Accuracy: 0.9418, Loss: 0.0347\n",
            "Epoch   1 Batch  530/538 - Train Accuracy: 0.9408, Validation Accuracy: 0.9570, Loss: 0.0411\n",
            "Epoch   2 Batch   10/538 - Train Accuracy: 0.9549, Validation Accuracy: 0.9370, Loss: 0.0388\n",
            "Epoch   2 Batch   20/538 - Train Accuracy: 0.9639, Validation Accuracy: 0.9466, Loss: 0.0419\n",
            "Epoch   2 Batch   30/538 - Train Accuracy: 0.9646, Validation Accuracy: 0.9357, Loss: 0.0424\n",
            "Epoch   2 Batch   40/538 - Train Accuracy: 0.9501, Validation Accuracy: 0.9435, Loss: 0.0278\n",
            "Epoch   2 Batch   50/538 - Train Accuracy: 0.9547, Validation Accuracy: 0.9375, Loss: 0.0343\n",
            "Epoch   2 Batch   60/538 - Train Accuracy: 0.9539, Validation Accuracy: 0.9375, Loss: 0.0354\n",
            "Epoch   2 Batch   70/538 - Train Accuracy: 0.9511, Validation Accuracy: 0.9526, Loss: 0.0337\n",
            "Epoch   2 Batch   80/538 - Train Accuracy: 0.9598, Validation Accuracy: 0.9455, Loss: 0.0375\n",
            "Epoch   2 Batch   90/538 - Train Accuracy: 0.9609, Validation Accuracy: 0.9585, Loss: 0.0387\n",
            "Epoch   2 Batch  100/538 - Train Accuracy: 0.9682, Validation Accuracy: 0.9558, Loss: 0.0310\n",
            "Epoch   2 Batch  110/538 - Train Accuracy: 0.9578, Validation Accuracy: 0.9451, Loss: 0.0328\n",
            "Epoch   2 Batch  120/538 - Train Accuracy: 0.9627, Validation Accuracy: 0.9508, Loss: 0.0244\n",
            "Epoch   2 Batch  130/538 - Train Accuracy: 0.9624, Validation Accuracy: 0.9560, Loss: 0.0339\n",
            "Epoch   2 Batch  140/538 - Train Accuracy: 0.9510, Validation Accuracy: 0.9561, Loss: 0.0406\n",
            "Epoch   2 Batch  150/538 - Train Accuracy: 0.9574, Validation Accuracy: 0.9487, Loss: 0.0296\n",
            "Epoch   2 Batch  160/538 - Train Accuracy: 0.9448, Validation Accuracy: 0.9522, Loss: 0.0293\n",
            "Epoch   2 Batch  170/538 - Train Accuracy: 0.9570, Validation Accuracy: 0.9531, Loss: 0.0358\n",
            "Epoch   2 Batch  180/538 - Train Accuracy: 0.9578, Validation Accuracy: 0.9474, Loss: 0.0298\n",
            "Epoch   2 Batch  190/538 - Train Accuracy: 0.9600, Validation Accuracy: 0.9457, Loss: 0.0433\n",
            "Epoch   2 Batch  200/538 - Train Accuracy: 0.9555, Validation Accuracy: 0.9519, Loss: 0.0259\n",
            "Epoch   2 Batch  210/538 - Train Accuracy: 0.9472, Validation Accuracy: 0.9510, Loss: 0.0345\n",
            "Epoch   2 Batch  220/538 - Train Accuracy: 0.9503, Validation Accuracy: 0.9487, Loss: 0.0310\n",
            "Epoch   2 Batch  230/538 - Train Accuracy: 0.9670, Validation Accuracy: 0.9485, Loss: 0.0265\n",
            "Epoch   2 Batch  240/538 - Train Accuracy: 0.9600, Validation Accuracy: 0.9524, Loss: 0.0262\n",
            "Epoch   2 Batch  250/538 - Train Accuracy: 0.9574, Validation Accuracy: 0.9430, Loss: 0.0273\n",
            "Epoch   2 Batch  260/538 - Train Accuracy: 0.9541, Validation Accuracy: 0.9545, Loss: 0.0319\n",
            "Epoch   2 Batch  270/538 - Train Accuracy: 0.9748, Validation Accuracy: 0.9654, Loss: 0.0232\n",
            "Epoch   2 Batch  280/538 - Train Accuracy: 0.9686, Validation Accuracy: 0.9513, Loss: 0.0271\n",
            "Epoch   2 Batch  290/538 - Train Accuracy: 0.9715, Validation Accuracy: 0.9611, Loss: 0.0265\n",
            "Epoch   2 Batch  300/538 - Train Accuracy: 0.9639, Validation Accuracy: 0.9455, Loss: 0.0312\n",
            "Epoch   2 Batch  310/538 - Train Accuracy: 0.9721, Validation Accuracy: 0.9572, Loss: 0.0362\n",
            "Epoch   2 Batch  320/538 - Train Accuracy: 0.9708, Validation Accuracy: 0.9652, Loss: 0.0272\n",
            "Epoch   2 Batch  330/538 - Train Accuracy: 0.9766, Validation Accuracy: 0.9512, Loss: 0.0301\n",
            "Epoch   2 Batch  340/538 - Train Accuracy: 0.9500, Validation Accuracy: 0.9432, Loss: 0.0291\n",
            "Epoch   2 Batch  350/538 - Train Accuracy: 0.9568, Validation Accuracy: 0.9517, Loss: 0.0331\n",
            "Epoch   2 Batch  360/538 - Train Accuracy: 0.9637, Validation Accuracy: 0.9593, Loss: 0.0236\n",
            "Epoch   2 Batch  370/538 - Train Accuracy: 0.9713, Validation Accuracy: 0.9471, Loss: 0.0257\n",
            "Epoch   2 Batch  380/538 - Train Accuracy: 0.9734, Validation Accuracy: 0.9547, Loss: 0.0219\n",
            "Epoch   2 Batch  390/538 - Train Accuracy: 0.9516, Validation Accuracy: 0.9482, Loss: 0.0240\n",
            "Epoch   2 Batch  400/538 - Train Accuracy: 0.9756, Validation Accuracy: 0.9579, Loss: 0.0283\n",
            "Epoch   2 Batch  410/538 - Train Accuracy: 0.9711, Validation Accuracy: 0.9611, Loss: 0.0255\n",
            "Epoch   2 Batch  420/538 - Train Accuracy: 0.9580, Validation Accuracy: 0.9583, Loss: 0.0252\n",
            "Epoch   2 Batch  430/538 - Train Accuracy: 0.9564, Validation Accuracy: 0.9574, Loss: 0.0268\n",
            "Epoch   2 Batch  440/538 - Train Accuracy: 0.9682, Validation Accuracy: 0.9625, Loss: 0.0265\n",
            "Epoch   2 Batch  450/538 - Train Accuracy: 0.9479, Validation Accuracy: 0.9567, Loss: 0.0377\n",
            "Epoch   2 Batch  460/538 - Train Accuracy: 0.9699, Validation Accuracy: 0.9615, Loss: 0.0289\n",
            "Epoch   2 Batch  470/538 - Train Accuracy: 0.9628, Validation Accuracy: 0.9648, Loss: 0.0262\n",
            "Epoch   2 Batch  480/538 - Train Accuracy: 0.9714, Validation Accuracy: 0.9393, Loss: 0.0232\n",
            "Epoch   2 Batch  490/538 - Train Accuracy: 0.9561, Validation Accuracy: 0.9590, Loss: 0.0238\n",
            "Epoch   2 Batch  500/538 - Train Accuracy: 0.9840, Validation Accuracy: 0.9508, Loss: 0.0165\n",
            "Epoch   2 Batch  510/538 - Train Accuracy: 0.9738, Validation Accuracy: 0.9542, Loss: 0.0235\n",
            "Epoch   2 Batch  520/538 - Train Accuracy: 0.9734, Validation Accuracy: 0.9625, Loss: 0.0254\n",
            "Epoch   2 Batch  530/538 - Train Accuracy: 0.9582, Validation Accuracy: 0.9590, Loss: 0.0296\n",
            "Epoch   3 Batch   10/538 - Train Accuracy: 0.9625, Validation Accuracy: 0.9631, Loss: 0.0255\n",
            "Epoch   3 Batch   20/538 - Train Accuracy: 0.9814, Validation Accuracy: 0.9718, Loss: 0.0288\n",
            "Epoch   3 Batch   30/538 - Train Accuracy: 0.9643, Validation Accuracy: 0.9446, Loss: 0.0243\n",
            "Epoch   3 Batch   40/538 - Train Accuracy: 0.9705, Validation Accuracy: 0.9592, Loss: 0.0168\n",
            "Epoch   3 Batch   50/538 - Train Accuracy: 0.9641, Validation Accuracy: 0.9592, Loss: 0.0247\n",
            "Epoch   3 Batch   60/538 - Train Accuracy: 0.9787, Validation Accuracy: 0.9679, Loss: 0.0263\n",
            "Epoch   3 Batch   70/538 - Train Accuracy: 0.9751, Validation Accuracy: 0.9703, Loss: 0.0227\n",
            "Epoch   3 Batch   80/538 - Train Accuracy: 0.9572, Validation Accuracy: 0.9609, Loss: 0.0227\n",
            "Epoch   3 Batch   90/538 - Train Accuracy: 0.9714, Validation Accuracy: 0.9700, Loss: 0.0265\n",
            "Epoch   3 Batch  100/538 - Train Accuracy: 0.9771, Validation Accuracy: 0.9650, Loss: 0.0180\n",
            "Epoch   3 Batch  110/538 - Train Accuracy: 0.9691, Validation Accuracy: 0.9604, Loss: 0.0207\n",
            "Epoch   3 Batch  120/538 - Train Accuracy: 0.9746, Validation Accuracy: 0.9695, Loss: 0.0162\n",
            "Epoch   3 Batch  130/538 - Train Accuracy: 0.9682, Validation Accuracy: 0.9606, Loss: 0.0240\n",
            "Epoch   3 Batch  140/538 - Train Accuracy: 0.9664, Validation Accuracy: 0.9498, Loss: 0.0294\n",
            "Epoch   3 Batch  150/538 - Train Accuracy: 0.9746, Validation Accuracy: 0.9567, Loss: 0.0201\n",
            "Epoch   3 Batch  160/538 - Train Accuracy: 0.9682, Validation Accuracy: 0.9616, Loss: 0.0201\n",
            "Epoch   3 Batch  170/538 - Train Accuracy: 0.9715, Validation Accuracy: 0.9457, Loss: 0.0224\n",
            "Epoch   3 Batch  180/538 - Train Accuracy: 0.9701, Validation Accuracy: 0.9689, Loss: 0.0226\n",
            "Epoch   3 Batch  190/538 - Train Accuracy: 0.9693, Validation Accuracy: 0.9608, Loss: 0.0315\n",
            "Epoch   3 Batch  200/538 - Train Accuracy: 0.9834, Validation Accuracy: 0.9656, Loss: 0.0159\n",
            "Epoch   3 Batch  210/538 - Train Accuracy: 0.9792, Validation Accuracy: 0.9570, Loss: 0.0189\n",
            "Epoch   3 Batch  220/538 - Train Accuracy: 0.9632, Validation Accuracy: 0.9650, Loss: 0.0224\n",
            "Epoch   3 Batch  230/538 - Train Accuracy: 0.9740, Validation Accuracy: 0.9611, Loss: 0.0194\n",
            "Epoch   3 Batch  240/538 - Train Accuracy: 0.9707, Validation Accuracy: 0.9609, Loss: 0.0196\n",
            "Epoch   3 Batch  250/538 - Train Accuracy: 0.9818, Validation Accuracy: 0.9602, Loss: 0.0194\n",
            "Epoch   3 Batch  260/538 - Train Accuracy: 0.9609, Validation Accuracy: 0.9581, Loss: 0.0217\n",
            "Epoch   3 Batch  270/538 - Train Accuracy: 0.9828, Validation Accuracy: 0.9593, Loss: 0.0198\n",
            "Epoch   3 Batch  280/538 - Train Accuracy: 0.9847, Validation Accuracy: 0.9663, Loss: 0.0168\n",
            "Epoch   3 Batch  290/538 - Train Accuracy: 0.9797, Validation Accuracy: 0.9590, Loss: 0.0177\n",
            "Epoch   3 Batch  300/538 - Train Accuracy: 0.9784, Validation Accuracy: 0.9581, Loss: 0.0174\n",
            "Epoch   3 Batch  310/538 - Train Accuracy: 0.9738, Validation Accuracy: 0.9663, Loss: 0.0260\n",
            "Epoch   3 Batch  320/538 - Train Accuracy: 0.9719, Validation Accuracy: 0.9604, Loss: 0.0190\n",
            "Epoch   3 Batch  330/538 - Train Accuracy: 0.9781, Validation Accuracy: 0.9735, Loss: 0.0189\n",
            "Epoch   3 Batch  340/538 - Train Accuracy: 0.9674, Validation Accuracy: 0.9604, Loss: 0.0203\n",
            "Epoch   3 Batch  350/538 - Train Accuracy: 0.9758, Validation Accuracy: 0.9606, Loss: 0.0233\n",
            "Epoch   3 Batch  360/538 - Train Accuracy: 0.9896, Validation Accuracy: 0.9727, Loss: 0.0143\n",
            "Epoch   3 Batch  370/538 - Train Accuracy: 0.9787, Validation Accuracy: 0.9609, Loss: 0.0170\n",
            "Epoch   3 Batch  380/538 - Train Accuracy: 0.9842, Validation Accuracy: 0.9702, Loss: 0.0175\n",
            "Epoch   3 Batch  390/538 - Train Accuracy: 0.9596, Validation Accuracy: 0.9609, Loss: 0.0201\n",
            "Epoch   3 Batch  400/538 - Train Accuracy: 0.9747, Validation Accuracy: 0.9615, Loss: 0.0190\n",
            "Epoch   3 Batch  410/538 - Train Accuracy: 0.9891, Validation Accuracy: 0.9656, Loss: 0.0195\n",
            "Epoch   3 Batch  420/538 - Train Accuracy: 0.9723, Validation Accuracy: 0.9590, Loss: 0.0181\n",
            "Epoch   3 Batch  430/538 - Train Accuracy: 0.9754, Validation Accuracy: 0.9581, Loss: 0.0190\n",
            "Epoch   3 Batch  440/538 - Train Accuracy: 0.9729, Validation Accuracy: 0.9611, Loss: 0.0241\n",
            "Epoch   3 Batch  450/538 - Train Accuracy: 0.9600, Validation Accuracy: 0.9588, Loss: 0.0298\n",
            "Epoch   3 Batch  460/538 - Train Accuracy: 0.9639, Validation Accuracy: 0.9606, Loss: 0.0259\n",
            "Epoch   3 Batch  470/538 - Train Accuracy: 0.9715, Validation Accuracy: 0.9590, Loss: 0.0243\n",
            "Epoch   3 Batch  480/538 - Train Accuracy: 0.9821, Validation Accuracy: 0.9538, Loss: 0.0179\n",
            "Epoch   3 Batch  490/538 - Train Accuracy: 0.9669, Validation Accuracy: 0.9632, Loss: 0.0203\n",
            "Epoch   3 Batch  500/538 - Train Accuracy: 0.9892, Validation Accuracy: 0.9661, Loss: 0.0123\n",
            "Epoch   3 Batch  510/538 - Train Accuracy: 0.9834, Validation Accuracy: 0.9565, Loss: 0.0176\n",
            "Epoch   3 Batch  520/538 - Train Accuracy: 0.9809, Validation Accuracy: 0.9577, Loss: 0.0204\n",
            "Epoch   3 Batch  530/538 - Train Accuracy: 0.9738, Validation Accuracy: 0.9693, Loss: 0.0226\n",
            "Epoch   4 Batch   10/538 - Train Accuracy: 0.9738, Validation Accuracy: 0.9544, Loss: 0.0201\n",
            "Epoch   4 Batch   20/538 - Train Accuracy: 0.9749, Validation Accuracy: 0.9815, Loss: 0.0185\n",
            "Epoch   4 Batch   30/538 - Train Accuracy: 0.9611, Validation Accuracy: 0.9652, Loss: 0.0205\n",
            "Epoch   4 Batch   40/538 - Train Accuracy: 0.9792, Validation Accuracy: 0.9703, Loss: 0.0126\n",
            "Epoch   4 Batch   50/538 - Train Accuracy: 0.9750, Validation Accuracy: 0.9734, Loss: 0.0180\n",
            "Epoch   4 Batch   60/538 - Train Accuracy: 0.9844, Validation Accuracy: 0.9766, Loss: 0.0190\n",
            "Epoch   4 Batch   70/538 - Train Accuracy: 0.9740, Validation Accuracy: 0.9750, Loss: 0.0138\n",
            "Epoch   4 Batch   80/538 - Train Accuracy: 0.9906, Validation Accuracy: 0.9712, Loss: 0.0139\n",
            "Epoch   4 Batch   90/538 - Train Accuracy: 0.9842, Validation Accuracy: 0.9570, Loss: 0.0160\n",
            "Epoch   4 Batch  100/538 - Train Accuracy: 0.9838, Validation Accuracy: 0.9632, Loss: 0.0129\n",
            "Epoch   4 Batch  110/538 - Train Accuracy: 0.9785, Validation Accuracy: 0.9597, Loss: 0.0192\n",
            "Epoch   4 Batch  120/538 - Train Accuracy: 0.9926, Validation Accuracy: 0.9572, Loss: 0.0110\n",
            "Epoch   4 Batch  130/538 - Train Accuracy: 0.9821, Validation Accuracy: 0.9680, Loss: 0.0173\n",
            "Epoch   4 Batch  140/538 - Train Accuracy: 0.9701, Validation Accuracy: 0.9657, Loss: 0.0227\n",
            "Epoch   4 Batch  150/538 - Train Accuracy: 0.9844, Validation Accuracy: 0.9631, Loss: 0.0183\n",
            "Epoch   4 Batch  160/538 - Train Accuracy: 0.9684, Validation Accuracy: 0.9641, Loss: 0.0156\n",
            "Epoch   4 Batch  170/538 - Train Accuracy: 0.9676, Validation Accuracy: 0.9622, Loss: 0.0192\n",
            "Epoch   4 Batch  180/538 - Train Accuracy: 0.9771, Validation Accuracy: 0.9673, Loss: 0.0177\n",
            "Epoch   4 Batch  190/538 - Train Accuracy: 0.9665, Validation Accuracy: 0.9673, Loss: 0.0228\n",
            "Epoch   4 Batch  200/538 - Train Accuracy: 0.9787, Validation Accuracy: 0.9672, Loss: 0.0127\n",
            "Epoch   4 Batch  210/538 - Train Accuracy: 0.9825, Validation Accuracy: 0.9581, Loss: 0.0200\n",
            "Epoch   4 Batch  220/538 - Train Accuracy: 0.9738, Validation Accuracy: 0.9684, Loss: 0.0169\n",
            "Epoch   4 Batch  230/538 - Train Accuracy: 0.9844, Validation Accuracy: 0.9688, Loss: 0.0149\n",
            "Epoch   4 Batch  240/538 - Train Accuracy: 0.9779, Validation Accuracy: 0.9672, Loss: 0.0173\n",
            "Epoch   4 Batch  250/538 - Train Accuracy: 0.9850, Validation Accuracy: 0.9585, Loss: 0.0177\n",
            "Epoch   4 Batch  260/538 - Train Accuracy: 0.9615, Validation Accuracy: 0.9640, Loss: 0.0175\n",
            "Epoch   4 Batch  270/538 - Train Accuracy: 0.9879, Validation Accuracy: 0.9583, Loss: 0.0144\n",
            "Epoch   4 Batch  280/538 - Train Accuracy: 0.9859, Validation Accuracy: 0.9703, Loss: 0.0135\n",
            "Epoch   4 Batch  290/538 - Train Accuracy: 0.9869, Validation Accuracy: 0.9663, Loss: 0.0142\n",
            "Epoch   4 Batch  300/538 - Train Accuracy: 0.9753, Validation Accuracy: 0.9647, Loss: 0.0160\n",
            "Epoch   4 Batch  310/538 - Train Accuracy: 0.9793, Validation Accuracy: 0.9622, Loss: 0.0210\n",
            "Epoch   4 Batch  320/538 - Train Accuracy: 0.9771, Validation Accuracy: 0.9744, Loss: 0.0138\n",
            "Epoch   4 Batch  330/538 - Train Accuracy: 0.9892, Validation Accuracy: 0.9641, Loss: 0.0139\n",
            "Epoch   4 Batch  340/538 - Train Accuracy: 0.9838, Validation Accuracy: 0.9716, Loss: 0.0151\n",
            "Epoch   4 Batch  350/538 - Train Accuracy: 0.9853, Validation Accuracy: 0.9688, Loss: 0.0180\n",
            "Epoch   4 Batch  360/538 - Train Accuracy: 0.9877, Validation Accuracy: 0.9679, Loss: 0.0122\n",
            "Epoch   4 Batch  370/538 - Train Accuracy: 0.9812, Validation Accuracy: 0.9743, Loss: 0.0169\n",
            "Epoch   4 Batch  380/538 - Train Accuracy: 0.9850, Validation Accuracy: 0.9670, Loss: 0.0132\n",
            "Epoch   4 Batch  390/538 - Train Accuracy: 0.9751, Validation Accuracy: 0.9677, Loss: 0.0156\n",
            "Epoch   4 Batch  400/538 - Train Accuracy: 0.9831, Validation Accuracy: 0.9771, Loss: 0.0127\n",
            "Epoch   4 Batch  410/538 - Train Accuracy: 0.9883, Validation Accuracy: 0.9737, Loss: 0.0107\n",
            "Epoch   4 Batch  420/538 - Train Accuracy: 0.9850, Validation Accuracy: 0.9712, Loss: 0.0168\n",
            "Epoch   4 Batch  430/538 - Train Accuracy: 0.9729, Validation Accuracy: 0.9583, Loss: 0.0156\n",
            "Epoch   4 Batch  440/538 - Train Accuracy: 0.9828, Validation Accuracy: 0.9602, Loss: 0.0153\n",
            "Epoch   4 Batch  450/538 - Train Accuracy: 0.9580, Validation Accuracy: 0.9711, Loss: 0.0233\n",
            "Epoch   4 Batch  460/538 - Train Accuracy: 0.9831, Validation Accuracy: 0.9686, Loss: 0.0160\n",
            "Epoch   4 Batch  470/538 - Train Accuracy: 0.9792, Validation Accuracy: 0.9567, Loss: 0.0137\n",
            "Epoch   4 Batch  480/538 - Train Accuracy: 0.9929, Validation Accuracy: 0.9611, Loss: 0.0120\n",
            "Epoch   4 Batch  490/538 - Train Accuracy: 0.9727, Validation Accuracy: 0.9652, Loss: 0.0158\n",
            "Epoch   4 Batch  500/538 - Train Accuracy: 0.9941, Validation Accuracy: 0.9737, Loss: 0.0073\n",
            "Epoch   4 Batch  510/538 - Train Accuracy: 0.9888, Validation Accuracy: 0.9604, Loss: 0.0122\n",
            "Epoch   4 Batch  520/538 - Train Accuracy: 0.9809, Validation Accuracy: 0.9682, Loss: 0.0188\n",
            "Epoch   4 Batch  530/538 - Train Accuracy: 0.9789, Validation Accuracy: 0.9657, Loss: 0.0179\n",
            "Epoch   5 Batch   10/538 - Train Accuracy: 0.9758, Validation Accuracy: 0.9664, Loss: 0.0139\n",
            "Epoch   5 Batch   20/538 - Train Accuracy: 0.9797, Validation Accuracy: 0.9712, Loss: 0.0180\n",
            "Epoch   5 Batch   30/538 - Train Accuracy: 0.9814, Validation Accuracy: 0.9673, Loss: 0.0153\n",
            "Epoch   5 Batch   40/538 - Train Accuracy: 0.9847, Validation Accuracy: 0.9680, Loss: 0.0115\n",
            "Epoch   5 Batch   50/538 - Train Accuracy: 0.9799, Validation Accuracy: 0.9689, Loss: 0.0159\n",
            "Epoch   5 Batch   60/538 - Train Accuracy: 0.9840, Validation Accuracy: 0.9806, Loss: 0.0159\n",
            "Epoch   5 Batch   70/538 - Train Accuracy: 0.9743, Validation Accuracy: 0.9698, Loss: 0.0118\n",
            "Epoch   5 Batch   80/538 - Train Accuracy: 0.9832, Validation Accuracy: 0.9751, Loss: 0.0124\n",
            "Epoch   5 Batch   90/538 - Train Accuracy: 0.9795, Validation Accuracy: 0.9538, Loss: 0.0133\n",
            "Epoch   5 Batch  100/538 - Train Accuracy: 0.9852, Validation Accuracy: 0.9746, Loss: 0.0101\n",
            "Epoch   5 Batch  110/538 - Train Accuracy: 0.9807, Validation Accuracy: 0.9744, Loss: 0.0140\n",
            "Epoch   5 Batch  120/538 - Train Accuracy: 0.9889, Validation Accuracy: 0.9728, Loss: 0.0104\n",
            "Epoch   5 Batch  130/538 - Train Accuracy: 0.9898, Validation Accuracy: 0.9695, Loss: 0.0156\n",
            "Epoch   5 Batch  140/538 - Train Accuracy: 0.9846, Validation Accuracy: 0.9700, Loss: 0.0170\n",
            "Epoch   5 Batch  150/538 - Train Accuracy: 0.9848, Validation Accuracy: 0.9735, Loss: 0.0139\n",
            "Epoch   5 Batch  160/538 - Train Accuracy: 0.9805, Validation Accuracy: 0.9703, Loss: 0.0170\n",
            "Epoch   5 Batch  170/538 - Train Accuracy: 0.9760, Validation Accuracy: 0.9664, Loss: 0.0156\n",
            "Epoch   5 Batch  180/538 - Train Accuracy: 0.9708, Validation Accuracy: 0.9684, Loss: 0.0146\n",
            "Epoch   5 Batch  190/538 - Train Accuracy: 0.9808, Validation Accuracy: 0.9725, Loss: 0.0186\n",
            "Epoch   5 Batch  200/538 - Train Accuracy: 0.9809, Validation Accuracy: 0.9632, Loss: 0.0094\n",
            "Epoch   5 Batch  210/538 - Train Accuracy: 0.9900, Validation Accuracy: 0.9743, Loss: 0.0160\n",
            "Epoch   5 Batch  220/538 - Train Accuracy: 0.9764, Validation Accuracy: 0.9597, Loss: 0.0140\n",
            "Epoch   5 Batch  230/538 - Train Accuracy: 0.9828, Validation Accuracy: 0.9691, Loss: 0.0115\n",
            "Epoch   5 Batch  240/538 - Train Accuracy: 0.9791, Validation Accuracy: 0.9785, Loss: 0.0126\n",
            "Epoch   5 Batch  250/538 - Train Accuracy: 0.9920, Validation Accuracy: 0.9769, Loss: 0.0140\n",
            "Epoch   5 Batch  260/538 - Train Accuracy: 0.9702, Validation Accuracy: 0.9570, Loss: 0.0155\n",
            "Epoch   5 Batch  270/538 - Train Accuracy: 0.9900, Validation Accuracy: 0.9629, Loss: 0.0110\n",
            "Epoch   5 Batch  280/538 - Train Accuracy: 0.9797, Validation Accuracy: 0.9688, Loss: 0.0123\n",
            "Epoch   5 Batch  290/538 - Train Accuracy: 0.9924, Validation Accuracy: 0.9734, Loss: 0.0113\n",
            "Epoch   5 Batch  300/538 - Train Accuracy: 0.9913, Validation Accuracy: 0.9716, Loss: 0.0144\n",
            "Epoch   5 Batch  310/538 - Train Accuracy: 0.9875, Validation Accuracy: 0.9645, Loss: 0.0150\n",
            "Epoch   5 Batch  320/538 - Train Accuracy: 0.9834, Validation Accuracy: 0.9759, Loss: 0.0107\n",
            "Epoch   5 Batch  330/538 - Train Accuracy: 0.9900, Validation Accuracy: 0.9661, Loss: 0.0131\n",
            "Epoch   5 Batch  340/538 - Train Accuracy: 0.9816, Validation Accuracy: 0.9677, Loss: 0.0115\n",
            "Epoch   5 Batch  350/538 - Train Accuracy: 0.9797, Validation Accuracy: 0.9643, Loss: 0.0120\n",
            "Epoch   5 Batch  360/538 - Train Accuracy: 0.9910, Validation Accuracy: 0.9705, Loss: 0.0108\n",
            "Epoch   5 Batch  370/538 - Train Accuracy: 0.9844, Validation Accuracy: 0.9750, Loss: 0.0099\n",
            "Epoch   5 Batch  380/538 - Train Accuracy: 0.9871, Validation Accuracy: 0.9627, Loss: 0.0148\n",
            "Epoch   5 Batch  390/538 - Train Accuracy: 0.9734, Validation Accuracy: 0.9608, Loss: 0.0124\n",
            "Epoch   5 Batch  400/538 - Train Accuracy: 0.9859, Validation Accuracy: 0.9748, Loss: 0.0117\n",
            "Epoch   5 Batch  410/538 - Train Accuracy: 0.9895, Validation Accuracy: 0.9723, Loss: 0.0096\n",
            "Epoch   5 Batch  420/538 - Train Accuracy: 0.9871, Validation Accuracy: 0.9689, Loss: 0.0142\n",
            "Epoch   5 Batch  430/538 - Train Accuracy: 0.9814, Validation Accuracy: 0.9659, Loss: 0.0114\n",
            "Epoch   5 Batch  440/538 - Train Accuracy: 0.9820, Validation Accuracy: 0.9656, Loss: 0.0140\n",
            "Epoch   5 Batch  450/538 - Train Accuracy: 0.9784, Validation Accuracy: 0.9632, Loss: 0.0183\n",
            "Epoch   5 Batch  460/538 - Train Accuracy: 0.9855, Validation Accuracy: 0.9703, Loss: 0.0123\n",
            "Epoch   5 Batch  470/538 - Train Accuracy: 0.9833, Validation Accuracy: 0.9703, Loss: 0.0130\n",
            "Epoch   5 Batch  480/538 - Train Accuracy: 0.9883, Validation Accuracy: 0.9656, Loss: 0.0120\n",
            "Epoch   5 Batch  490/538 - Train Accuracy: 0.9801, Validation Accuracy: 0.9725, Loss: 0.0126\n",
            "Epoch   5 Batch  500/538 - Train Accuracy: 0.9938, Validation Accuracy: 0.9599, Loss: 0.0077\n",
            "Epoch   5 Batch  510/538 - Train Accuracy: 0.9840, Validation Accuracy: 0.9682, Loss: 0.0119\n",
            "Epoch   5 Batch  520/538 - Train Accuracy: 0.9791, Validation Accuracy: 0.9583, Loss: 0.0146\n",
            "Epoch   5 Batch  530/538 - Train Accuracy: 0.9836, Validation Accuracy: 0.9672, Loss: 0.0117\n",
            "Epoch   6 Batch   10/538 - Train Accuracy: 0.9836, Validation Accuracy: 0.9698, Loss: 0.0118\n",
            "Epoch   6 Batch   20/538 - Train Accuracy: 0.9788, Validation Accuracy: 0.9732, Loss: 0.0163\n",
            "Epoch   6 Batch   30/538 - Train Accuracy: 0.9766, Validation Accuracy: 0.9748, Loss: 0.0148\n",
            "Epoch   6 Batch   40/538 - Train Accuracy: 0.9890, Validation Accuracy: 0.9615, Loss: 0.0086\n",
            "Epoch   6 Batch   50/538 - Train Accuracy: 0.9793, Validation Accuracy: 0.9757, Loss: 0.0121\n",
            "Epoch   6 Batch   60/538 - Train Accuracy: 0.9826, Validation Accuracy: 0.9785, Loss: 0.0125\n",
            "Epoch   6 Batch   70/538 - Train Accuracy: 0.9825, Validation Accuracy: 0.9645, Loss: 0.0092\n",
            "Epoch   6 Batch   80/538 - Train Accuracy: 0.9871, Validation Accuracy: 0.9652, Loss: 0.0104\n",
            "Epoch   6 Batch   90/538 - Train Accuracy: 0.9870, Validation Accuracy: 0.9643, Loss: 0.0125\n",
            "Epoch   6 Batch  100/538 - Train Accuracy: 0.9906, Validation Accuracy: 0.9778, Loss: 0.0106\n",
            "Epoch   6 Batch  110/538 - Train Accuracy: 0.9820, Validation Accuracy: 0.9869, Loss: 0.0113\n",
            "Epoch   6 Batch  120/538 - Train Accuracy: 0.9809, Validation Accuracy: 0.9785, Loss: 0.0107\n",
            "Epoch   6 Batch  130/538 - Train Accuracy: 0.9851, Validation Accuracy: 0.9719, Loss: 0.0147\n",
            "Epoch   6 Batch  140/538 - Train Accuracy: 0.9863, Validation Accuracy: 0.9767, Loss: 0.0128\n",
            "Epoch   6 Batch  150/538 - Train Accuracy: 0.9828, Validation Accuracy: 0.9769, Loss: 0.0145\n",
            "Epoch   6 Batch  160/538 - Train Accuracy: 0.9834, Validation Accuracy: 0.9771, Loss: 0.0123\n",
            "Epoch   6 Batch  170/538 - Train Accuracy: 0.9771, Validation Accuracy: 0.9673, Loss: 0.0137\n",
            "Epoch   6 Batch  180/538 - Train Accuracy: 0.9799, Validation Accuracy: 0.9654, Loss: 0.0113\n",
            "Epoch   6 Batch  190/538 - Train Accuracy: 0.9877, Validation Accuracy: 0.9819, Loss: 0.0162\n",
            "Epoch   6 Batch  200/538 - Train Accuracy: 0.9869, Validation Accuracy: 0.9792, Loss: 0.0089\n",
            "Epoch   6 Batch  210/538 - Train Accuracy: 0.9929, Validation Accuracy: 0.9707, Loss: 0.0101\n",
            "Epoch   6 Batch  220/538 - Train Accuracy: 0.9955, Validation Accuracy: 0.9716, Loss: 0.0121\n",
            "Epoch   6 Batch  230/538 - Train Accuracy: 0.9779, Validation Accuracy: 0.9670, Loss: 0.0137\n",
            "Epoch   6 Batch  240/538 - Train Accuracy: 0.9773, Validation Accuracy: 0.9709, Loss: 0.0135\n",
            "Epoch   6 Batch  250/538 - Train Accuracy: 0.9934, Validation Accuracy: 0.9638, Loss: 0.0117\n",
            "Epoch   6 Batch  260/538 - Train Accuracy: 0.9823, Validation Accuracy: 0.9719, Loss: 0.0165\n",
            "Epoch   6 Batch  270/538 - Train Accuracy: 0.9867, Validation Accuracy: 0.9625, Loss: 0.0108\n",
            "Epoch   6 Batch  280/538 - Train Accuracy: 0.9900, Validation Accuracy: 0.9709, Loss: 0.0086\n",
            "Epoch   6 Batch  290/538 - Train Accuracy: 0.9918, Validation Accuracy: 0.9709, Loss: 0.0107\n",
            "Epoch   6 Batch  300/538 - Train Accuracy: 0.9885, Validation Accuracy: 0.9712, Loss: 0.0120\n",
            "Epoch   6 Batch  310/538 - Train Accuracy: 0.9818, Validation Accuracy: 0.9712, Loss: 0.0119\n",
            "Epoch   6 Batch  320/538 - Train Accuracy: 0.9820, Validation Accuracy: 0.9741, Loss: 0.0117\n",
            "Epoch   6 Batch  330/538 - Train Accuracy: 0.9924, Validation Accuracy: 0.9650, Loss: 0.0099\n",
            "Epoch   6 Batch  340/538 - Train Accuracy: 0.9955, Validation Accuracy: 0.9656, Loss: 0.0103\n",
            "Epoch   6 Batch  350/538 - Train Accuracy: 0.9924, Validation Accuracy: 0.9593, Loss: 0.0114\n",
            "Epoch   6 Batch  360/538 - Train Accuracy: 0.9893, Validation Accuracy: 0.9723, Loss: 0.0104\n",
            "Epoch   6 Batch  370/538 - Train Accuracy: 0.9879, Validation Accuracy: 0.9785, Loss: 0.0093\n",
            "Epoch   6 Batch  380/538 - Train Accuracy: 0.9951, Validation Accuracy: 0.9773, Loss: 0.0084\n",
            "Epoch   6 Batch  390/538 - Train Accuracy: 0.9859, Validation Accuracy: 0.9766, Loss: 0.0108\n",
            "Epoch   6 Batch  400/538 - Train Accuracy: 0.9860, Validation Accuracy: 0.9762, Loss: 0.0107\n",
            "Epoch   6 Batch  410/538 - Train Accuracy: 0.9840, Validation Accuracy: 0.9735, Loss: 0.0082\n",
            "Epoch   6 Batch  420/538 - Train Accuracy: 0.9824, Validation Accuracy: 0.9673, Loss: 0.0127\n",
            "Epoch   6 Batch  430/538 - Train Accuracy: 0.9854, Validation Accuracy: 0.9709, Loss: 0.0116\n",
            "Epoch   6 Batch  440/538 - Train Accuracy: 0.9873, Validation Accuracy: 0.9657, Loss: 0.0132\n",
            "Epoch   6 Batch  450/538 - Train Accuracy: 0.9823, Validation Accuracy: 0.9757, Loss: 0.0176\n",
            "Epoch   6 Batch  460/538 - Train Accuracy: 0.9829, Validation Accuracy: 0.9737, Loss: 0.0126\n",
            "Epoch   6 Batch  470/538 - Train Accuracy: 0.9840, Validation Accuracy: 0.9705, Loss: 0.0122\n",
            "Epoch   6 Batch  480/538 - Train Accuracy: 0.9901, Validation Accuracy: 0.9650, Loss: 0.0088\n",
            "Epoch   6 Batch  490/538 - Train Accuracy: 0.9808, Validation Accuracy: 0.9732, Loss: 0.0106\n",
            "Epoch   6 Batch  500/538 - Train Accuracy: 0.9936, Validation Accuracy: 0.9696, Loss: 0.0059\n",
            "Epoch   6 Batch  510/538 - Train Accuracy: 0.9918, Validation Accuracy: 0.9636, Loss: 0.0092\n",
            "Epoch   6 Batch  520/538 - Train Accuracy: 0.9846, Validation Accuracy: 0.9751, Loss: 0.0117\n",
            "Epoch   6 Batch  530/538 - Train Accuracy: 0.9844, Validation Accuracy: 0.9801, Loss: 0.0130\n",
            "Epoch   7 Batch   10/538 - Train Accuracy: 0.9891, Validation Accuracy: 0.9680, Loss: 0.0110\n",
            "Epoch   7 Batch   20/538 - Train Accuracy: 0.9851, Validation Accuracy: 0.9790, Loss: 0.0156\n",
            "Epoch   7 Batch   30/538 - Train Accuracy: 0.9717, Validation Accuracy: 0.9773, Loss: 0.0112\n",
            "Epoch   7 Batch   40/538 - Train Accuracy: 0.9876, Validation Accuracy: 0.9686, Loss: 0.0100\n",
            "Epoch   7 Batch   50/538 - Train Accuracy: 0.9852, Validation Accuracy: 0.9760, Loss: 0.0093\n",
            "Epoch   7 Batch   60/538 - Train Accuracy: 0.9912, Validation Accuracy: 0.9879, Loss: 0.0158\n",
            "Epoch   7 Batch   70/538 - Train Accuracy: 0.9864, Validation Accuracy: 0.9728, Loss: 0.0107\n",
            "Epoch   7 Batch   80/538 - Train Accuracy: 0.9924, Validation Accuracy: 0.9725, Loss: 0.0069\n",
            "Epoch   7 Batch   90/538 - Train Accuracy: 0.9846, Validation Accuracy: 0.9755, Loss: 0.0085\n",
            "Epoch   7 Batch  100/538 - Train Accuracy: 0.9904, Validation Accuracy: 0.9817, Loss: 0.0081\n",
            "Epoch   7 Batch  110/538 - Train Accuracy: 0.9887, Validation Accuracy: 0.9743, Loss: 0.0089\n",
            "Epoch   7 Batch  120/538 - Train Accuracy: 0.9926, Validation Accuracy: 0.9753, Loss: 0.0100\n",
            "Epoch   7 Batch  130/538 - Train Accuracy: 0.9870, Validation Accuracy: 0.9737, Loss: 0.0112\n",
            "Epoch   7 Batch  140/538 - Train Accuracy: 0.9865, Validation Accuracy: 0.9709, Loss: 0.0129\n",
            "Epoch   7 Batch  150/538 - Train Accuracy: 0.9861, Validation Accuracy: 0.9641, Loss: 0.0090\n",
            "Epoch   7 Batch  160/538 - Train Accuracy: 0.9903, Validation Accuracy: 0.9764, Loss: 0.0107\n",
            "Epoch   7 Batch  170/538 - Train Accuracy: 0.9885, Validation Accuracy: 0.9616, Loss: 0.0119\n",
            "Epoch   7 Batch  180/538 - Train Accuracy: 0.9857, Validation Accuracy: 0.9719, Loss: 0.0102\n",
            "Epoch   7 Batch  190/538 - Train Accuracy: 0.9894, Validation Accuracy: 0.9853, Loss: 0.0143\n",
            "Epoch   7 Batch  200/538 - Train Accuracy: 0.9912, Validation Accuracy: 0.9727, Loss: 0.0066\n",
            "Epoch   7 Batch  210/538 - Train Accuracy: 0.9952, Validation Accuracy: 0.9732, Loss: 0.0118\n",
            "Epoch   7 Batch  220/538 - Train Accuracy: 0.9818, Validation Accuracy: 0.9778, Loss: 0.0117\n",
            "Epoch   7 Batch  230/538 - Train Accuracy: 0.9832, Validation Accuracy: 0.9677, Loss: 0.0091\n",
            "Epoch   7 Batch  240/538 - Train Accuracy: 0.9838, Validation Accuracy: 0.9796, Loss: 0.0108\n",
            "Epoch   7 Batch  250/538 - Train Accuracy: 0.9938, Validation Accuracy: 0.9647, Loss: 0.0087\n",
            "Epoch   7 Batch  260/538 - Train Accuracy: 0.9825, Validation Accuracy: 0.9787, Loss: 0.0156\n",
            "Epoch   7 Batch  270/538 - Train Accuracy: 0.9971, Validation Accuracy: 0.9787, Loss: 0.0062\n",
            "Epoch   7 Batch  280/538 - Train Accuracy: 0.9890, Validation Accuracy: 0.9759, Loss: 0.0098\n",
            "Epoch   7 Batch  290/538 - Train Accuracy: 0.9908, Validation Accuracy: 0.9700, Loss: 0.0068\n",
            "Epoch   7 Batch  300/538 - Train Accuracy: 0.9952, Validation Accuracy: 0.9727, Loss: 0.0123\n",
            "Epoch   7 Batch  310/538 - Train Accuracy: 0.9844, Validation Accuracy: 0.9727, Loss: 0.0095\n",
            "Epoch   7 Batch  320/538 - Train Accuracy: 0.9859, Validation Accuracy: 0.9705, Loss: 0.0135\n",
            "Epoch   7 Batch  330/538 - Train Accuracy: 0.9907, Validation Accuracy: 0.9783, Loss: 0.0094\n",
            "Epoch   7 Batch  340/538 - Train Accuracy: 0.9945, Validation Accuracy: 0.9782, Loss: 0.0081\n",
            "Epoch   7 Batch  350/538 - Train Accuracy: 0.9918, Validation Accuracy: 0.9753, Loss: 0.0104\n",
            "Epoch   7 Batch  360/538 - Train Accuracy: 0.9949, Validation Accuracy: 0.9693, Loss: 0.0101\n",
            "Epoch   7 Batch  370/538 - Train Accuracy: 0.9887, Validation Accuracy: 0.9764, Loss: 0.0107\n",
            "Epoch   7 Batch  380/538 - Train Accuracy: 0.9930, Validation Accuracy: 0.9748, Loss: 0.0110\n",
            "Epoch   7 Batch  390/538 - Train Accuracy: 0.9881, Validation Accuracy: 0.9750, Loss: 0.0092\n",
            "Epoch   7 Batch  400/538 - Train Accuracy: 0.9892, Validation Accuracy: 0.9707, Loss: 0.0081\n",
            "Epoch   7 Batch  410/538 - Train Accuracy: 0.9951, Validation Accuracy: 0.9673, Loss: 0.0082\n",
            "Epoch   7 Batch  420/538 - Train Accuracy: 0.9867, Validation Accuracy: 0.9714, Loss: 0.0114\n",
            "Epoch   7 Batch  430/538 - Train Accuracy: 0.9854, Validation Accuracy: 0.9693, Loss: 0.0115\n",
            "Epoch   7 Batch  440/538 - Train Accuracy: 0.9941, Validation Accuracy: 0.9618, Loss: 0.0151\n",
            "Epoch   7 Batch  450/538 - Train Accuracy: 0.9771, Validation Accuracy: 0.9675, Loss: 0.0165\n",
            "Epoch   7 Batch  460/538 - Train Accuracy: 0.9851, Validation Accuracy: 0.9711, Loss: 0.0108\n",
            "Epoch   7 Batch  470/538 - Train Accuracy: 0.9877, Validation Accuracy: 0.9803, Loss: 0.0097\n",
            "Epoch   7 Batch  480/538 - Train Accuracy: 0.9879, Validation Accuracy: 0.9721, Loss: 0.0110\n",
            "Epoch   7 Batch  490/538 - Train Accuracy: 0.9879, Validation Accuracy: 0.9775, Loss: 0.0102\n",
            "Epoch   7 Batch  500/538 - Train Accuracy: 0.9963, Validation Accuracy: 0.9648, Loss: 0.0074\n",
            "Epoch   7 Batch  510/538 - Train Accuracy: 0.9933, Validation Accuracy: 0.9719, Loss: 0.0091\n",
            "Epoch   7 Batch  520/538 - Train Accuracy: 0.9877, Validation Accuracy: 0.9759, Loss: 0.0133\n",
            "Epoch   7 Batch  530/538 - Train Accuracy: 0.9875, Validation Accuracy: 0.9689, Loss: 0.0092\n",
            "Epoch   8 Batch   10/538 - Train Accuracy: 0.9881, Validation Accuracy: 0.9767, Loss: 0.0097\n",
            "Epoch   8 Batch   20/538 - Train Accuracy: 0.9875, Validation Accuracy: 0.9741, Loss: 0.0128\n",
            "Epoch   8 Batch   30/538 - Train Accuracy: 0.9818, Validation Accuracy: 0.9684, Loss: 0.0115\n",
            "Epoch   8 Batch   40/538 - Train Accuracy: 0.9860, Validation Accuracy: 0.9753, Loss: 0.0103\n",
            "Epoch   8 Batch   50/538 - Train Accuracy: 0.9848, Validation Accuracy: 0.9730, Loss: 0.0099\n",
            "Epoch   8 Batch   60/538 - Train Accuracy: 0.9934, Validation Accuracy: 0.9796, Loss: 0.0116\n",
            "Epoch   8 Batch   70/538 - Train Accuracy: 0.9879, Validation Accuracy: 0.9769, Loss: 0.0072\n",
            "Epoch   8 Batch   80/538 - Train Accuracy: 0.9967, Validation Accuracy: 0.9826, Loss: 0.0056\n",
            "Epoch   8 Batch   90/538 - Train Accuracy: 0.9887, Validation Accuracy: 0.9760, Loss: 0.0113\n",
            "Epoch   8 Batch  100/538 - Train Accuracy: 0.9953, Validation Accuracy: 0.9796, Loss: 0.0074\n",
            "Epoch   8 Batch  110/538 - Train Accuracy: 0.9930, Validation Accuracy: 0.9801, Loss: 0.0063\n",
            "Epoch   8 Batch  120/538 - Train Accuracy: 0.9939, Validation Accuracy: 0.9799, Loss: 0.0087\n",
            "Epoch   8 Batch  130/538 - Train Accuracy: 0.9885, Validation Accuracy: 0.9771, Loss: 0.0115\n",
            "Epoch   8 Batch  140/538 - Train Accuracy: 0.9902, Validation Accuracy: 0.9732, Loss: 0.0125\n",
            "Epoch   8 Batch  150/538 - Train Accuracy: 0.9953, Validation Accuracy: 0.9810, Loss: 0.0089\n",
            "Epoch   8 Batch  160/538 - Train Accuracy: 0.9862, Validation Accuracy: 0.9657, Loss: 0.0138\n",
            "Epoch   8 Batch  170/538 - Train Accuracy: 0.9888, Validation Accuracy: 0.9778, Loss: 0.0111\n",
            "Epoch   8 Batch  180/538 - Train Accuracy: 0.9836, Validation Accuracy: 0.9632, Loss: 0.0111\n",
            "Epoch   8 Batch  190/538 - Train Accuracy: 0.9864, Validation Accuracy: 0.9661, Loss: 0.0128\n",
            "Epoch   8 Batch  200/538 - Train Accuracy: 0.9916, Validation Accuracy: 0.9751, Loss: 0.0077\n",
            "Epoch   8 Batch  210/538 - Train Accuracy: 0.9926, Validation Accuracy: 0.9698, Loss: 0.0116\n",
            "Epoch   8 Batch  220/538 - Train Accuracy: 0.9766, Validation Accuracy: 0.9721, Loss: 0.0107\n",
            "Epoch   8 Batch  230/538 - Train Accuracy: 0.9861, Validation Accuracy: 0.9748, Loss: 0.0112\n",
            "Epoch   8 Batch  240/538 - Train Accuracy: 0.9877, Validation Accuracy: 0.9737, Loss: 0.0089\n",
            "Epoch   8 Batch  250/538 - Train Accuracy: 0.9953, Validation Accuracy: 0.9806, Loss: 0.0104\n",
            "Epoch   8 Batch  260/538 - Train Accuracy: 0.9834, Validation Accuracy: 0.9691, Loss: 0.0109\n",
            "Epoch   8 Batch  270/538 - Train Accuracy: 0.9855, Validation Accuracy: 0.9751, Loss: 0.0084\n",
            "Epoch   8 Batch  280/538 - Train Accuracy: 0.9909, Validation Accuracy: 0.9602, Loss: 0.0084\n",
            "Epoch   8 Batch  290/538 - Train Accuracy: 0.9922, Validation Accuracy: 0.9744, Loss: 0.0089\n",
            "Epoch   8 Batch  300/538 - Train Accuracy: 0.9927, Validation Accuracy: 0.9817, Loss: 0.0089\n",
            "Epoch   8 Batch  310/538 - Train Accuracy: 0.9920, Validation Accuracy: 0.9709, Loss: 0.0112\n",
            "Epoch   8 Batch  320/538 - Train Accuracy: 0.9957, Validation Accuracy: 0.9748, Loss: 0.0076\n",
            "Epoch   8 Batch  330/538 - Train Accuracy: 0.9859, Validation Accuracy: 0.9712, Loss: 0.0089\n",
            "Epoch   8 Batch  340/538 - Train Accuracy: 0.9977, Validation Accuracy: 0.9727, Loss: 0.0080\n",
            "Epoch   8 Batch  350/538 - Train Accuracy: 0.9862, Validation Accuracy: 0.9735, Loss: 0.0080\n",
            "Epoch   8 Batch  360/538 - Train Accuracy: 0.9912, Validation Accuracy: 0.9824, Loss: 0.0083\n",
            "Epoch   8 Batch  370/538 - Train Accuracy: 0.9924, Validation Accuracy: 0.9806, Loss: 0.0089\n",
            "Epoch   8 Batch  380/538 - Train Accuracy: 0.9916, Validation Accuracy: 0.9693, Loss: 0.0085\n",
            "Epoch   8 Batch  390/538 - Train Accuracy: 0.9911, Validation Accuracy: 0.9727, Loss: 0.0077\n",
            "Epoch   8 Batch  400/538 - Train Accuracy: 0.9950, Validation Accuracy: 0.9728, Loss: 0.0099\n",
            "Epoch   8 Batch  410/538 - Train Accuracy: 0.9959, Validation Accuracy: 0.9766, Loss: 0.0073\n",
            "Epoch   8 Batch  420/538 - Train Accuracy: 0.9949, Validation Accuracy: 0.9700, Loss: 0.0095\n",
            "Epoch   8 Batch  430/538 - Train Accuracy: 0.9924, Validation Accuracy: 0.9732, Loss: 0.0094\n",
            "Epoch   8 Batch  440/538 - Train Accuracy: 0.9906, Validation Accuracy: 0.9643, Loss: 0.0071\n",
            "Epoch   8 Batch  450/538 - Train Accuracy: 0.9892, Validation Accuracy: 0.9759, Loss: 0.0116\n",
            "Epoch   8 Batch  460/538 - Train Accuracy: 0.9900, Validation Accuracy: 0.9707, Loss: 0.0103\n",
            "Epoch   8 Batch  470/538 - Train Accuracy: 0.9890, Validation Accuracy: 0.9675, Loss: 0.0079\n",
            "Epoch   8 Batch  480/538 - Train Accuracy: 0.9916, Validation Accuracy: 0.9741, Loss: 0.0075\n",
            "Epoch   8 Batch  490/538 - Train Accuracy: 0.9909, Validation Accuracy: 0.9675, Loss: 0.0086\n",
            "Epoch   8 Batch  500/538 - Train Accuracy: 0.9938, Validation Accuracy: 0.9759, Loss: 0.0088\n",
            "Epoch   8 Batch  510/538 - Train Accuracy: 0.9911, Validation Accuracy: 0.9759, Loss: 0.0093\n",
            "Epoch   8 Batch  520/538 - Train Accuracy: 0.9850, Validation Accuracy: 0.9703, Loss: 0.0101\n",
            "Epoch   8 Batch  530/538 - Train Accuracy: 0.9951, Validation Accuracy: 0.9732, Loss: 0.0088\n",
            "Epoch   9 Batch   10/538 - Train Accuracy: 0.9988, Validation Accuracy: 0.9650, Loss: 0.0066\n",
            "Epoch   9 Batch   20/538 - Train Accuracy: 0.9881, Validation Accuracy: 0.9727, Loss: 0.0108\n",
            "Epoch   9 Batch   30/538 - Train Accuracy: 0.9812, Validation Accuracy: 0.9759, Loss: 0.0115\n",
            "Epoch   9 Batch   40/538 - Train Accuracy: 0.9817, Validation Accuracy: 0.9668, Loss: 0.0105\n",
            "Epoch   9 Batch   50/538 - Train Accuracy: 0.9828, Validation Accuracy: 0.9764, Loss: 0.0105\n",
            "Epoch   9 Batch   60/538 - Train Accuracy: 0.9945, Validation Accuracy: 0.9787, Loss: 0.0103\n",
            "Epoch   9 Batch   70/538 - Train Accuracy: 0.9907, Validation Accuracy: 0.9727, Loss: 0.0093\n",
            "Epoch   9 Batch   80/538 - Train Accuracy: 0.9939, Validation Accuracy: 0.9702, Loss: 0.0080\n",
            "Epoch   9 Batch   90/538 - Train Accuracy: 0.9799, Validation Accuracy: 0.9673, Loss: 0.0105\n",
            "Epoch   9 Batch  100/538 - Train Accuracy: 0.9934, Validation Accuracy: 0.9808, Loss: 0.0073\n",
            "Epoch   9 Batch  110/538 - Train Accuracy: 0.9926, Validation Accuracy: 0.9833, Loss: 0.0094\n",
            "Epoch   9 Batch  120/538 - Train Accuracy: 0.9945, Validation Accuracy: 0.9824, Loss: 0.0066\n",
            "Epoch   9 Batch  130/538 - Train Accuracy: 0.9918, Validation Accuracy: 0.9806, Loss: 0.0098\n",
            "Epoch   9 Batch  140/538 - Train Accuracy: 0.9879, Validation Accuracy: 0.9680, Loss: 0.0125\n",
            "Epoch   9 Batch  150/538 - Train Accuracy: 0.9955, Validation Accuracy: 0.9707, Loss: 0.0072\n",
            "Epoch   9 Batch  160/538 - Train Accuracy: 0.9870, Validation Accuracy: 0.9688, Loss: 0.0073\n",
            "Epoch   9 Batch  170/538 - Train Accuracy: 0.9816, Validation Accuracy: 0.9737, Loss: 0.0114\n",
            "Epoch   9 Batch  180/538 - Train Accuracy: 0.9942, Validation Accuracy: 0.9684, Loss: 0.0071\n",
            "Epoch   9 Batch  190/538 - Train Accuracy: 0.9935, Validation Accuracy: 0.9750, Loss: 0.0108\n",
            "Epoch   9 Batch  200/538 - Train Accuracy: 0.9939, Validation Accuracy: 0.9759, Loss: 0.0063\n",
            "Epoch   9 Batch  210/538 - Train Accuracy: 0.9914, Validation Accuracy: 0.9737, Loss: 0.0082\n",
            "Epoch   9 Batch  220/538 - Train Accuracy: 0.9900, Validation Accuracy: 0.9721, Loss: 0.0094\n",
            "Epoch   9 Batch  230/538 - Train Accuracy: 0.9984, Validation Accuracy: 0.9629, Loss: 0.0042\n",
            "Epoch   9 Batch  240/538 - Train Accuracy: 0.9889, Validation Accuracy: 0.9696, Loss: 0.0093\n",
            "Epoch   9 Batch  250/538 - Train Accuracy: 0.9939, Validation Accuracy: 0.9831, Loss: 0.0066\n",
            "Epoch   9 Batch  260/538 - Train Accuracy: 0.9820, Validation Accuracy: 0.9746, Loss: 0.0120\n",
            "Epoch   9 Batch  270/538 - Train Accuracy: 0.9938, Validation Accuracy: 0.9654, Loss: 0.0072\n",
            "Epoch   9 Batch  280/538 - Train Accuracy: 0.9978, Validation Accuracy: 0.9664, Loss: 0.0087\n",
            "Epoch   9 Batch  290/538 - Train Accuracy: 0.9896, Validation Accuracy: 0.9730, Loss: 0.0091\n",
            "Epoch   9 Batch  300/538 - Train Accuracy: 0.9875, Validation Accuracy: 0.9746, Loss: 0.0117\n",
            "Epoch   9 Batch  310/538 - Train Accuracy: 0.9912, Validation Accuracy: 0.9668, Loss: 0.0101\n",
            "Epoch   9 Batch  320/538 - Train Accuracy: 0.9929, Validation Accuracy: 0.9757, Loss: 0.0077\n",
            "Epoch   9 Batch  330/538 - Train Accuracy: 0.9980, Validation Accuracy: 0.9737, Loss: 0.0071\n",
            "Epoch   9 Batch  340/538 - Train Accuracy: 0.9965, Validation Accuracy: 0.9691, Loss: 0.0077\n",
            "Epoch   9 Batch  350/538 - Train Accuracy: 0.9937, Validation Accuracy: 0.9725, Loss: 0.0097\n",
            "Epoch   9 Batch  360/538 - Train Accuracy: 0.9910, Validation Accuracy: 0.9712, Loss: 0.0053\n",
            "Epoch   9 Batch  370/538 - Train Accuracy: 0.9887, Validation Accuracy: 0.9711, Loss: 0.0074\n",
            "Epoch   9 Batch  380/538 - Train Accuracy: 0.9986, Validation Accuracy: 0.9693, Loss: 0.0057\n",
            "Epoch   9 Batch  390/538 - Train Accuracy: 0.9931, Validation Accuracy: 0.9679, Loss: 0.0050\n",
            "Epoch   9 Batch  400/538 - Train Accuracy: 0.9972, Validation Accuracy: 0.9780, Loss: 0.0063\n",
            "Epoch   9 Batch  410/538 - Train Accuracy: 0.9898, Validation Accuracy: 0.9792, Loss: 0.0076\n",
            "Epoch   9 Batch  420/538 - Train Accuracy: 0.9842, Validation Accuracy: 0.9686, Loss: 0.0084\n",
            "Epoch   9 Batch  430/538 - Train Accuracy: 0.9875, Validation Accuracy: 0.9721, Loss: 0.0076\n",
            "Epoch   9 Batch  440/538 - Train Accuracy: 0.9938, Validation Accuracy: 0.9679, Loss: 0.0090\n",
            "Epoch   9 Batch  450/538 - Train Accuracy: 0.9872, Validation Accuracy: 0.9711, Loss: 0.0111\n",
            "Epoch   9 Batch  460/538 - Train Accuracy: 0.9872, Validation Accuracy: 0.9666, Loss: 0.0103\n",
            "Epoch   9 Batch  470/538 - Train Accuracy: 0.9939, Validation Accuracy: 0.9748, Loss: 0.0076\n",
            "Epoch   9 Batch  480/538 - Train Accuracy: 0.9944, Validation Accuracy: 0.9798, Loss: 0.0087\n",
            "Epoch   9 Batch  490/538 - Train Accuracy: 0.9872, Validation Accuracy: 0.9666, Loss: 0.0103\n",
            "Epoch   9 Batch  500/538 - Train Accuracy: 0.9922, Validation Accuracy: 0.9712, Loss: 0.0058\n",
            "Epoch   9 Batch  510/538 - Train Accuracy: 1.0000, Validation Accuracy: 0.9705, Loss: 0.0062\n",
            "Epoch   9 Batch  520/538 - Train Accuracy: 0.9873, Validation Accuracy: 0.9636, Loss: 0.0088\n",
            "Epoch   9 Batch  530/538 - Train Accuracy: 0.9967, Validation Accuracy: 0.9647, Loss: 0.0109\n",
            "Model Trained and Saved\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Bk3O6lPEfx3o",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Save Parameters\n",
        "<p>Save the batch_size and save_path parameters for inference.</p>"
      ]
    },
    {
      "metadata": {
        "id": "PSXl3riGf4EV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "save_params(save_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "prayQmZ_zWpf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "save_params(batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "O0djcFTIBk5m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "_, (source_vocab_to_int, target_vocab_to_int), (source_int_to_vocab, target_int_to_vocab) = load_preprocess()\n",
        "load_path = load_params()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "iSuqWsTxgC22",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Sentence to Sequence\n",
        "<p>To feed a sentence into the model for translation, I first need to preprocess it. Implementing the\n",
        "function sentence_to_seq() to preprocess new sentences.<p>\n",
        "<ul><li> Convert the sentence to lowercase</li>\n",
        "<li>Convert words into ids using vocab_to_int </li>\n",
        "  <li> Convert words not in the vocabulary, to the < UNK > word id. </li></ul>"
      ]
    },
    {
      "metadata": {
        "id": "K58AslvDBk5w",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def sentence_to_seq(sentence, vocab_to_int):\n",
        "    \"\"\"\n",
        "    Convert a sentence to a sequence of ids\n",
        "    :param sentence: String\n",
        "    :param vocab_to_int: Dictionary to go from the words to an id\n",
        "    :return: List of word ids\n",
        "    \"\"\"\n",
        "    return [vocab_to_int.get(word, vocab_to_int.get('<UNK>')) for word in sentence.lower().split()]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "akbF-OVTga9u",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### Translate\n",
        "<p>This will translate translate_sentence from English to French. </p>"
      ]
    },
    {
      "metadata": {
        "id": "mTi4T2nzBk6G",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        },
        "outputId": "eb00788c-d856-4196-f093-ef47a5b49cec"
      },
      "cell_type": "code",
      "source": [
        "translate_sentence = 'he saw a old yellow truck .'\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "DON'T MODIFY ANYTHING IN THIS CELL\n",
        "\"\"\"\n",
        "translate_sentence = sentence_to_seq(translate_sentence, source_vocab_to_int)\n",
        "\n",
        "loaded_graph = tf.Graph()\n",
        "with tf.Session(graph=loaded_graph) as sess:\n",
        "    # Load saved model\n",
        "    loader = tf.train.import_meta_graph(load_path + '.meta')\n",
        "    loader.restore(sess, load_path)\n",
        "\n",
        "    input_data = loaded_graph.get_tensor_by_name('input:0')\n",
        "    logits = loaded_graph.get_tensor_by_name('predictions:0')\n",
        "    target_sequence_length = loaded_graph.get_tensor_by_name('target_sequence_length:0')\n",
        "    source_sequence_length = loaded_graph.get_tensor_by_name('source_sequence_length:0')\n",
        "    keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')\n",
        "\n",
        "    translate_logits = sess.run(logits, {input_data: [translate_sentence]*batch_size,\n",
        "                                         target_sequence_length: [len(translate_sentence)*2]*batch_size,\n",
        "                                         source_sequence_length: [len(translate_sentence)]*batch_size,\n",
        "                                         keep_prob: 1.0})[0]\n",
        "\n",
        "print('Input')\n",
        "print('  Word Ids:      {}'.format([i for i in translate_sentence]))\n",
        "print('  English Words: {}'.format([source_int_to_vocab[i] for i in translate_sentence]))\n",
        "\n",
        "print('\\nPrediction')\n",
        "print('  Word Ids:      {}'.format([i for i in translate_logits]))\n",
        "print('  French Words: {}'.format(\" \".join([target_int_to_vocab[i] for i in translate_logits])))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from checkpoints/dev\n",
            "Input\n",
            "  Word Ids:      [219, 148, 65, 35, 184, 33, 110]\n",
            "  English Words: ['he', 'saw', 'a', 'old', 'yellow', 'truck', '.']\n",
            "\n",
            "Prediction\n",
            "  Word Ids:      [280, 35, 162, 342, 72, 67, 44, 48, 1]\n",
            "  French Words: il a vu un vieux camion jaune . <EOS>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "iRS_-_6Jpd6I",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}