TensorFlow Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Identify the Digits. Let’s import all the required modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.misc import imread\n",
    "from sklearn.metrics import accuracy_score\n",
    "import tensorflow as tf\n",
    "import imageio\n",
    "import matplotlib.pylab as pylab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set a seed value, so that we can control our models randomness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To stop potential randomness\n",
    "seed = 128\n",
    "rng = np.random.RandomState(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set directory paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root_dir = os.path.abspath('../')\n",
    "data_dir = os.path.join(root_dir, 'tensorflow/data')\n",
    "sub_dir = os.path.join(root_dir, 'tensorflow/sub')\n",
    "\n",
    "# check for existence\n",
    "os.path.exists(root_dir)\n",
    "os.path.exists(data_dir)\n",
    "os.path.exists(sub_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read the datasets. These are in .csv formats, and have a filename along with the appropriate labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.png</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.png</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.png</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.png</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.png</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filename  label\n",
       "0    0.png      4\n",
       "1    1.png      9\n",
       "2    2.png      1\n",
       "3    3.png      7\n",
       "4    4.png      3"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(os.path.join(data_dir, 'Train', 'train.csv'))\n",
    "test = pd.read_csv(os.path.join(data_dir, 'Test.csv'))\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABhJJREFUeJzt3c+LzV8cx/HvRxNSUzcp5Vc2YmejIfEHWEqx8g/ILGyksDFlxXq2LGZnZSkWFlgMNQtslIWRTCmxkGL62Ayb79xzmTufc7mvx2M57+49J83TKafPR9O27X/A+Nsw6g0AdYgdQogdQogdQogdQkzUXKxpGv/0Dx1r27ZZ7edOdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdgghdghR9VXSdKPX6/Wdbd26tfjZL1++FOfv379f0574+zjZIYTYIYTYIYTYIYTYIYTYIYTYIYR79t908ODBvrNt27Z1uvbp06eL8+PHj/ed7d+/v/jZpaWl4nx+fr44n56eLs7fvHlTnFOPkx1CiB1CiB1CiB1CiB1CiB1CiB1CNG3b1lusaeot9od27dpVnD9//rzvbHJycr2388949uxZcT41NVVpJ/zUtm2z2s+d7BBC7BBC7BBC7BBC7BBC7BBC7BDCPfuKQe9P37RpU6WdjJfr16/3nV29erXiTnK4Z4dwYocQYocQYocQYocQYocQXiW9YvPmzcV5zSvKf0nTrHrL88uRI0cq7YRBnOwQQuwQQuwQQuwQQuwQQuwQQuwQwj37ipmZmeL8ypUrna39+PHj4nzQf3v87du3vrPZ2dk17emnO3fuFOc7d+4c6vupx8kOIcQOIcQOIcQOIcQOIcQOIcQOIdyzr7h27Vpx/ujRo76zhYWFodb+/Plzcf7169ehvr9kYqL8K/D9+/fO1qYuJzuEEDuEEDuEEDuEEDuEEDuEEDuEcM++Ynl5uTi/d+9epZ3Udfbs2eJ8z549lXZC15zsEELsEELsEELsEELsEELsEELsEMI9+5jbu3dvcX7jxo06G2HknOwQQuwQQuwQQuwQQuwQQuwQwtXbmJueni7Oe71ep+vfunWr0+/n9znZIYTYIYTYIYTYIYTYIYTYIYTYIYR79jFw+fLlvrMLFy50uvbt27eL87m5uU7X5/c52SGE2CGE2CGE2CGE2CGE2CGE2CGEe/Z/wKFDh4rzkydP9p21bTvU2ouLi8X5zMzMUN9PPU52CCF2CCF2CCF2CCF2CCF2CCF2CNEMew/7R4s1Tb3FxsiTJ0+K86mpqc7WPnz4cHH+9OnTztZmbdq2bVb7uZMdQogdQogdQogdQogdQogdQogdQnievYING8p/p168eLE4H/Q8+zAePHhQnL99+7Y4n5ycLM63bNnyx3v66ejRo8X5MH8uExPlX/1z584V5zdv3izOBz3nv7y8XJx3wckOIcQOIcQOIcQOIcQOIcQOITziWsH27duL83fv3lXayfp79epVcb5v375KO/m7nD9/vjifnZ3tbG2PuEI4sUMIsUMIsUMIsUMIsUMIsUMI9+wVHDt2rDh/+PBhpZ2sv6ZZ9Ur3l5q/X3+TpaWl4nzHjh2dre2eHcKJHUKIHUKIHUKIHUKIHUKIHUJ4lXQFJ06cGPUWqOzu3buj3sL/ONkhhNghhNghhNghhNghhNghhNghhHv2Cu7fv1+cX7p0qdJO1t8on2dfXFwszj99+tR3trCwUPzs7t27i/O5ubni/MWLF8X5KDjZIYTYIYTYIYTYIYTYIYTYIYRXSVewcePG4vzUqVPF+ZkzZ4rz0n+LfODAgeJnhzXoiun169d9Z4Ourz58+FCcv3z5sjgf9DrnceVV0hBO7BBC7BBC7BBC7BBC7BBC7BDCPfsY6PV6a5qth48fPxbnpcdM6YZ7dggndgghdgghdgghdgghdgghdgjhnh3GjHt2CCd2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CCF2CNG0bTvqPQAVONkhhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghhNghxA8Q0vUSA1FrfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img_name = rng.choice(train.filename)\n",
    "filepath = os.path.join(data_dir, 'Train', 'Images', 'train', img_name)\n",
    "\n",
    "img = imageio.imread(filepath, as_gray=True)\n",
    "\n",
    "pylab.imshow(img, cmap='gray')\n",
    "pylab.axis('off')\n",
    "pylab.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Show image in numpy array format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Array([[  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "         16.,  34.,  24.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,  47.,  67.,  67.,  67., 153.,\n",
       "        214., 253., 230., 177.,  30.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,  28., 240., 253., 253., 253., 253.,\n",
       "        253., 253., 253., 253., 202.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0., 246., 253., 253., 253., 253., 253.,\n",
       "        253., 253., 253., 253., 253., 131.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0., 254., 253., 253., 253., 253., 253.,\n",
       "        253., 253., 253., 253., 254., 187.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0., 129., 253., 253., 253., 195., 165.,\n",
       "        160.,  55.,  55., 227., 254., 187.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   3.,  70.,  22.,  22.,   8.,   0.,\n",
       "          0.,   0.,   4., 223., 254., 187.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,  97., 253., 254., 187.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,  32., 237., 253., 254., 187.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0., 141., 253., 253., 254.,  91.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "        123., 254., 254., 254., 159.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  52.,\n",
       "        204., 253., 253., 224.,  34.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 199.,\n",
       "        253., 253., 253.,  52.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   2., 120., 251.,\n",
       "        253., 253., 186.,  24.,  13.,  23.,  23.,  81., 133., 133., 137.,\n",
       "        243., 128.,   3.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  23., 253., 253.,\n",
       "        253., 253., 253., 253., 216., 253., 253., 253., 253., 253., 253.,\n",
       "        253., 253., 105.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  60., 253., 253.,\n",
       "        253., 253., 253., 253., 254., 253., 253., 253., 253., 253., 253.,\n",
       "        253., 253., 230.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 133., 253., 253.,\n",
       "        253., 253., 253., 253., 254., 253., 253., 253., 253., 253., 253.,\n",
       "        253., 253., 152.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  65., 253., 253.,\n",
       "        253., 253., 253., 253., 255., 253., 253., 253., 224., 209., 161.,\n",
       "        190.,  99.,  56.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   6.,  89., 176.,\n",
       "        213., 253., 253., 253., 210., 176.,  85.,  65.,  23.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "         16.,  32.,  32.,  32.,  15.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.]], dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store all our images as numpy arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = []\n",
    "for img_name in train.filename:\n",
    "    image_path = os.path.join(data_dir, 'Train', 'Images', 'train', img_name)\n",
    "    img = imageio.imread(image_path, as_gray=True)\n",
    "    img = img.astype('float32')\n",
    "    temp.append(img)\n",
    "    \n",
    "train_x = np.stack(temp)\n",
    "\n",
    "temp = []\n",
    "for img_name in test.filename:\n",
    "    image_path = os.path.join(data_dir, 'Train', 'Images', 'test', img_name)\n",
    "    img = imageio.imread(image_path, as_gray=True)\n",
    "    img = img.astype('float32')\n",
    "    temp.append(img)\n",
    "\n",
    "test_x = np.stack(temp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split size of 70:30 for train set vs validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_size = int(train_x.shape[0]*0.7)\n",
    "\n",
    "train_x, val_x = train_x[:split_size], train_x[split_size:]\n",
    "train_y, val_y = train.label.values[:split_size], train.label.values[split_size:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define some helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dense_to_one_hot(labels_dense, num_classes=10):\n",
    "    \"\"\"Convert class labels from scalars to one-hot vectors\"\"\"\n",
    "    num_labels = labels_dense.shape[0]\n",
    "    index_offset = np.arange(num_labels) * num_classes\n",
    "    labels_one_hot = np.zeros((num_labels, num_classes))\n",
    "    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
    "    \n",
    "    return labels_one_hot\n",
    "\n",
    "def preproc(unclean_batch_x):\n",
    "    \"\"\"Convert values to range 0-1\"\"\"\n",
    "    temp_batch = unclean_batch_x / unclean_batch_x.max()\n",
    "    \n",
    "    return temp_batch\n",
    "\n",
    "def batch_creator(batch_size, dataset_length, dataset_name):\n",
    "    \"\"\"Create batch with random samples and return appropriate format\"\"\"\n",
    "    batch_mask = rng.choice(dataset_length, batch_size)\n",
    "    \n",
    "    batch_x = eval(dataset_name + '_x')[[batch_mask]].reshape(-1, input_num_units)\n",
    "    batch_x = preproc(batch_x)\n",
    "    \n",
    "    if dataset_name == 'train':\n",
    "        batch_y = eval(dataset_name).ix[batch_mask, 'label'].values\n",
    "        batch_y = dense_to_one_hot(batch_y)\n",
    "        \n",
    "    return batch_x, batch_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a neural network architecture with 3 layers;  input, hidden and output. The number of neurons in input and output are fixed, as the input is our 28 x 28 image and the output is a 10 x 1 vector representing the class. We take 500 neurons in the hidden layer. This number can vary according to your need. We also assign values to remaining variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "### set all variables\n",
    "\n",
    "# number of neurons in each layer\n",
    "input_num_units = 28*28\n",
    "hidden_num_units = 500\n",
    "output_num_units = 10\n",
    "\n",
    "# define placeholders\n",
    "x = tf.placeholder(tf.float32, [None, input_num_units])\n",
    "y = tf.placeholder(tf.float32, [None, output_num_units])\n",
    "\n",
    "# set remaining variables\n",
    "epochs = 5\n",
    "batch_size = 128\n",
    "learning_rate = 0.01\n",
    "\n",
    "### define weights and biases of the neural network (refer this article if you don't understand the terminologies)\n",
    "\n",
    "weights = {\n",
    "    'hidden': tf.Variable(tf.random_normal([input_num_units, hidden_num_units], seed=seed)),\n",
    "    'output': tf.Variable(tf.random_normal([hidden_num_units, output_num_units], seed=seed))\n",
    "}\n",
    "\n",
    "biases = {\n",
    "    'hidden': tf.Variable(tf.random_normal([hidden_num_units], seed=seed)),\n",
    "    'output': tf.Variable(tf.random_normal([output_num_units], seed=seed))\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create neural networks computational graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_layer = tf.add(tf.matmul(x, weights['hidden']), biases['hidden'])\n",
    "hidden_layer = tf.nn.relu(hidden_layer)\n",
    "\n",
    "output_layer = tf.matmul(hidden_layer, weights['output']) + biases['output']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define cost of our neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits = output_layer, labels=y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the optimizer, i.e. our backpropogation algorithm (Adam, Gradient Descent algorithm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initialize all the variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a session, and run the neural network in the session. Then validate the models accuracy on validation set that just created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ipykernel_launcher.py:20: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ipykernel_launcher.py:24: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 cost = 9.02525\n",
      "Epoch: 2 cost = 1.73982\n",
      "Epoch: 3 cost = 0.89383\n",
      "Epoch: 4 cost = 0.59209\n",
      "Epoch: 5 cost = 0.46220\n",
      "\n",
      "Training complete!\n",
      "Validation Accuracy: 0.957483\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    # create initialized variables\n",
    "    sess.run(init)\n",
    "    \n",
    "    ### for each epoch, do:\n",
    "    ###   for each batch, do:\n",
    "    ###     create pre-processed batch\n",
    "    ###     run optimizer by feeding batch\n",
    "    ###     find cost and reiterate to minimize\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        avg_cost = 0\n",
    "        total_batch = int(train.shape[0]/batch_size)\n",
    "        for i in range(total_batch):\n",
    "            batch_x, batch_y = batch_creator(batch_size, train_x.shape[0], 'train')\n",
    "            _, c = sess.run([optimizer, cost], feed_dict = {x: batch_x, y: batch_y})\n",
    "            \n",
    "            avg_cost += c / total_batch\n",
    "            \n",
    "        print(\"Epoch:\", (epoch+1), \"cost =\", \"{:.5f}\".format(avg_cost))\n",
    "    \n",
    "    print(\"\\nTraining complete!\")\n",
    "    \n",
    "    \n",
    "    # find predictions on val set\n",
    "    pred_temp = tf.equal(tf.argmax(output_layer, 1), tf.argmax(y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(pred_temp, \"float\"))\n",
    "    print( \"Validation Accuracy:\", accuracy.eval({x: val_x.reshape(-1, input_num_units), y: dense_to_one_hot(val_y)}))\n",
    "    \n",
    "    predict = tf.argmax(output_layer, 1)\n",
    "    pred = predict.eval({x: test_x.reshape(-1, input_num_units)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the model and visualize its predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction is:  9\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAABm1JREFUeJzt3b9vjf0fx/Fz9G78GNDEJEyV0MQgQRsWg9UiSojEj4SpsXQQiYiNjaHC6Nc/ICYMRGJgKQPWThKiGoSKkJzvfCfO++Tu6fe0+no8xr7y6bkGz/tK7ivXabPVajWApW/ZQl8A0BtihxBihxBihxBihxD/9PLDms2m//UP/2etVqv5p5+7s0MIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUMIsUOIfxb6AlhYfX195T46Olrup0+fLvehoaG228zMTHn2/Pnz5X7//v1y59/c2SGE2CGE2CGE2CGE2CGE2CFEs9Vq9e7Dms3efViQgYGBttv4+Hh59sCBA+W+ZcuWcv/582e5v3jxou22a9eu8uy3b9/KffPmzeU+PT1d7ktVq9Vq/unn7uwQQuwQQuwQQuwQQuwQQuwQQuwQwiuuf4Hh4eFyf/jwYdtt7dq15dnJyclyHxsbK/cbN26Ue2VkZKTcnz9/Xu5Hjhwp94mJif98TUuZOzuEEDuEEDuEEDuEEDuEEDuEEDuE8Jx9Edi2bVu5P3jwoNzXrFnTdrtz50559syZM+X+9evXcu9Gf39/V+enpqbm6UoyuLNDCLFDCLFDCLFDCLFDCLFDCLFDCM/Ze2DZsvq/qRcuXCj36nvhG41G49GjR22348ePl2cX0qFDh7o6v3Xr1nJ/9uxZ2+3z589dffbfyJ0dQogdQogdQogdQogdQogdQogdQvj77D3Q6Tn5zMxMuX/69KncBwcH225fvnwpz3bS6Z3zPXv2lHv1LP3YsWPl2eXLl5d7J7dv3267nThxoqvfvZj5++wQTuwQQuwQQuwQQuwQQuwQwiuuf4HVq1eX+9mzZ9tu09PT5dmhoaFy37lzZ7l3+hrshfTkyZOFvoRFxZ0dQogdQogdQogdQogdQogdQogdQnjFtQdWrVpV7k+fPi33HTt2zOfl/Mvv37/L/c2bN+VefY11o1H/WeXr16+XZzt5+fJluY+MjLTdfv361dVnL2ZecYVwYocQYocQYocQYocQYocQYocQ3mfvgdnZ2XLfvXt3ue/fv7/cq3fKv3//Xp69d+9euXd6zt7J9u3buzpfuXv3brkv5Wfpc+HODiHEDiHEDiHEDiHEDiHEDiHEDiG8z05XVqxYUe43b95sux0+fLg8++rVq3IfHh4u99Tn7N5nh3BihxBihxBihxBihxBihxBihxDeZ6crJ0+eLPfqWfqPHz/KsxcvXiz31Ofoc+XODiHEDiHEDiHEDiHEDiHEDiG84kpp06ZN5f727dty7+/vb7uNj4+XZ69evVru/JlXXCGc2CGE2CGE2CGE2CGE2CGE2CGEV1zDrVy5stwvX75c7tVz9Eaj0Xj9+nXb7dq1a+VZ5pc7O4QQO4QQO4QQO4QQO4QQO4QQO4TwnD3cvn37yn10dLTcZ2dny/3cuXNtN18F3Vvu7BBC7BBC7BBC7BBC7BBC7BBC7BDC98YvcevWrSv3ycnJct+4cWO5P378uNz37t1b7sw/3xsP4cQOIcQOIcQOIcQOIcQOIbziusQdPHiw3Ds9Wnv//n25Hz169D9fEwvDnR1CiB1CiB1CiB1CiB1CiB1CiB1CeM6+BGzYsKHtNjEx0dXvvnXrVrl3eg7P4uHODiHEDiHEDiHEDiHEDiHEDiHEDiE8Z18CxsbG2m59fX3l2Q8fPpT7lStX5nRNLD7u7BBC7BBC7BBC7BBC7BBC7BBC7BDCc/a/wMDAQLmfOnVqzr/70qVL5f7x48c5/24WF3d2CCF2CCF2CCF2CCF2CCF2CCF2CNFstVq9+7Bms3cftoSsX7++3N+9e9d2m5qaKs8ODg6Wey//fTA/Wq1W808/d2eHEGKHEGKHEGKHEGKHEGKHEB69wRLj0RuEEzuEEDuEEDuEEDuEEDuEEDuEEDuEEDuEEDuEEDuEEDuEEDuEEDuEEDuE6On77MDCcWeHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEGKHEP8DXDYNM0LblvcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "img_name = rng.choice(test.filename)\n",
    "filepath = os.path.join(data_dir, 'Train', 'Images', 'test', img_name)\n",
    "\n",
    "img = imageio.imread(filepath, as_gray=True)\n",
    "\n",
    "test_index = int(img_name.split('.')[0]) - 49000\n",
    "print( \"Prediction is: \", pred[test_index])\n",
    "\n",
    "pylab.imshow(img, cmap='gray')\n",
    "pylab.axis('off')\n",
    "pylab.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}