3. O'Reilly Generate from image.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Generating captions from images\n",
    "This notebook builds on the previous caption generation notebook. The key difference however is that the image feature embedding is generated from an image passed through the VGG-16 network, as opposed to just pulling the feature embedding from an already precomputed set of feature embeddings for the Flickr-30K dataset. This allows the user to generate captions for their own images in addition to generating captions for the Flickr-30K images.\n",
    "\n",
    "*Note:* The `model_path` variable should be the same as the `model_path` variable set in `O'Reilly Training.ipynb`\n",
    "\n",
    "# Make sure to run the 'O'Reilly Training.ipynb' notebook for at least one epoch before running this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import cv2\n",
    "import skimage\n",
    "\n",
    "import tensorflow.python.platform\n",
    "from keras.preprocessing import sequence\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Downloading Data\n",
    "In order to run this notebook you will need to download a pretrained TensorFlow model for [VGG-16](https://drive.google.com/file/d/0B2vTU3h54lTyaDczbFhsZFpsUGs/view?usp=sharing) generated from the original Caffe model from the VGG-16 paper. \n",
    "\n",
    "Place this download in the `./data/` folder.\n",
    "\n",
    "The graph model should now be saved at `./data/vgg16-20160129.tfmodel` ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model_path = './models/tensorflow'\n",
    "vgg_path = './data/vgg16-20160129.tfmodel'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Pick your image\n",
    "Set `image_path` to point to the image you'd like to generate a caption for."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "image_path = './image_path.jpg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "dim_embed = 256\n",
    "dim_hidden = 256\n",
    "dim_in = 4096\n",
    "batch_size = 1\n",
    "learning_rate = 0.001\n",
    "momentum = 0.9\n",
    "n_epochs = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class Caption_Generator():\n",
    "    def __init__(self, dim_in, dim_embed, dim_hidden, batch_size, n_lstm_steps, n_words, init_b=None):\n",
    "\n",
    "        self.dim_in = dim_in\n",
    "        self.dim_embed = dim_embed\n",
    "        self.dim_hidden = dim_hidden\n",
    "        self.batch_size = batch_size\n",
    "        self.n_lstm_steps = n_lstm_steps\n",
    "        self.n_words = n_words\n",
    "        \n",
    "        # declare the variables to be used for our word embeddings\n",
    "        with tf.device(\"/cpu:0\"):\n",
    "            self.word_embedding = tf.Variable(tf.random_uniform([self.n_words, self.dim_embed], -0.1, 0.1), name='word_embedding')\n",
    "\n",
    "        self.embedding_bias = tf.Variable(tf.zeros([dim_embed]), name='embedding_bias')\n",
    "        \n",
    "        # declare the LSTM itself\n",
    "        self.lstm = tf.contrib.rnn.BasicLSTMCell(dim_hidden)\n",
    "        \n",
    "        # declare the variables to be used to embed the image feature embedding to the word embedding space\n",
    "        self.img_embedding = tf.Variable(tf.random_uniform([dim_in, dim_hidden], -0.1, 0.1), name='img_embedding')\n",
    "        self.img_embedding_bias = tf.Variable(tf.zeros([dim_hidden]), name='img_embedding_bias')\n",
    "\n",
    "        # declare the variables to go from an LSTM output to a word encoding output\n",
    "        self.word_encoding = tf.Variable(tf.random_uniform([dim_hidden, n_words], -0.1, 0.1), name='word_encoding')\n",
    "        \n",
    "        # optional initialization setter for encoding bias variable \n",
    "        if init_b is not None:\n",
    "            self.word_encoding_bias = tf.Variable(init_b, name='word_encoding_bias')\n",
    "        else:\n",
    "            self.word_encoding_bias = tf.Variable(tf.zeros([n_words]), name='word_encoding_bias')\n",
    "\n",
    "    def build_model(self):\n",
    "        # declaring the placeholders for our extracted image feature vectors, our caption, and our mask\n",
    "        # (describes how long our caption is with an array of 0/1 values of length `maxlen`  \n",
    "        img = tf.placeholder(tf.float32, [self.batch_size, self.dim_in])\n",
    "        caption_placeholder = tf.placeholder(tf.int32, [self.batch_size, self.n_lstm_steps])\n",
    "        mask = tf.placeholder(tf.float32, [self.batch_size, self.n_lstm_steps])\n",
    "        \n",
    "        # getting an initial LSTM embedding from our image_imbedding\n",
    "        image_embedding = tf.matmul(img, self.img_embedding) + self.img_embedding_bias\n",
    "        \n",
    "        # setting initial state of our LSTM\n",
    "        state = self.lstm.zero_state(self.batch_size, dtype=tf.float32)\n",
    "\n",
    "        total_loss = 0.0\n",
    "        with tf.variable_scope(\"RNN\"):\n",
    "            for i in range(self.n_lstm_steps): \n",
    "                if i > 0:\n",
    "                   #if this isn’t the first iteration of our LSTM we need to get the word_embedding corresponding\n",
    "                   # to the (i-1)th word in our caption \n",
    "                    with tf.device(\"/cpu:0\"):\n",
    "                        current_embedding = tf.nn.embedding_lookup(self.word_embedding, caption_placeholder[:,i-1]) + self.embedding_bias\n",
    "                else:\n",
    "                     #if this is the first iteration of our LSTM we utilize the embedded image as our input \n",
    "                    current_embedding = image_embedding\n",
    "                if i > 0: \n",
    "                    # allows us to reuse the LSTM tensor variable on each iteration\n",
    "                    tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                out, state = self.lstm(current_embedding, state)\n",
    "\n",
    "                \n",
    "                if i > 0:\n",
    "                    #get the one-hot representation of the next word in our caption \n",
    "                    labels = tf.expand_dims(caption_placeholder[:, i], 1)\n",
    "                    ix_range=tf.range(0, self.batch_size, 1)\n",
    "                    ixs = tf.expand_dims(ix_range, 1)\n",
    "                    concat = tf.concat([ixs, labels],1)\n",
    "                    onehot = tf.sparse_to_dense(\n",
    "                            concat, tf.stack([self.batch_size, self.n_words]), 1.0, 0.0)\n",
    "\n",
    "\n",
    "                    #perform a softmax classification to generate the next word in the caption\n",
    "                    logit = tf.matmul(out, self.word_encoding) + self.word_encoding_bias\n",
    "                    xentropy = tf.nn.softmax_cross_entropy_with_logits(logits=logit, labels=onehot)\n",
    "                    xentropy = xentropy * mask[:,i]\n",
    "\n",
    "                    loss = tf.reduce_sum(xentropy)\n",
    "                    total_loss += loss\n",
    "\n",
    "            total_loss = total_loss / tf.reduce_sum(mask[:,1:])\n",
    "            return total_loss, img,  caption_placeholder, mask\n",
    "\n",
    "\n",
    "    def build_generator(self, maxlen, batchsize=1):\n",
    "        #same setup as `build_model` function \n",
    "        img = tf.placeholder(tf.float32, [self.batch_size, self.dim_in])\n",
    "        image_embedding = tf.matmul(img, self.img_embedding) + self.img_embedding_bias\n",
    "        state = self.lstm.zero_state(batchsize,dtype=tf.float32)\n",
    "\n",
    "        #declare list to hold the words of our generated captions\n",
    "        all_words = []\n",
    "        with tf.variable_scope(\"RNN\"):\n",
    "            # in the first iteration we have no previous word, so we directly pass in the image embedding\n",
    "            # and set the `previous_word` to the embedding of the start token ([0]) for the future iterations\n",
    "            output, state = self.lstm(image_embedding, state)\n",
    "            previous_word = tf.nn.embedding_lookup(self.word_embedding, [0]) + self.embedding_bias\n",
    "\n",
    "            for i in range(maxlen):\n",
    "                tf.get_variable_scope().reuse_variables()\n",
    "\n",
    "                out, state = self.lstm(previous_word, state)\n",
    "\n",
    "\n",
    "                # get a one-hot word encoding from the output of the LSTM\n",
    "                logit = tf.matmul(out, self.word_encoding) + self.word_encoding_bias\n",
    "                best_word = tf.argmax(logit, 1)\n",
    "\n",
    "                with tf.device(\"/cpu:0\"):\n",
    "                    # get the embedding of the best_word to use as input to the next iteration of our LSTM \n",
    "                    previous_word = tf.nn.embedding_lookup(self.word_embedding, best_word)\n",
    "\n",
    "                previous_word += self.embedding_bias\n",
    "\n",
    "                all_words.append(best_word)\n",
    "\n",
    "        return img, all_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "if not os.path.exists('data/ixtoword.npy'):\n",
    "    print ('You must run 1. O\\'reilly Training.ipynb first.')\n",
    "else:\n",
    "    tf.reset_default_graph()\n",
    "    with open(vgg_path,'rb') as f:\n",
    "        fileContent = f.read()\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(fileContent)\n",
    "\n",
    "    images = tf.placeholder(\"float32\", [1, 224, 224, 3])\n",
    "    tf.import_graph_def(graph_def, input_map={\"images\":images})\n",
    "\n",
    "    ixtoword = np.load('data/ixtoword.npy').tolist()\n",
    "    n_words = len(ixtoword)\n",
    "    maxlen=15\n",
    "    graph = tf.get_default_graph()\n",
    "    sess = tf.InteractiveSession(graph=graph)\n",
    "    caption_generator = Caption_Generator(dim_in, dim_hidden, dim_embed, batch_size, maxlen+2, n_words)\n",
    "    graph = tf.get_default_graph()\n",
    "\n",
    "    image, generated_words = caption_generator.build_generator(maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def crop_image(x, target_height=227, target_width=227, as_float=True):\n",
    "    image = cv2.imread(x)\n",
    "    if as_float:\n",
    "        image = image.astype(np.float32)\n",
    "\n",
    "    if len(image.shape) == 2:\n",
    "        image = np.tile(image[:,:,None], 3)\n",
    "    elif len(image.shape) == 4:\n",
    "        image = image[:,:,:,0]\n",
    "\n",
    "    height, width, rgb = image.shape\n",
    "    if width == height:\n",
    "        resized_image = cv2.resize(image, (target_height,target_width))\n",
    "\n",
    "    elif height < width:\n",
    "        resized_image = cv2.resize(image, (int(width * float(target_height)/height), target_width))\n",
    "        cropping_length = int((resized_image.shape[1] - target_height) / 2)\n",
    "        resized_image = resized_image[:,cropping_length:resized_image.shape[1] - cropping_length]\n",
    "\n",
    "    else:\n",
    "        resized_image = cv2.resize(image, (target_height, int(height * float(target_width) / width)))\n",
    "        cropping_length = int((resized_image.shape[0] - target_width) / 2)\n",
    "        resized_image = resized_image[cropping_length:resized_image.shape[0] - cropping_length,:]\n",
    "\n",
    "    return cv2.resize(resized_image, (target_height, target_width))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def read_image(path):\n",
    "\n",
    "     img = crop_image(path, target_height=224, target_width=224)\n",
    "     if img.shape[2] == 4:\n",
    "         img = img[:,:,:3]\n",
    "\n",
    "     img = img[None, ...]\n",
    "     return img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Note:\n",
    "In order for the call to `saver.restore(sess, tf.train.latest_checkpoint(model_path))` to work you must have run `O' Reilly Training.ipynb` for 1 full epoch. This is because the call in `O' Reilly Training.ipynb` to save the graph to `model_path` only occurs after successfully completing one full epoch of training. If you would like to run this notebook to sanity check the code, uncomment `sanity_check=True`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def test(sess,image,generated_words,ixtoword,test_image_path=0): # Naive greedy search\n",
    "\n",
    "    \n",
    "\n",
    "    feat = read_image(test_image_path)\n",
    "    fc7 = sess.run(graph.get_tensor_by_name(\"import/Relu_1:0\"), feed_dict={images:feat})\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    sanity_check=False\n",
    "    # sanity_check=True\n",
    "    if not sanity_check:\n",
    "        saved_path=tf.train.latest_checkpoint(model_path)\n",
    "        saver.restore(sess, saved_path)\n",
    "    else:\n",
    "        tf.global_variables_initializer().run()\n",
    "\n",
    "    generated_word_index= sess.run(generated_words, feed_dict={image:fc7})\n",
    "    generated_word_index = np.hstack(generated_word_index)\n",
    "    generated_words = [ixtoword[x] for x in generated_word_index]\n",
    "    punctuation = np.argmax(np.array(generated_words) == '.')+1\n",
    "\n",
    "    generated_words = generated_words[:punctuation]\n",
    "    generated_sentence = ' '.join(generated_words)\n",
    "    print(generated_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "test(sess,image,generated_words,ixtoword, image_path)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}