Autoencoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" Deep Auto-Encoder implementation\n",
    "    \n",
    "    An auto-encoder works as follows:\n",
    "\n",
    "    Data of dimension k is reduced to a lower dimension j using a matrix multiplication:\n",
    "    softmax(W*x + b)  = x'\n",
    "    \n",
    "    where W is matrix from R^k --> R^j\n",
    "\n",
    "    A reconstruction matrix W' maps back from R^j --> R^k\n",
    "\n",
    "    so our reconstruction function is softmax'(W' * x' + b') \n",
    "\n",
    "    Now the point of the auto-encoder is to create a reduction matrix (values for W, b) \n",
    "    that is \"good\" at reconstructing  the original data. \n",
    "\n",
    "    Thus we want to minimize  ||softmax'(W' * (softmax(W *x+ b)) + b')  - x||\n",
    "x\n",
    "    A deep auto-encoder is nothing more than stacking successive layers of these reductions.\n",
    "\"\"\"\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import random\n",
    "import os, sys\n",
    "import re\n",
    "from PIL import Image\n",
    "from scipy.misc import imresize\n",
    "from IPython.display import display\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import scipy.spatial.distance as dist\n",
    "import scipy.stats as ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rgb2gray (rgb):\n",
    "    return np.dot (rgb[...,:3], [0.299, 0.587, 0.144])\n",
    "\n",
    "def imshow_gray (im):\n",
    "    plt.imshow (im, interpolation='nearest', cmap=plt.get_cmap ('gray'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 8987 Images.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "min_rows = 32\n",
    "min_cols = 32\n",
    "d = min_rows * min_cols\n",
    "X = []\n",
    "C = []\n",
    "\n",
    "for base, dirs, files in os.walk ('Images'):\n",
    "    for filename in files:\n",
    "        name_jpeg = re.match (r'^(.*)\\.JPEG$', filename)\n",
    "        if name_jpeg:\n",
    "            filepath = os.path.join (base, filename)\n",
    "            im0 = imresize(np.asarray (Image.open(filepath, 'r')), (min_rows, min_cols))\n",
    "            if len(im0.shape) == 3:\n",
    "                im = rgb2gray (im0)\n",
    "            X.extend(np.reshape (im, (1, d)))\n",
    "            C.append(os.path.split(base)[1])\n",
    "        \n",
    "print(\"Found\", len (X), \"Images.\\n\")\n",
    "X = np.asarray(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X -= np.mean(X, axis = 0)\n",
    "cov = np.dot(X.T, X) / X.shape[0]\n",
    "U,S,V = np.linalg.svd(cov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xrot = np.dot(X, U)\n",
    "Xrot_reduced = np.dot(X, U[:,:32])\n",
    "Xwhite = Xrot / np.sqrt(S + 1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def create(x, layer_sizes):\n",
    "    \n",
    "    # Build the encoding layers\n",
    "    next_layer_input = x\n",
    "\n",
    "    encoding_matrices = []\n",
    "    bias_matrices = []\n",
    "    for dim in layer_sizes:\n",
    "        input_dim = int(next_layer_input.get_shape()[1])\n",
    "        \n",
    "        # Initialize W using random values in interval [-1/sqrt(n) , 1/sqrt(n)]\n",
    "        W = tf.Variable(tf.random_uniform([input_dim, dim], -1.0 / math.sqrt(input_dim), 1.0 / math.sqrt(input_dim)))\n",
    "        \n",
    "        # Initialize b to zero\n",
    "        b = tf.Variable(tf.zeros([dim]))\n",
    "        \n",
    "        # We are going to use tied-weights so store the W matrix for later reference.\n",
    "        encoding_matrices.append(W)\n",
    "        bias_matrices.append(b)\n",
    "        \n",
    "        output = tf.nn.tanh(tf.matmul(next_layer_input,W) + b)\n",
    "        \n",
    "        # the input into the next layer is the output of this layer\n",
    "        next_layer_input = output\n",
    "        \n",
    "    # The fully encoded x value is now stored in the next_layer_input\n",
    "    encoded_x = next_layer_input\n",
    "    #print(\"0:\", encoded_x.get_shape())\n",
    "    \n",
    "    # build the reconstruction layers by reversing the reductions\n",
    "    layer_sizes.reverse()\n",
    "    encoding_matrices.reverse()\n",
    "    bias_matrices.reverse()\n",
    "    \n",
    "    for i, dim in enumerate(layer_sizes[1:] + [ int(x.get_shape()[1])]) :\n",
    "        #print(\"1:\", layer_sizes[1:] + [ int(x.get_shape()[1])])\n",
    "        #print(\"2:\", i, dim)\n",
    "        # we are using tied weights, so just lookup the encoding matrix for this step and transpose it\n",
    "        W = tf.transpose(encoding_matrices[i])\n",
    "        b = tf.Variable(tf.zeros([dim]))\n",
    "        output = tf.nn.tanh(tf.matmul(next_layer_input, W) + b)\n",
    "        next_layer_input = output\n",
    "        \n",
    "    # the fully encoded and reconstructed value of x is here:\n",
    "    reconstructed_x = next_layer_input\n",
    "    \n",
    "    return {\n",
    "        'encoded': encoded_x,\n",
    "        'decoded': reconstructed_x,\n",
    "        'cost' : tf.sqrt(tf.reduce_mean(tf.square(x-reconstructed_x)))\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def deep_test(hlayers, X):\n",
    "    sess = tf.Session()\n",
    "    start_dim = X.shape[1]\n",
    "    x = tf.placeholder(\"float\", [None, start_dim])\n",
    "    autoencoder = create(x, hlayers)\n",
    "    init = tf.initialize_all_variables()\n",
    "    sess.run(init)\n",
    "    train_step = tf.train.GradientDescentOptimizer(0.1).minimize(autoencoder['cost'])\n",
    "    for jj in range(10):\n",
    "        nnSet = np.random.choice(trainix, X.shape[0])\n",
    "        for i in nnSet:\n",
    "            # send one image at a time:\n",
    "            batch = []\n",
    "            batch.append(np.random.normal(X[i], 0.05))\n",
    "            sess.run(train_step, feed_dict={x: batch})\n",
    "    jj=0 \n",
    "    for i in testix:\n",
    "        batch = []\n",
    "        batch.append(np.random.normal(X[i], 0.001))\n",
    "        X_enc[jj] = sess.run(autoencoder['encoded'], feed_dict={x: batch})\n",
    "        jj+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testix = np.random.randint(0, Xwhite.shape[0], int(0.2*Xwhite.shape[0]))\n",
    "trainix = [x for x in range(Xwhite.shape[0]) if x not in testix]\n",
    "X_enc = np.zeros((len(testix), 30))\n",
    "C_test = [C[x] for x in testix]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    deep_test([50, 30], Xwhite)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#np.save(\"X_enc\", X_enc)\n",
    "#np.save(\"X\", X)\n",
    "#np.save(\"C\", C)\n",
    "#np.save(\"corrLUnsorted\", corrL)\n",
    "#np.save(\"corrLSorted\", corrLS)\n",
    "#X = np.load('X.npy')\n",
    "#X_enc = np.load(\"X_enc.npy\")\n",
    "#C = np.load(\"C.npy\")\n",
    "#corrL = np.load(\"corrLSorted.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Pearson's correlation coefficients\n",
    "corrL = dist.squareform(dist.pdist(X_enc, lambda x, y: ss.pearsonr(x, y)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corrLS = corrL\n",
    "for i in range(corrL.shape[0]):\n",
    "    corrLS[i] = sorted(range(len(corrL[i])), key=lambda j: abs(corrL[i,j]), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Animal',\n",
       " 'Fungus',\n",
       " 'Geological Formation',\n",
       " 'Person',\n",
       " 'Plant, flora, plant life',\n",
       " 'Sport'}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "confuMat = pd.DataFrame(data=0.0, \n",
    "index = ['Animal','Geological Formation','Person','Plant, flora, plant life','Fungus', 'Sport'], \n",
    "columns = ['Animal','Geological Formation','Person','Plant, flora, plant life','Fungus','Sport', 'Accuracy'])\n",
    "\n",
    "from collections import Counter\n",
    "for i in range(corrLS.shape[0]):\n",
    "    predClass = Counter(C_test[x] for x in map(int, corrLS[i,:5])).most_common(1)[0][0]\n",
    "    confuMat[C_test[i]][predClass] +=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(confuMat.shape[0]):\n",
    "    confuMat.iloc[i]['Accuracy'] = confuMat.iloc[i][i]/np.sum(confuMat.iloc[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal</th>\n",
       "      <th>Geological Formation</th>\n",
       "      <th>Person</th>\n",
       "      <th>Plant, flora, plant life</th>\n",
       "      <th>Fungus</th>\n",
       "      <th>Sport</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Animal</th>\n",
       "      <td>51.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.225664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Geological Formation</th>\n",
       "      <td>89.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>0.262055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Person</th>\n",
       "      <td>41.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.140845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Plant, flora, plant life</th>\n",
       "      <td>42.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>0.222222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fungus</th>\n",
       "      <td>25.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.195531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sport</th>\n",
       "      <td>71.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>123.0</td>\n",
       "      <td>0.290780</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Animal  Geological Formation  Person  \\\n",
       "Animal                      51.0                  43.0    32.0   \n",
       "Geological Formation        89.0                 125.0    53.0   \n",
       "Person                      41.0                  38.0    30.0   \n",
       "Plant, flora, plant life    42.0                  48.0    48.0   \n",
       "Fungus                      25.0                  29.0    29.0   \n",
       "Sport                       71.0                  66.0    63.0   \n",
       "\n",
       "                          Plant, flora, plant life  Fungus  Sport  Accuracy  \n",
       "Animal                                        33.0    25.0   42.0  0.225664  \n",
       "Geological Formation                          68.0    46.0   96.0  0.262055  \n",
       "Person                                        31.0    25.0   48.0  0.140845  \n",
       "Plant, flora, plant life                      62.0    31.0   48.0  0.222222  \n",
       "Fungus                                        23.0    35.0   38.0  0.195531  \n",
       "Sport                                         52.0    48.0  123.0  0.290780  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confuMat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "testix = np.random.randint(0, X.shape[0], int(0.2*X.shape[0]))\n",
    "trainix = [x for x in range(X.shape[0]) if x not in testix]\n",
    "X_enc = np.zeros((len(testix), 30))\n",
    "C_test = [C[x] for x in testix]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_enc = np.zeros((X.shape[0], 30))\n",
    "if __name__ == '__main__':\n",
    "    deep_test([30], X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#X_encSm = X_enc\n",
    "#np.save(\"X_encSm\", X_encSm)\n",
    "#X_encSm = np.load('X_encSm.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import scipy.spatial.distance as dist\n",
    "import scipy.stats as ss\n",
    "# Pearson's correlation coefficients\n",
    "corrLSm = dist.squareform(dist.pdist(X_enc, lambda x, y: ss.pearsonr(x, y)[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#np.save('corrLSm', corrLSm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corrLSmS = corrLSm\n",
    "for i in range(corrLSm.shape[0]):\n",
    "    corrLSmS[i] = sorted(range(len(corrLSm[i])), key=lambda j: abs(corrLSm[i,j]), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#np.save('corrLSmS', corrLSmS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "confuMatSm = pd.DataFrame(data=0.0, \n",
    "index = ['Animal','Geological Formation','Person','Plant, flora, plant life','Fungus', 'Sport'], \n",
    "columns = ['Animal','Geological Formation','Person','Plant, flora, plant life','Fungus','Sport', 'Accuracy'])\n",
    "\n",
    "from collections import Counter\n",
    "for i in range(corrLSmS.shape[0]):\n",
    "    predClass = Counter(C[x] for x in map(int, corrLSmS[i,:10])).most_common(1)[0][0]\n",
    "    confuMatSm[C[i]][predClass] +=1\n",
    "\n",
    "for i in range(confuMatSm.shape[0]):\n",
    "    confuMatSm.iloc[i]['Accuracy'] = confuMatSm.iloc[i][i]/np.sum(confuMatSm.iloc[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "confuMatSm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}