CNNnumpy_stepbystep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "  \"\"\"  Padding with zeros all images of the dataset X.  X -- array of shape (m, n_H, n_W, n_C) representing a batch of m images\n",
    "    pad -- integer i.e. amount of padding around each image on vertical and horizontal dimensions.  Returns:\n",
    "    X_pad -- padded image of shape (m, n_H + 2*pad, n_W + 2*pad, n_C)\n",
    "    \"\"\"\n",
    "def zero_pad(X, pad):\n",
    "    X_pad = np.pad(X, ((0, 0), (pad, pad), (pad, pad), (0, 0)), 'constant', constant_values=0)       \n",
    "    return X_pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x.shape = (4, 3, 3, 2)\n",
      "x_pad.shape = (4, 7, 7, 2)\n",
      "x[1, 1] = [[ 1.40754     0.12910158]\n",
      " [ 1.6169496   0.50274088]\n",
      " [ 1.55880554  0.1094027 ]]\n",
      "x_pad[1, 1] = [[ 0.  0.]\n",
      " [ 0.  0.]\n",
      " [ 0.  0.]\n",
      " [ 0.  0.]\n",
      " [ 0.  0.]\n",
      " [ 0.  0.]\n",
      " [ 0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Testing\"\"\"\n",
    "x = np.random.randn(4, 3, 3, 2)\n",
    "x_pad = zero_pad(x, 2)\n",
    "print (\"x.shape =\", x.shape)\n",
    "print (\"x_pad.shape =\", x_pad.shape)\n",
    "print (\"x[1, 1] =\", x[1, 1])\n",
    "print (\"x_pad[1, 1] =\", x_pad[1, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1dea306add8>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " \"\"\"  Showing the original image and the padded image using matplotlib plots    \"\"\"\n",
    "fig, axarr = plt.subplots(1, 2)\n",
    "axarr[0].set_title('x')\n",
    "axarr[0].imshow(x[0,:,:,0])\n",
    "axarr[1].set_title('x_pad')\n",
    "axarr[1].imshow(x_pad[0,:,:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A single slice of output activation of previous layer is taken i.e. a_slice_prev of shape [f,f,nCprev] and one filter defined by parameter W is applied. Here W is matrix of shape [f,f,nCprev] and bias b is [1,1,1]\"\"\"\n",
    "def convolution_singlestep(a_slice_prev, W, b) :\n",
    "        s = np.multiply(a_slice_prev, W) + b\n",
    "        Z=np.sum(s)\n",
    "        return Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Z = 54.8155337988\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Testing\"\"\"\n",
    "a_slice_prev = np.random.randn(4, 4, 3)\n",
    "W = np.random.randn(4, 4, 3)\n",
    "b = np.random.randn(1, 1, 1)\n",
    "\n",
    "Z = convolution_singlestep(a_slice_prev, W, b)\n",
    "print(\"Z =\", Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"This is the forward prop step. The input for this is A_prev which is output activation of prev layer. its shape is [m, nHprev, nWprev, nCprev] for a batch of m inputs.There are F filters/weights denoted  by W.  hparameters -- python dictionary containing \"stride\" and \"pad\".  Returns:\n",
    "    Z -- conv output of shape (m, n_H, n_W, n_C) and cache -- cache of values needed for the conv_backward() function\"\"\"\n",
    "def conv_forward(A_prev, W, b, hparameters):\n",
    "    # Retrieve dimensions from A_prev's shape \n",
    "    (m, nHprev, nWprev, nCprev) = A_prev.shape\n",
    "    \n",
    "    # Retrieve dimensions from W's shape \n",
    "    (f, f, nCprev, nC) = W.shape\n",
    "    stride = hparameters['stride']\n",
    "    pad = hparameters['pad']\n",
    "    \n",
    "  \n",
    "    nH = int((nHprev - f + 2 * pad) / stride) + 1\n",
    "    nW = int((nWprev - f + 2 * pad) / stride) + 1\n",
    "    \n",
    "    # Initialize the output volume Z with zeros.\n",
    "    Z = np.zeros((m, nH, nW, nC))   \n",
    "    A_prev_pad = zero_pad(A_prev, pad)\n",
    "    \n",
    "    for i in range(m):                                 \n",
    "        a_prev_pad = A_prev_pad[i]                     # Select ith training example's padded activation\n",
    "        for h in range(nH):                           # loop over vertical axis of the output volume\n",
    "            for w in range(nW):                       # loop over horizontal axis of the output volume\n",
    "                for c in range(nC):                   # loop over channels\n",
    "                    # Find the corners of the current \"slice\"\n",
    "                    vert_start = h * stride\n",
    "                    vert_end = vert_start + f\n",
    "                    horiz_start = w * stride\n",
    "                    horiz_end = horiz_start + f\n",
    "                    # Use the corners to define the (3D) slice of a_prev_pad\n",
    "                    a_slice_prev = a_prev_pad[vert_start:vert_end, horiz_start:horiz_end, :]\n",
    "                   \n",
    "                    Z[i, h, w, c] = convolution_singlestep(a_slice_prev, W[...,c], b[...,c])\n",
    "                                        \n",
    "     # Making sure output shape is correct\n",
    "    assert(Z.shape == (m, nH, nW, nC))\n",
    "    \n",
    "    # Save information in \"cache\" for the backprop\n",
    "    cache = (A_prev, W, b, hparameters)\n",
    "    \n",
    "    return Z, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Z's mean = 0.155859324889\n",
      "cache_conv[0][1][2][3] = [-0.20075807  0.18656139  0.41005165]\n"
     ]
    }
   ],
   "source": [
    "A_prev = np.random.randn(10, 4, 4, 3)\n",
    "W = np.random.randn(2, 2, 3, 8)\n",
    "b = np.random.randn(1, 1, 1, 8)\n",
    "hparameters = {\"pad\" : 2,\"stride\": 1}\n",
    "\n",
    "Z, cache_conv = conv_forward(A_prev, W, b, hparameters)\n",
    "print(\"Z's mean =\", np.mean(Z))\n",
    "print(\"cache_conv[0][1][2][3] =\", cache_conv[0][1][2][3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Now implementing max pool and avg pool in same function with help of if-else. There are no parameters but there are hyperparameters such as window size f.\"\"\"\n",
    "def pool_forward(A_prev, hparameters, mode = \"max\"):\n",
    "     # Retrieve dimensions from the input shape\n",
    "    (m, nHprev, nWprev, nCprev) = A_prev.shape\n",
    "    \n",
    "    # Retrieve hyperparameters from \"hparameters\"\n",
    "    f = hparameters[\"f\"]\n",
    "    stride = hparameters[\"stride\"]\n",
    "    \n",
    "    nH = int(1 + (nHprev - f) / stride)\n",
    "    nW = int(1 + (nWprev - f) / stride)\n",
    "    nC = nCprev\n",
    "    \n",
    "    # Initialize output matrix A\n",
    "    A = np.zeros((m, nH, nW, nC))              \n",
    "    \n",
    "    for i in range(m):                         \n",
    "        for h in range(nH):                   \n",
    "            for w in range(nW):               \n",
    "                for c in range (nC):          \n",
    "                    \n",
    "                    vert_start = h * stride\n",
    "                    vert_end = vert_start + f\n",
    "                    horiz_start = w * stride\n",
    "                    horiz_end = horiz_start + f\n",
    "                    \n",
    "                    a_prev_slice = A_prev[i, vert_start:vert_end, horiz_start:horiz_end, c]\n",
    "                    \n",
    "                    if mode == \"max\":\n",
    "                        A[i, h, w, c] = np.max(a_prev_slice)\n",
    "                    elif mode == \"average\":\n",
    "                        A[i, h, w, c] = np.mean(a_prev_slice)\n",
    "    \n",
    " \n",
    "    cache = (A_prev, hparameters)\n",
    "    \n",
    "    assert(A.shape == (m, nH, nW, nC))\n",
    "    \n",
    "    return A, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mode = max\n",
      "A = [[[[ 1.80358898  1.92381543  1.16128569]]]\n",
      "\n",
      "\n",
      " [[[ 1.1253235   1.63169151  1.81252782]]]]\n",
      "\n",
      "mode = average\n",
      "A = [[[[ 0.41217816  0.03753545 -0.25178582]]]\n",
      "\n",
      "\n",
      " [[[ 0.15540833 -0.07897916  0.23579956]]]]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Testing\"\"\"\n",
    "A_prev = np.random.randn(2, 4, 4, 3)\n",
    "hparameters = {\"stride\" : 1, \"f\": 4}\n",
    "\n",
    "A, cache = pool_forward(A_prev, hparameters)\n",
    "print(\"mode = max\")\n",
    "print(\"A =\", A)\n",
    "print()\n",
    "A, cache = pool_forward(A_prev, hparameters, mode = \"average\")\n",
    "print(\"mode = average\")\n",
    "print(\"A =\", A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Now the backprop is done. Input is dZ which is gradient of cost w.r.t. output of conv layer Z. Its is of shape [m,nH,nw,nC]. cache is output of conv_forward. Returns dA_prev w.r.t. input of conv layer A-prev and of shape [m, nHprev,nWprev, nCprev], dW of shape (f, f, n_C_prev, n_C) and db of shape [1,1,1,nC]\"\"\"\n",
    "def conv_backward(dZ, cache):\n",
    "    (A_prev, W, b, hparameters) = cache\n",
    "    \n",
    "    (m, nHprev, nWprev, nCprev) = A_prev.shape      \n",
    "    (f, f, nCprev, nC) = W.shape\n",
    "    \n",
    "    stride = hparameters[\"stride\"]\n",
    "    pad = hparameters[\"pad\"]    \n",
    "    \n",
    "    (m, nH, nW, nC) = dZ.shape\n",
    "    \n",
    "    # Initialize dA_prev, dW, db with the correct shapes\n",
    "    dA_prev = np.zeros((m, nHprev, nWprev, nCprev))                           \n",
    "    dW = np.zeros((f, f, nCprev, nC))\n",
    "    db = np.zeros((1, 1, 1, nC))\n",
    "\n",
    "    # Pad A_prev and dA_prev\n",
    "    A_prev_pad = zero_pad(A_prev, pad)\n",
    "    dA_prev_pad = zero_pad(dA_prev, pad)\n",
    "    \n",
    "    for i in range(m):                     \n",
    "        \n",
    "    \n",
    "        a_prev_pad = A_prev_pad[i]\n",
    "        da_prev_pad = dA_prev_pad[i]\n",
    "        \n",
    "        for h in range(nH):                \n",
    "            for w in range(nW):             \n",
    "                for c in range(nC):          \n",
    "                    \n",
    "                    vert_start = h\n",
    "                    vert_end = vert_start + f\n",
    "                    horiz_start = w\n",
    "                    horiz_end = horiz_start + f                    \n",
    "                   \n",
    "                    a_slice = a_prev_pad[vert_start:vert_end, horiz_start:horiz_end, :]\n",
    "                  \n",
    "                    da_prev_pad[vert_start:vert_end, horiz_start:horiz_end, :] += W[:,:,:,c] * dZ[i, h, w, c]\n",
    "                    dW[:,:,:,c] += a_slice * dZ[i, h, w, c]\n",
    "                    db[:,:,:,c] += dZ[i, h, w, c]\n",
    "                    \n",
    "        # Set the ith training example's dA_prev to the unpaded da_prev_pad\n",
    "        dA_prev[i, :, :, :] = da_prev_pad[pad:-pad, pad:-pad, :]\n",
    "    \n",
    "  \n",
    "    assert(dA_prev.shape == (m, nHprev, nWprev, nCprev))\n",
    "\n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dA_mean = 9.60899067587\n",
      "dW_mean = 10.5817412755\n",
      "db_mean = 76.3710691956\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Testing\"\"\"\n",
    "dA, dW, db = conv_backward(Z, cache_conv)\n",
    "print(\"dA_mean =\", np.mean(dA))\n",
    "print(\"dW_mean =\", np.mean(dW))\n",
    "print(\"db_mean =\", np.mean(db))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x =  [[ 0.3180143  -0.89027155  0.11133727]\n",
      " [-0.01952256 -0.83998891 -2.29820588]]\n",
      "mask =  [[ True False False]\n",
      " [False False False]]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Even though a pooling layer has no parameters for backprop to update, you still need to backpropagation the gradient through the pooling layer in order to compute gradients for layers that came before the pooling layer. The helper function create_mask_from_window() creates a \"mask\" matrix which keeps track of where the maximum of the matrix is. True (1) indicates the position of the maximum in X, the other entries are False (0). \"\"\"\n",
    "def create_mask_from_window(x):\n",
    "  \n",
    "    mask = x == np.max(x)   \n",
    "    return mask\n",
    "        \n",
    "\"\"\"Here mask and input x are arrays of same shape [f,f]\"\"\"               \n",
    "   \n",
    "\"\"\"Testing\"\"\" \n",
    "x = np.random.randn(2,3)\n",
    "mask = create_mask_from_window(x)\n",
    "print('x = ', x)\n",
    "print(\"mask = \", mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "distributed value = [[ 0.5  0.5]\n",
      " [ 0.5  0.5]]\n"
     ]
    }
   ],
   "source": [
    "\"\"\" In average pooling, every element of the input window has equal influence on the output. So to implement backprop, we will now implement a helper function that reflects this i.e. to equally distribute a value dz through a matrix of dimension shape.\"\"\"\n",
    "def distribute_value(dz, shape):\n",
    "    \"\"\"shape -- the shape (n_H, n_W) of the output matrix for which we want to distribute the value of dz\n",
    "    a -- Array of size (n_H, n_W) for which we distributed the value of dz\n",
    "    \"\"\"   \n",
    "   \n",
    "    (nH, nW) = shape\n",
    " \n",
    "    average = dz / (nH * nW)\n",
    "    a = np.ones(shape) * average\n",
    "  \n",
    "    \n",
    "    return a\n",
    "\"\"\"Testing\"\"\"\n",
    "a = distribute_value(2, (2,2))\n",
    "print('distributed value =', a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Implementing the pool_backward function in both modes (\"max\" and \"average\").  cache -- cache output from the forward pass of the pooling layer, contains the layer's input and hparameters. dA -- gradient of cost with respect to the output of the pooling layer, same shape as A. Returns:\n",
    "    dA_prev -- gradient of cost with respect to the input of the pooling layer, same shape as A_prev\"\"\"\n",
    "def pool_backward(dA, cache, mode = \"max\"):\n",
    "     (A_prev, hparameters) = cache    \n",
    "   \n",
    "     stride = hparameters[\"stride\"]\n",
    "     f = hparameters[\"f\"]\n",
    "    \n",
    "   \n",
    "     m, nHprev, nWprev, nCprev = A_prev.shape\n",
    "     m, nH, nW, nC = dA.shape    \n",
    "   \n",
    "     dA_prev = np.zeros(A_prev.shape)\n",
    "    \n",
    "     for i in range(m):                     \n",
    "      \n",
    "        a_prev = A_prev[i]\n",
    "        for h in range(nH):              \n",
    "            for w in range(nW):              \n",
    "                for c in range(nC):       \n",
    "                   \n",
    "                    vert_start = h\n",
    "                    vert_end = vert_start + f\n",
    "                    horiz_start = w\n",
    "                    horiz_end = horiz_start + f\n",
    "                    \n",
    "              \n",
    "                    if mode == \"max\":\n",
    "                     \n",
    "                        a_prev_slice = a_prev[vert_start:vert_end, horiz_start:horiz_end, c]\n",
    "                     \n",
    "                        mask = create_mask_from_window(a_prev_slice)\n",
    "                        \n",
    "                        dA_prev[i, vert_start:vert_end, horiz_start:horiz_end, c] += np.multiply(mask, dA[i, h, w, c])\n",
    "                        \n",
    "                    elif mode == \"average\":\n",
    "                        \n",
    "                        da = dA[i, h, w, c]\n",
    "                       \n",
    "                        shape = (f, f)\n",
    "                       \n",
    "                        dA_prev[i, vert_start:vert_end, horiz_start:horiz_end, c] += distribute_value(da, shape)\n",
    "                        \n",
    "\n",
    "    \n",
    "     assert(dA_prev.shape == A_prev.shape)\n",
    "    \n",
    "     return dA_prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mode = max\n",
      "mean of dA =  0.138261821554\n",
      "dA_prev[1,1] =  [[ 0.          0.        ]\n",
      " [ 1.6389616   0.        ]\n",
      " [ 1.62439537 -2.10961157]]\n",
      "\n",
      "mode = average\n",
      "mean of dA =  0.138261821554\n",
      "dA_prev[1,1] =  [[ 0.53994345 -0.31677985]\n",
      " [ 0.94604229 -0.84418274]\n",
      " [ 0.40609884 -0.52740289]]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Testing\"\"\"\n",
    "A_prev = np.random.randn(5, 5, 3, 2)\n",
    "hparameters = {\"stride\" : 1, \"f\": 2}\n",
    "A, cache = pool_forward(A_prev, hparameters)\n",
    "dA = np.random.randn(5, 4, 2, 2)\n",
    "\n",
    "dA_prev = pool_backward(dA, cache, mode = \"max\")\n",
    "print(\"mode = max\")\n",
    "print('mean of dA = ', np.mean(dA))\n",
    "print('dA_prev[1,1] = ', dA_prev[1,1])  \n",
    "print()\n",
    "dA_prev = pool_backward(dA, cache, mode = \"average\")\n",
    "print(\"mode = average\")\n",
    "print('mean of dA = ', np.mean(dA))\n",
    "print('dA_prev[1,1] = ', dA_prev[1,1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}