Chapter14 - Exploding Gradients Examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Activations\n",
      "[0.93940638 0.96852968]\n",
      "[0.9919462  0.99121735]\n",
      "[0.99301385 0.99302901]\n",
      "[0.9930713  0.99307098]\n",
      "[0.99307285 0.99307285]\n",
      "[0.99307291 0.99307291]\n",
      "[0.99307291 0.99307291]\n",
      "[0.99307291 0.99307291]\n",
      "[0.99307291 0.99307291]\n",
      "[0.99307291 0.99307291]\n",
      "\n",
      "Gradients\n",
      "[0.03439552 0.03439552]\n",
      "[0.00118305 0.00118305]\n",
      "[4.06916726e-05 4.06916726e-05]\n",
      "[1.39961115e-06 1.39961115e-06]\n",
      "[4.81403643e-08 4.81403637e-08]\n",
      "[1.65582672e-09 1.65582765e-09]\n",
      "[5.69682675e-11 5.69667160e-11]\n",
      "[1.97259346e-12 1.97517920e-12]\n",
      "[8.45387597e-14 8.02306381e-14]\n",
      "[1.45938177e-14 2.16938983e-14]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "sigmoid = lambda x:1/(1 + np.exp(-x))\n",
    "relu = lambda x:(x>0).astype(float)*x\n",
    "\n",
    "weights = np.array([[1,4],[4,1]])\n",
    "activation = sigmoid(np.array([1,0.01]))\n",
    "\n",
    "print(\"Activations\")\n",
    "activations = list()\n",
    "for iter in range(10):\n",
    "    activation = sigmoid(activation.dot(weights))\n",
    "    activations.append(activation)\n",
    "    print(activation)\n",
    "print(\"\\nGradients\")\n",
    "gradient = np.ones_like(activation)\n",
    "for activation in reversed(activations):\n",
    "    gradient = (activation * (1 - activation) * gradient)\n",
    "    gradient = gradient.dot(weights.transpose())\n",
    "    print(gradient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Relu Activations\n",
      "[23.71814585 23.98025559]\n",
      "[119.63916823 118.852839  ]\n",
      "[595.05052421 597.40951192]\n",
      "[2984.68857188 2977.61160877]\n",
      "[14895.13500696 14916.36589628]\n",
      "[74560.59859209 74496.90592414]\n",
      "[372548.22228863 372739.30029248]\n",
      "[1863505.42345854 1862932.18944699]\n",
      "[9315234.18124649 9316953.88328115]\n",
      "[46583049.71437107 46577890.60826711]\n",
      "\n",
      "Relu Gradients\n",
      "[5. 5.]\n",
      "[25. 25.]\n",
      "[125. 125.]\n",
      "[625. 625.]\n",
      "[3125. 3125.]\n",
      "[15625. 15625.]\n",
      "[78125. 78125.]\n",
      "[390625. 390625.]\n",
      "[1953125. 1953125.]\n",
      "[9765625. 9765625.]\n"
     ]
    }
   ],
   "source": [
    "print(\"Relu Activations\")\n",
    "activations = list()\n",
    "for iter in range(10):\n",
    "    activation = relu(activation.dot(weights))\n",
    "    activations.append(activation)\n",
    "    print(activation)\n",
    "\n",
    "print(\"\\nRelu Gradients\")\n",
    "gradient = np.ones_like(activation)\n",
    "for activation in reversed(activations):\n",
    "    gradient = ((activation > 0) * gradient).dot(weights.transpose())\n",
    "    print(gradient)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}