Chapter10 - Intro to Convolutional Neural Networks - Learning Edges and Corners.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upgrading our MNIST Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "I:0 Test-Acc:0.0288 Train-Acc:0.055\n",
      "I:1 Test-Acc:0.0273 Train-Acc:0.037\n",
      "I:2 Test-Acc:0.028 Train-Acc:0.037\n",
      "I:3 Test-Acc:0.0292 Train-Acc:0.04\n",
      "I:4 Test-Acc:0.0339 Train-Acc:0.046\n",
      "I:5 Test-Acc:0.0478 Train-Acc:0.068\n",
      "I:6 Test-Acc:0.076 Train-Acc:0.083\n",
      "I:7 Test-Acc:0.1316 Train-Acc:0.096\n",
      "I:8 Test-Acc:0.2137 Train-Acc:0.127\n",
      "I:9 Test-Acc:0.2941 Train-Acc:0.148\n",
      "I:10 Test-Acc:0.3563 Train-Acc:0.181\n",
      "I:11 Test-Acc:0.4023 Train-Acc:0.209\n",
      "I:12 Test-Acc:0.4358 Train-Acc:0.238\n",
      "I:13 Test-Acc:0.4473 Train-Acc:0.286\n",
      "I:14 Test-Acc:0.4389 Train-Acc:0.274\n",
      "I:15 Test-Acc:0.3951 Train-Acc:0.257\n",
      "I:16 Test-Acc:0.2222 Train-Acc:0.243\n",
      "I:17 Test-Acc:0.0613 Train-Acc:0.112\n",
      "I:18 Test-Acc:0.0266 Train-Acc:0.035\n",
      "I:19 Test-Acc:0.0127 Train-Acc:0.026\n",
      "I:20 Test-Acc:0.0133 Train-Acc:0.022\n",
      "I:21 Test-Acc:0.0185 Train-Acc:0.038\n",
      "I:22 Test-Acc:0.0363 Train-Acc:0.038\n",
      "I:23 Test-Acc:0.0928 Train-Acc:0.067\n",
      "I:24 Test-Acc:0.1994 Train-Acc:0.081\n",
      "I:25 Test-Acc:0.3086 Train-Acc:0.154\n",
      "I:26 Test-Acc:0.4276 Train-Acc:0.204\n",
      "I:27 Test-Acc:0.5323 Train-Acc:0.256\n",
      "I:28 Test-Acc:0.5919 Train-Acc:0.305\n",
      "I:29 Test-Acc:0.6324 Train-Acc:0.341\n",
      "I:30 Test-Acc:0.6608 Train-Acc:0.426\n",
      "I:31 Test-Acc:0.6815 Train-Acc:0.439\n",
      "I:32 Test-Acc:0.7048 Train-Acc:0.462\n",
      "I:33 Test-Acc:0.7171 Train-Acc:0.484\n",
      "I:34 Test-Acc:0.7313 Train-Acc:0.505\n",
      "I:35 Test-Acc:0.7355 Train-Acc:0.53\n",
      "I:36 Test-Acc:0.7417 Train-Acc:0.548\n",
      "I:37 Test-Acc:0.747 Train-Acc:0.534\n",
      "I:38 Test-Acc:0.7491 Train-Acc:0.55\n",
      "I:39 Test-Acc:0.7459 Train-Acc:0.562\n",
      "I:40 Test-Acc:0.7352 Train-Acc:0.54\n",
      "I:41 Test-Acc:0.7082 Train-Acc:0.496\n",
      "I:42 Test-Acc:0.6487 Train-Acc:0.456\n",
      "I:43 Test-Acc:0.5209 Train-Acc:0.353\n",
      "I:44 Test-Acc:0.3305 Train-Acc:0.234\n",
      "I:45 Test-Acc:0.2052 Train-Acc:0.174\n",
      "I:46 Test-Acc:0.2149 Train-Acc:0.136\n",
      "I:47 Test-Acc:0.2679 Train-Acc:0.171\n",
      "I:48 Test-Acc:0.3237 Train-Acc:0.172\n",
      "I:49 Test-Acc:0.3581 Train-Acc:0.186\n",
      "I:50 Test-Acc:0.4202 Train-Acc:0.21\n",
      "I:51 Test-Acc:0.5165 Train-Acc:0.223\n",
      "I:52 Test-Acc:0.6007 Train-Acc:0.262\n",
      "I:53 Test-Acc:0.6476 Train-Acc:0.308\n",
      "I:54 Test-Acc:0.676 Train-Acc:0.363\n",
      "I:55 Test-Acc:0.696 Train-Acc:0.402\n",
      "I:56 Test-Acc:0.7077 Train-Acc:0.434\n",
      "I:57 Test-Acc:0.7204 Train-Acc:0.441\n",
      "I:58 Test-Acc:0.7303 Train-Acc:0.475\n",
      "I:59 Test-Acc:0.7359 Train-Acc:0.475\n",
      "I:60 Test-Acc:0.7401 Train-Acc:0.525\n",
      "I:61 Test-Acc:0.7493 Train-Acc:0.517\n",
      "I:62 Test-Acc:0.7533 Train-Acc:0.517\n",
      "I:63 Test-Acc:0.7606 Train-Acc:0.538\n",
      "I:64 Test-Acc:0.7644 Train-Acc:0.554\n",
      "I:65 Test-Acc:0.7724 Train-Acc:0.57\n",
      "I:66 Test-Acc:0.7788 Train-Acc:0.586\n",
      "I:67 Test-Acc:0.7855 Train-Acc:0.595\n",
      "I:68 Test-Acc:0.7853 Train-Acc:0.591\n",
      "I:69 Test-Acc:0.7925 Train-Acc:0.605\n",
      "I:70 Test-Acc:0.7973 Train-Acc:0.64\n",
      "I:71 Test-Acc:0.8013 Train-Acc:0.621\n",
      "I:72 Test-Acc:0.8029 Train-Acc:0.626\n",
      "I:73 Test-Acc:0.8092 Train-Acc:0.631\n",
      "I:74 Test-Acc:0.8099 Train-Acc:0.638\n",
      "I:75 Test-Acc:0.8156 Train-Acc:0.661\n",
      "I:76 Test-Acc:0.8156 Train-Acc:0.639\n",
      "I:77 Test-Acc:0.8184 Train-Acc:0.65\n",
      "I:78 Test-Acc:0.8216 Train-Acc:0.67\n",
      "I:79 Test-Acc:0.8246 Train-Acc:0.675\n",
      "I:80 Test-Acc:0.8237 Train-Acc:0.666\n",
      "I:81 Test-Acc:0.8273 Train-Acc:0.673\n",
      "I:82 Test-Acc:0.8273 Train-Acc:0.704\n",
      "I:83 Test-Acc:0.8314 Train-Acc:0.674\n",
      "I:84 Test-Acc:0.8292 Train-Acc:0.686\n",
      "I:85 Test-Acc:0.8335 Train-Acc:0.699\n",
      "I:86 Test-Acc:0.8359 Train-Acc:0.694\n",
      "I:87 Test-Acc:0.8375 Train-Acc:0.704\n",
      "I:88 Test-Acc:0.8373 Train-Acc:0.697\n",
      "I:89 Test-Acc:0.8398 Train-Acc:0.704\n",
      "I:90 Test-Acc:0.8393 Train-Acc:0.687\n",
      "I:91 Test-Acc:0.8436 Train-Acc:0.705\n",
      "I:92 Test-Acc:0.8437 Train-Acc:0.711\n",
      "I:93 Test-Acc:0.8446 Train-Acc:0.721\n",
      "I:94 Test-Acc:0.845 Train-Acc:0.719\n",
      "I:95 Test-Acc:0.8469 Train-Acc:0.724\n",
      "I:96 Test-Acc:0.8476 Train-Acc:0.726\n",
      "I:97 Test-Acc:0.848 Train-Acc:0.718\n",
      "I:98 Test-Acc:0.8496 Train-Acc:0.719\n",
      "I:99 Test-Acc:0.85 Train-Acc:0.73\n",
      "I:100 Test-Acc:0.8511 Train-Acc:0.737\n",
      "I:101 Test-Acc:0.8503 Train-Acc:0.73\n",
      "I:102 Test-Acc:0.8504 Train-Acc:0.717\n",
      "I:103 Test-Acc:0.8528 Train-Acc:0.74\n",
      "I:104 Test-Acc:0.8532 Train-Acc:0.733\n",
      "I:105 Test-Acc:0.8537 Train-Acc:0.73\n",
      "I:106 Test-Acc:0.8568 Train-Acc:0.721\n",
      "I:107 Test-Acc:0.857 Train-Acc:0.75\n",
      "I:108 Test-Acc:0.8558 Train-Acc:0.731\n",
      "I:109 Test-Acc:0.8578 Train-Acc:0.744\n",
      "I:110 Test-Acc:0.8588 Train-Acc:0.754\n",
      "I:111 Test-Acc:0.8579 Train-Acc:0.732\n",
      "I:112 Test-Acc:0.8582 Train-Acc:0.747\n",
      "I:113 Test-Acc:0.8593 Train-Acc:0.747\n",
      "I:114 Test-Acc:0.8598 Train-Acc:0.751\n",
      "I:115 Test-Acc:0.8603 Train-Acc:0.74\n",
      "I:116 Test-Acc:0.86 Train-Acc:0.753\n",
      "I:117 Test-Acc:0.8588 Train-Acc:0.746\n",
      "I:118 Test-Acc:0.861 Train-Acc:0.741\n",
      "I:119 Test-Acc:0.8616 Train-Acc:0.731\n",
      "I:120 Test-Acc:0.8629 Train-Acc:0.753\n",
      "I:121 Test-Acc:0.8609 Train-Acc:0.743\n",
      "I:122 Test-Acc:0.8627 Train-Acc:0.752\n",
      "I:123 Test-Acc:0.8646 Train-Acc:0.76\n",
      "I:124 Test-Acc:0.8649 Train-Acc:0.766\n",
      "I:125 Test-Acc:0.8659 Train-Acc:0.752\n",
      "I:126 Test-Acc:0.868 Train-Acc:0.756\n",
      "I:127 Test-Acc:0.8648 Train-Acc:0.767\n",
      "I:128 Test-Acc:0.8662 Train-Acc:0.747\n",
      "I:129 Test-Acc:0.8669 Train-Acc:0.753\n",
      "I:130 Test-Acc:0.8694 Train-Acc:0.753\n",
      "I:131 Test-Acc:0.8692 Train-Acc:0.76\n",
      "I:132 Test-Acc:0.8658 Train-Acc:0.756\n",
      "I:133 Test-Acc:0.8666 Train-Acc:0.769\n",
      "I:134 Test-Acc:0.8692 Train-Acc:0.77\n",
      "I:135 Test-Acc:0.8681 Train-Acc:0.757\n",
      "I:136 Test-Acc:0.8705 Train-Acc:0.77\n",
      "I:137 Test-Acc:0.8706 Train-Acc:0.77\n",
      "I:138 Test-Acc:0.8684 Train-Acc:0.768\n",
      "I:139 Test-Acc:0.8664 Train-Acc:0.774\n",
      "I:140 Test-Acc:0.8666 Train-Acc:0.756\n",
      "I:141 Test-Acc:0.8705 Train-Acc:0.783\n",
      "I:142 Test-Acc:0.87 Train-Acc:0.775\n",
      "I:143 Test-Acc:0.8729 Train-Acc:0.769\n",
      "I:144 Test-Acc:0.8725 Train-Acc:0.776\n",
      "I:145 Test-Acc:0.8721 Train-Acc:0.772\n",
      "I:146 Test-Acc:0.8718 Train-Acc:0.765\n",
      "I:147 Test-Acc:0.8746 Train-Acc:0.777\n",
      "I:148 Test-Acc:0.8746 Train-Acc:0.77\n",
      "I:149 Test-Acc:0.8734 Train-Acc:0.778\n",
      "I:150 Test-Acc:0.873 Train-Acc:0.785\n",
      "I:151 Test-Acc:0.8732 Train-Acc:0.76\n",
      "I:152 Test-Acc:0.8727 Train-Acc:0.779\n",
      "I:153 Test-Acc:0.8754 Train-Acc:0.772\n",
      "I:154 Test-Acc:0.8729 Train-Acc:0.773\n",
      "I:155 Test-Acc:0.8758 Train-Acc:0.784\n",
      "I:156 Test-Acc:0.8732 Train-Acc:0.774\n",
      "I:157 Test-Acc:0.8743 Train-Acc:0.782\n",
      "I:158 Test-Acc:0.8762 Train-Acc:0.772\n",
      "I:159 Test-Acc:0.8755 Train-Acc:0.79\n",
      "I:160 Test-Acc:0.8751 Train-Acc:0.774\n",
      "I:161 Test-Acc:0.8749 Train-Acc:0.782\n",
      "I:162 Test-Acc:0.8744 Train-Acc:0.78\n",
      "I:163 Test-Acc:0.8765 Train-Acc:0.782\n",
      "I:164 Test-Acc:0.8738 Train-Acc:0.796\n",
      "I:165 Test-Acc:0.8753 Train-Acc:0.798\n",
      "I:166 Test-Acc:0.8767 Train-Acc:0.794\n",
      "I:167 Test-Acc:0.8746 Train-Acc:0.784\n",
      "I:168 Test-Acc:0.8769 Train-Acc:0.796\n",
      "I:169 Test-Acc:0.8758 Train-Acc:0.789\n",
      "I:170 Test-Acc:0.8764 Train-Acc:0.79\n",
      "I:171 Test-Acc:0.873 Train-Acc:0.791\n",
      "I:172 Test-Acc:0.8765 Train-Acc:0.797\n",
      "I:173 Test-Acc:0.8772 Train-Acc:0.789\n",
      "I:174 Test-Acc:0.8778 Train-Acc:0.781\n",
      "I:175 Test-Acc:0.8758 Train-Acc:0.799\n",
      "I:176 Test-Acc:0.8773 Train-Acc:0.785\n",
      "I:177 Test-Acc:0.8766 Train-Acc:0.796\n",
      "I:178 Test-Acc:0.8782 Train-Acc:0.803\n",
      "I:179 Test-Acc:0.8789 Train-Acc:0.794\n",
      "I:180 Test-Acc:0.8778 Train-Acc:0.794\n",
      "I:181 Test-Acc:0.8778 Train-Acc:0.8\n",
      "I:182 Test-Acc:0.8785 Train-Acc:0.791\n",
      "I:183 Test-Acc:0.8777 Train-Acc:0.787\n",
      "I:184 Test-Acc:0.8769 Train-Acc:0.781\n",
      "I:185 Test-Acc:0.8765 Train-Acc:0.786\n",
      "I:186 Test-Acc:0.8765 Train-Acc:0.793\n",
      "I:187 Test-Acc:0.8785 Train-Acc:0.796\n",
      "I:188 Test-Acc:0.879 Train-Acc:0.789\n",
      "I:189 Test-Acc:0.8763 Train-Acc:0.79\n",
      "I:190 Test-Acc:0.8774 Train-Acc:0.787\n",
      "I:191 Test-Acc:0.8766 Train-Acc:0.782\n",
      "I:192 Test-Acc:0.8803 Train-Acc:0.798\n",
      "I:193 Test-Acc:0.8781 Train-Acc:0.789\n",
      "I:194 Test-Acc:0.8795 Train-Acc:0.785\n",
      "I:195 Test-Acc:0.8791 Train-Acc:0.807\n",
      "I:196 Test-Acc:0.8778 Train-Acc:0.796\n",
      "I:197 Test-Acc:0.8783 Train-Acc:0.801\n",
      "I:198 Test-Acc:0.8778 Train-Acc:0.81\n",
      "I:199 Test-Acc:0.8771 Train-Acc:0.784\n",
      "I:200 Test-Acc:0.8776 Train-Acc:0.792\n",
      "I:201 Test-Acc:0.8784 Train-Acc:0.794\n",
      "I:202 Test-Acc:0.8787 Train-Acc:0.795\n",
      "I:203 Test-Acc:0.8803 Train-Acc:0.781\n",
      "I:204 Test-Acc:0.8798 Train-Acc:0.804\n",
      "I:205 Test-Acc:0.8779 Train-Acc:0.779\n",
      "I:206 Test-Acc:0.8788 Train-Acc:0.792\n",
      "I:207 Test-Acc:0.8764 Train-Acc:0.793\n",
      "I:208 Test-Acc:0.8792 Train-Acc:0.792\n",
      "I:209 Test-Acc:0.8798 Train-Acc:0.803\n",
      "I:210 Test-Acc:0.8788 Train-Acc:0.804\n",
      "I:211 Test-Acc:0.8793 Train-Acc:0.797\n",
      "I:212 Test-Acc:0.8764 Train-Acc:0.791\n",
      "I:213 Test-Acc:0.8801 Train-Acc:0.801\n",
      "I:214 Test-Acc:0.8814 Train-Acc:0.799\n",
      "I:215 Test-Acc:0.8806 Train-Acc:0.79\n",
      "I:216 Test-Acc:0.8799 Train-Acc:0.8\n",
      "I:217 Test-Acc:0.8803 Train-Acc:0.802\n",
      "I:218 Test-Acc:0.8782 Train-Acc:0.807\n",
      "I:219 Test-Acc:0.8818 Train-Acc:0.797\n",
      "I:220 Test-Acc:0.8793 Train-Acc:0.799\n",
      "I:221 Test-Acc:0.8789 Train-Acc:0.815\n",
      "I:222 Test-Acc:0.8791 Train-Acc:0.816\n",
      "I:223 Test-Acc:0.8793 Train-Acc:0.809\n",
      "I:224 Test-Acc:0.8814 Train-Acc:0.795\n",
      "I:225 Test-Acc:0.8798 Train-Acc:0.799\n",
      "I:226 Test-Acc:0.8805 Train-Acc:0.806\n",
      "I:227 Test-Acc:0.88 Train-Acc:0.808\n",
      "I:228 Test-Acc:0.8782 Train-Acc:0.801\n",
      "I:229 Test-Acc:0.8802 Train-Acc:0.814\n",
      "I:230 Test-Acc:0.8807 Train-Acc:0.8\n",
      "I:231 Test-Acc:0.8809 Train-Acc:0.798\n",
      "I:232 Test-Acc:0.8805 Train-Acc:0.82\n",
      "I:233 Test-Acc:0.8795 Train-Acc:0.794\n",
      "I:234 Test-Acc:0.8807 Train-Acc:0.806\n",
      "I:235 Test-Acc:0.8806 Train-Acc:0.808\n",
      "I:236 Test-Acc:0.8787 Train-Acc:0.802\n",
      "I:237 Test-Acc:0.8796 Train-Acc:0.81\n",
      "I:238 Test-Acc:0.8766 Train-Acc:0.805\n",
      "I:239 Test-Acc:0.8781 Train-Acc:0.792\n",
      "I:240 Test-Acc:0.8787 Train-Acc:0.809\n",
      "I:241 Test-Acc:0.8762 Train-Acc:0.802\n",
      "I:242 Test-Acc:0.8775 Train-Acc:0.811\n",
      "I:243 Test-Acc:0.8804 Train-Acc:0.814\n",
      "I:244 Test-Acc:0.8794 Train-Acc:0.804\n",
      "I:245 Test-Acc:0.8788 Train-Acc:0.801\n",
      "I:246 Test-Acc:0.8777 Train-Acc:0.795\n",
      "I:247 Test-Acc:0.8785 Train-Acc:0.808\n",
      "I:248 Test-Acc:0.8788 Train-Acc:0.803\n",
      "I:249 Test-Acc:0.8773 Train-Acc:0.813\n",
      "I:250 Test-Acc:0.8786 Train-Acc:0.808\n",
      "I:251 Test-Acc:0.8787 Train-Acc:0.803\n",
      "I:252 Test-Acc:0.8789 Train-Acc:0.812\n",
      "I:253 Test-Acc:0.8792 Train-Acc:0.804\n",
      "I:254 Test-Acc:0.8779 Train-Acc:0.815\n",
      "I:255 Test-Acc:0.8796 Train-Acc:0.811\n",
      "I:256 Test-Acc:0.8798 Train-Acc:0.806\n",
      "I:257 Test-Acc:0.88 Train-Acc:0.803\n",
      "I:258 Test-Acc:0.8776 Train-Acc:0.795\n",
      "I:259 Test-Acc:0.8798 Train-Acc:0.803\n",
      "I:260 Test-Acc:0.8799 Train-Acc:0.805\n",
      "I:261 Test-Acc:0.8789 Train-Acc:0.807\n",
      "I:262 Test-Acc:0.8784 Train-Acc:0.804\n",
      "I:263 Test-Acc:0.8792 Train-Acc:0.806\n",
      "I:264 Test-Acc:0.8777 Train-Acc:0.796\n",
      "I:265 Test-Acc:0.8785 Train-Acc:0.821\n",
      "I:266 Test-Acc:0.8794 Train-Acc:0.81\n",
      "I:267 Test-Acc:0.8783 Train-Acc:0.816\n",
      "I:268 Test-Acc:0.8777 Train-Acc:0.812\n",
      "I:269 Test-Acc:0.8791 Train-Acc:0.812\n",
      "I:270 Test-Acc:0.878 Train-Acc:0.813\n",
      "I:271 Test-Acc:0.8784 Train-Acc:0.82\n",
      "I:272 Test-Acc:0.8792 Train-Acc:0.821\n",
      "I:273 Test-Acc:0.8781 Train-Acc:0.823\n",
      "I:274 Test-Acc:0.8788 Train-Acc:0.816\n",
      "I:275 Test-Acc:0.8793 Train-Acc:0.82\n",
      "I:276 Test-Acc:0.8781 Train-Acc:0.829\n",
      "I:277 Test-Acc:0.8795 Train-Acc:0.809\n",
      "I:278 Test-Acc:0.875 Train-Acc:0.806\n",
      "I:279 Test-Acc:0.8795 Train-Acc:0.813\n",
      "I:280 Test-Acc:0.88 Train-Acc:0.816\n",
      "I:281 Test-Acc:0.8796 Train-Acc:0.819\n",
      "I:282 Test-Acc:0.8802 Train-Acc:0.809\n",
      "I:283 Test-Acc:0.8804 Train-Acc:0.811\n",
      "I:284 Test-Acc:0.8779 Train-Acc:0.808\n",
      "I:285 Test-Acc:0.8816 Train-Acc:0.82\n",
      "I:286 Test-Acc:0.8792 Train-Acc:0.822\n",
      "I:287 Test-Acc:0.8791 Train-Acc:0.817\n",
      "I:288 Test-Acc:0.8769 Train-Acc:0.814\n",
      "I:289 Test-Acc:0.8785 Train-Acc:0.807\n",
      "I:290 Test-Acc:0.8778 Train-Acc:0.817\n",
      "I:291 Test-Acc:0.8794 Train-Acc:0.82\n",
      "I:292 Test-Acc:0.8804 Train-Acc:0.824\n",
      "I:293 Test-Acc:0.8779 Train-Acc:0.812\n",
      "I:294 Test-Acc:0.8784 Train-Acc:0.816\n",
      "I:295 Test-Acc:0.877 Train-Acc:0.817\n",
      "I:296 Test-Acc:0.8767 Train-Acc:0.826\n",
      "I:297 Test-Acc:0.8774 Train-Acc:0.816\n",
      "I:298 Test-Acc:0.8774 Train-Acc:0.804\n",
      "I:299 Test-Acc:0.8774 Train-Acc:0.814"
     ]
    }
   ],
   "source": [
    "import numpy as np, sys\n",
    "np.random.seed(1)\n",
    "\n",
    "from keras.datasets import mnist\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "images, labels = (x_train[0:1000].reshape(1000,28*28) / 255,\n",
    "                  y_train[0:1000])\n",
    "\n",
    "\n",
    "one_hot_labels = np.zeros((len(labels),10))\n",
    "for i,l in enumerate(labels):\n",
    "    one_hot_labels[i][l] = 1\n",
    "labels = one_hot_labels\n",
    "\n",
    "test_images = x_test.reshape(len(x_test),28*28) / 255\n",
    "test_labels = np.zeros((len(y_test),10))\n",
    "for i,l in enumerate(y_test):\n",
    "    test_labels[i][l] = 1\n",
    "\n",
    "def tanh(x):\n",
    "    return np.tanh(x)\n",
    "\n",
    "def tanh2deriv(output):\n",
    "    return 1 - (output ** 2)\n",
    "\n",
    "def softmax(x):\n",
    "    temp = np.exp(x)\n",
    "    return temp / np.sum(temp, axis=1, keepdims=True)\n",
    "\n",
    "alpha, iterations = (2, 300)\n",
    "pixels_per_image, num_labels = (784, 10)\n",
    "batch_size = 128\n",
    "\n",
    "input_rows = 28\n",
    "input_cols = 28\n",
    "\n",
    "kernel_rows = 3\n",
    "kernel_cols = 3\n",
    "num_kernels = 16\n",
    "\n",
    "hidden_size = ((input_rows - kernel_rows) * \n",
    "               (input_cols - kernel_cols)) * num_kernels\n",
    "\n",
    "# weights_0_1 = 0.02*np.random.random((pixels_per_image,hidden_size))-0.01\n",
    "kernels = 0.02*np.random.random((kernel_rows*kernel_cols,\n",
    "                                 num_kernels))-0.01\n",
    "\n",
    "weights_1_2 = 0.2*np.random.random((hidden_size,\n",
    "                                    num_labels)) - 0.1\n",
    "\n",
    "\n",
    "\n",
    "def get_image_section(layer,row_from, row_to, col_from, col_to):\n",
    "    section = layer[:,row_from:row_to,col_from:col_to]\n",
    "    return section.reshape(-1,1,row_to-row_from, col_to-col_from)\n",
    "\n",
    "for j in range(iterations):\n",
    "    correct_cnt = 0\n",
    "    for i in range(int(len(images) / batch_size)):\n",
    "        batch_start, batch_end=((i * batch_size),((i+1)*batch_size))\n",
    "        layer_0 = images[batch_start:batch_end]\n",
    "        layer_0 = layer_0.reshape(layer_0.shape[0],28,28)\n",
    "        layer_0.shape\n",
    "\n",
    "        sects = list()\n",
    "        for row_start in range(layer_0.shape[1]-kernel_rows):\n",
    "            for col_start in range(layer_0.shape[2] - kernel_cols):\n",
    "                sect = get_image_section(layer_0,\n",
    "                                         row_start,\n",
    "                                         row_start+kernel_rows,\n",
    "                                         col_start,\n",
    "                                         col_start+kernel_cols)\n",
    "                sects.append(sect)\n",
    "\n",
    "        expanded_input = np.concatenate(sects,axis=1)\n",
    "        es = expanded_input.shape\n",
    "        flattened_input = expanded_input.reshape(es[0]*es[1],-1)\n",
    "\n",
    "        kernel_output = flattened_input.dot(kernels)\n",
    "        layer_1 = tanh(kernel_output.reshape(es[0],-1))\n",
    "        dropout_mask = np.random.randint(2,size=layer_1.shape)\n",
    "        layer_1 *= dropout_mask * 2\n",
    "        layer_2 = softmax(np.dot(layer_1,weights_1_2))\n",
    "\n",
    "        for k in range(batch_size):\n",
    "            labelset = labels[batch_start+k:batch_start+k+1]\n",
    "            _inc = int(np.argmax(layer_2[k:k+1]) == \n",
    "                               np.argmax(labelset))\n",
    "            correct_cnt += _inc\n",
    "\n",
    "        layer_2_delta = (labels[batch_start:batch_end]-layer_2)\\\n",
    "                        / (batch_size * layer_2.shape[0])\n",
    "        layer_1_delta = layer_2_delta.dot(weights_1_2.T) * \\\n",
    "                        tanh2deriv(layer_1)\n",
    "        layer_1_delta *= dropout_mask\n",
    "        weights_1_2 += alpha * layer_1.T.dot(layer_2_delta)\n",
    "        l1d_reshape = layer_1_delta.reshape(kernel_output.shape)\n",
    "        k_update = flattened_input.T.dot(l1d_reshape)\n",
    "        kernels -= alpha * k_update\n",
    "    \n",
    "    test_correct_cnt = 0\n",
    "\n",
    "    for i in range(len(test_images)):\n",
    "\n",
    "        layer_0 = test_images[i:i+1]\n",
    "#         layer_1 = tanh(np.dot(layer_0,weights_0_1))\n",
    "        layer_0 = layer_0.reshape(layer_0.shape[0],28,28)\n",
    "        layer_0.shape\n",
    "\n",
    "        sects = list()\n",
    "        for row_start in range(layer_0.shape[1]-kernel_rows):\n",
    "            for col_start in range(layer_0.shape[2] - kernel_cols):\n",
    "                sect = get_image_section(layer_0,\n",
    "                                         row_start,\n",
    "                                         row_start+kernel_rows,\n",
    "                                         col_start,\n",
    "                                         col_start+kernel_cols)\n",
    "                sects.append(sect)\n",
    "\n",
    "        expanded_input = np.concatenate(sects,axis=1)\n",
    "        es = expanded_input.shape\n",
    "        flattened_input = expanded_input.reshape(es[0]*es[1],-1)\n",
    "\n",
    "        kernel_output = flattened_input.dot(kernels)\n",
    "        layer_1 = tanh(kernel_output.reshape(es[0],-1))\n",
    "        layer_2 = np.dot(layer_1,weights_1_2)\n",
    "\n",
    "        test_correct_cnt += int(np.argmax(layer_2) == \n",
    "                                np.argmax(test_labels[i:i+1]))\n",
    "    if(j % 1 == 0):\n",
    "        sys.stdout.write(\"\\n\"+ \\\n",
    "         \"I:\" + str(j) + \\\n",
    "         \" Test-Acc:\"+str(test_correct_cnt/float(len(test_images)))+\\\n",
    "         \" Train-Acc:\" + str(correct_cnt/float(len(images))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}