Lenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import mnist\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import np_utils\n",
    "\n",
    "# imports used to build the deep learning model\n",
    "from keras.models import Sequential\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.layers.core import Activation\n",
    "from keras.layers.core import Flatten\n",
    "from keras.layers.core import Dense\n",
    "from keras.layers import Dropout\n",
    "\n",
    "from TNT import kernels_cluster\n",
    "from keras.models import load_model\n",
    "\n",
    "import numpy as np\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Definition of a LeNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_lenet(width, height, depth, classes, weightsPath=None):\n",
    "    # Initialize the model\n",
    "    model = Sequential()\n",
    "\n",
    "    # The first set of CONV => RELU => POOL layers\n",
    "    # If you need a traditional lenet, you can change the to padding=\"vailed\"\n",
    "    model.add(Conv2D(20, (5, 5), padding=\"same\",\n",
    "                     input_shape=(height, width, depth), name='CONV1'))\n",
    "    model.add(Activation(\"relu\", name='relu1'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='Pooling1'))\n",
    "\n",
    "    # The second set of CONV => RELU => POOL layers\n",
    "    model.add(Conv2D(50, (5, 5), padding=\"same\", name='CONV2'))\n",
    "    model.add(Activation(\"relu\", name='relu2'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='Pooling2'))\n",
    "    model.add(Dropout(0.025))\n",
    "\n",
    "    # The set of FC => RELU layers\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(500))\n",
    "    model.add(Activation(\"relu\", name='relu3'))\n",
    "    model.add(Dropout(0.025))\n",
    "    \n",
    "    # The softmax classifier\n",
    "    model.add(Dense(classes))\n",
    "    model.add(Activation(\"softmax\", name='softmax'))\n",
    "\n",
    "    # If a weights path is supplied, then load the weights\n",
    "    if weightsPath is not None:\n",
    "        model.load_weights(weightsPath)\n",
    "\n",
    "    # Return the constructed network architecture\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Preparing The MNIST Dataset\n",
    "\n",
    "We prepare two types of input dataset. The first one is ternary type, the second one is normal type. <br>\n",
    "1. Ternary input: is that every pixel is converted to -1, 0 or 1. <br>\n",
    "2. Normal input: is that every pixel is in range 0~255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Loading the MNIST Normal dataset...\n"
     ]
    }
   ],
   "source": [
    "print(\"[INFO] Loading the MNIST Normal dataset...\")\n",
    "(trainData, trainLabels), (testData, testLabels) = mnist.load_data()\n",
    "trainData = trainData[:, :, :, np.newaxis]\n",
    "testData = testData[:, :, :, np.newaxis]\n",
    "# Rescale the data from values between [0 - 255] to [0 - 1.0]\n",
    "trainData = trainData / 255.0\n",
    "testData = testData / 255.0\n",
    "trainLabels = np_utils.to_categorical(trainLabels, 10)\n",
    "testLabels = np_utils.to_categorical(testLabels, 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Initializing The LeNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Building and compiling the LeNet model...\n",
      "WARNING:tensorflow:From /Users/Tenngre/anaconda/envs/tf/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /Users/Tenngre/anaconda/envs/tf/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "CONV1 (Conv2D)               (None, 28, 28, 20)        520       \n",
      "_________________________________________________________________\n",
      "relu1 (Activation)           (None, 28, 28, 20)        0         \n",
      "_________________________________________________________________\n",
      "Pooling1 (MaxPooling2D)      (None, 14, 14, 20)        0         \n",
      "_________________________________________________________________\n",
      "CONV2 (Conv2D)               (None, 14, 14, 50)        25050     \n",
      "_________________________________________________________________\n",
      "relu2 (Activation)           (None, 14, 14, 50)        0         \n",
      "_________________________________________________________________\n",
      "Pooling2 (MaxPooling2D)      (None, 7, 7, 50)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 7, 7, 50)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 2450)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 500)               1225500   \n",
      "_________________________________________________________________\n",
      "relu3 (Activation)           (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                5010      \n",
      "_________________________________________________________________\n",
      "softmax (Activation)         (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,256,080\n",
      "Trainable params: 1,256,080\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Build and Compile the model\n",
    "print(\"[INFO] Building and compiling the LeNet model...\")\n",
    "model = build_lenet(width=28, height=28, depth=1, classes=10)\n",
    "opt = SGD(lr=0.01)\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=opt, metrics=[\"accuracy\"])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Training the Lenet by Ternary Input or Normal Input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 Normal input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"[INFO] Using Normal Input to train a model...\")\n",
    "history = model.fit(trainData,\n",
    "                    trainLabels,\n",
    "                    batch_size=128,\n",
    "                    epochs=3,\n",
    "                    validation_data=(testData, testLabels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 10s 1ms/step\n",
      "[INFO] accuracy: 95.92%\n"
     ]
    }
   ],
   "source": [
    "(loss, accuracy) = model.evaluate(\n",
    "        testData, testLabels, batch_size=128, verbose=1)\n",
    "print(\"[INFO] accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./model/LeNet_epoch3.model', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 Ternar Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/Tenngre/anaconda/envs/tf/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "model = load_model('./model/LeNet_epoch3.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = model.get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[[-0.00435827 -0.01967984 -0.06710408  0.06321401 -0.06320184 -0.05127101\n",
      "     0.10110379 -0.00498402  0.0738114  -0.05353865  0.00485416  0.0258638\n",
      "     0.12360302 -0.03986602 -0.06357484 -0.10580987 -0.01490379  0.09963966\n",
      "     0.03353032  0.02118958]]\n",
      "\n",
      "  [[-0.0117531   0.05213843  0.09928356  0.20159774  0.01188824 -0.03079138\n",
      "     0.08051449 -0.02179168  0.03302421  0.00452769  0.0532811   0.0674924\n",
      "     0.18953568  0.07028069 -0.09385809 -0.0202191  -0.01219152  0.07385656\n",
      "    -0.04336262  0.01123895]]\n",
      "\n",
      "  [[-0.03193782 -0.06312631  0.05173955  0.07448452  0.04035129  0.02129952\n",
      "     0.00869875  0.05544435  0.1461038   0.0507984   0.05108389  0.11168292\n",
      "     0.04904866  0.07648798  0.03072315  0.06694351  0.07726569  0.04849633\n",
      "     0.14006896  0.11014114]]\n",
      "\n",
      "  [[-0.00787933 -0.0776803   0.13880318  0.25237063 -0.04807344  0.07678764\n",
      "    -0.0275468  -0.10517928  0.20285228 -0.03554668 -0.03340333  0.13602601\n",
      "     0.16514519  0.07247213  0.00947469 -0.01465164 -0.01946673 -0.03018964\n",
      "     0.0044082   0.09602119]]\n",
      "\n",
      "  [[-0.06235202  0.02910736  0.15605931  0.14151424  0.05167384  0.06492435\n",
      "    -0.12011625  0.00072959  0.17671713  0.05960422  0.15555184  0.10793111\n",
      "     0.15575735 -0.01598057 -0.08454949  0.148344   -0.03483005 -0.11366878\n",
      "     0.08737414 -0.0696504 ]]]\n",
      "\n",
      "\n",
      " [[[ 0.06494549  0.04825901  0.04608867  0.07701489 -0.05993285 -0.02895103\n",
      "     0.02879993  0.13280444  0.15065815  0.05597528  0.08017064  0.15426412\n",
      "     0.18027383 -0.00367803 -0.09099951  0.11720237  0.08599925 -0.00168028\n",
      "     0.11796108  0.03170283]]\n",
      "\n",
      "  [[-0.01826289  0.02390848  0.10998122  0.21902144  0.03904799 -0.10623926\n",
      "     0.01690796 -0.01115895  0.33293721  0.09246352  0.09947945  0.0166105\n",
      "     0.16712238  0.01935951 -0.10138366  0.07143378  0.02517279 -0.00517525\n",
      "     0.06353141 -0.05064769]]\n",
      "\n",
      "  [[ 0.06232813 -0.04067485 -0.07728908  0.27254772  0.11421649  0.03808641\n",
      "     0.04870418 -0.04889963  0.23553437  0.12371945  0.09471604  0.01874725\n",
      "     0.2414254   0.12272763 -0.06528312  0.18072882 -0.09687384 -0.00046045\n",
      "     0.16837451  0.04136565]]\n",
      "\n",
      "  [[-0.02570512  0.10160709 -0.06203042  0.25303346 -0.01602367 -0.03538483\n",
      "    -0.01125128  0.09153099  0.16182601  0.11645611  0.02769256  0.01851973\n",
      "     0.31121197  0.08806411 -0.05476141  0.15749876 -0.05020715 -0.05850545\n",
      "     0.17298315 -0.05034189]]\n",
      "\n",
      "  [[ 0.02696656 -0.05199277  0.00407294  0.15109979 -0.09870796  0.03689433\n",
      "    -0.04877407  0.04780663 -0.0280092   0.10981962  0.12657447 -0.027824\n",
      "     0.28961745  0.08050913 -0.01153267  0.20899901 -0.01743491 -0.02494592\n",
      "     0.12226623 -0.07284845]]]\n",
      "\n",
      "\n",
      " [[[-0.10643747 -0.07311577  0.01762117 -0.00288277  0.1728069  -0.03244304\n",
      "     0.07924442  0.20997275  0.26175493  0.08544417 -0.07271399  0.18077345\n",
      "     0.1544207  -0.01964751  0.01429246  0.03792574  0.05093374  0.06879216\n",
      "     0.10791804 -0.01545079]]\n",
      "\n",
      "  [[-0.08605142  0.06951737 -0.09856971  0.1250758   0.21692219  0.01891944\n",
      "     0.08269262  0.19888054  0.37056565  0.02251852  0.11072043  0.0583199\n",
      "     0.26057333 -0.08385321 -0.10177744  0.19673289  0.07825981 -0.0114231\n",
      "    -0.00492251  0.15015186]]\n",
      "\n",
      "  [[ 0.05910559  0.09473443  0.09927643  0.04492323  0.15214863 -0.08121326\n",
      "    -0.02831699  0.04179739  0.29596159 -0.07892604 -0.08542794  0.16600239\n",
      "     0.30444935 -0.03347292  0.09194568  0.06907433  0.03945638  0.07230635\n",
      "    -0.03533566  0.05228685]]\n",
      "\n",
      "  [[-0.03622774  0.04843633  0.01713933  0.08772536 -0.01514936 -0.02575658\n",
      "    -0.08634857  0.09904777  0.2114207  -0.02611267 -0.08629537  0.01671807\n",
      "     0.24375203 -0.01397049 -0.03197884  0.16862354  0.02359799 -0.08926916\n",
      "     0.04146152  0.06861872]]\n",
      "\n",
      "  [[ 0.09979059 -0.08648511 -0.03815742  0.08045418 -0.08572837 -0.00878031\n",
      "    -0.05170009  0.18184473  0.00597365  0.03003508 -0.08637859 -0.0527273\n",
      "     0.24190789 -0.03188134  0.0555204   0.09767196 -0.07450398 -0.07577799\n",
      "     0.1535057  -0.01210037]]]\n",
      "\n",
      "\n",
      " [[[-0.08441014 -0.02548266 -0.09791481 -0.13388224  0.1184753  -0.10508379\n",
      "     0.04114079  0.22155397  0.20315218 -0.08193905 -0.04000217  0.09201087\n",
      "     0.00679208 -0.08604646  0.07698853  0.14164767 -0.06090927  0.07342381\n",
      "     0.04439288  0.02641506]]\n",
      "\n",
      "  [[-0.09207145 -0.02909139  0.04886989 -0.12114567  0.19846201 -0.09434848\n",
      "     0.16796359  0.21622574  0.26215798  0.02464745 -0.05313511 -0.00763761\n",
      "     0.02968712  0.01224351  0.00169438  0.18372074  0.10925989  0.02533767\n",
      "    -0.01039246  0.07101851]]\n",
      "\n",
      "  [[ 0.07865305  0.00072564  0.07548856 -0.02795509  0.21337986 -0.05844011\n",
      "     0.15238357  0.16117169  0.09476484 -0.00054631 -0.04278982  0.04693601\n",
      "     0.16569969  0.03395582 -0.0630323   0.05630404  0.00096086 -0.02049677\n",
      "    -0.01587964  0.08556902]]\n",
      "\n",
      "  [[ 0.12075347  0.07302262 -0.04340858  0.07738796  0.22409461 -0.08442714\n",
      "     0.0470022   0.09957411  0.04004869 -0.07753942 -0.08919508 -0.06729991\n",
      "     0.1646103   0.10831375 -0.06729644  0.24781564  0.0670374  -0.06905966\n",
      "    -0.06713153 -0.08194428]]\n",
      "\n",
      "  [[ 0.0085888  -0.08353163  0.07433286  0.03688115  0.01693014  0.11081713\n",
      "     0.07090779  0.20967168  0.12355132  0.05236848  0.04330001 -0.02413918\n",
      "     0.26313284  0.0777436  -0.01529649  0.13671792 -0.08182697  0.04265261\n",
      "     0.04983046 -0.09042   ]]]\n",
      "\n",
      "\n",
      " [[[-0.06913865 -0.07760926 -0.08470799 -0.16330869  0.06341083  0.05964513\n",
      "     0.03397259  0.07241012  0.01003554 -0.08996693 -0.08678044  0.17180112\n",
      "    -0.13337535 -0.12578648  0.13573204  0.02748282 -0.01067578 -0.03445905\n",
      "     0.02752423  0.14263988]]\n",
      "\n",
      "  [[-0.04299769 -0.01128385 -0.05776047 -0.01604564  0.13374619 -0.02544969\n",
      "     0.15533683  0.19014338  0.17153752 -0.03726325 -0.00560867  0.11583049\n",
      "    -0.06078475 -0.04188559  0.06397285  0.04288049 -0.08825669 -0.04584052\n",
      "    -0.11195467  0.07144308]]\n",
      "\n",
      "  [[-0.04869052  0.0690979   0.10571862  0.06843176  0.22292234 -0.00109917\n",
      "     0.18795435  0.09425131  0.07777541  0.00507796 -0.10323952  0.0341868\n",
      "     0.06732406 -0.09554845  0.04002761  0.15592878  0.08267929 -0.06225707\n",
      "    -0.09844304  0.02423372]]\n",
      "\n",
      "  [[ 0.07486844 -0.07715759  0.06404155 -0.10502557  0.1778155  -0.02158701\n",
      "    -0.02728299  0.0548977  -0.02776126 -0.04455968 -0.01743615  0.02921754\n",
      "     0.19040273  0.04313725  0.07109396  0.05012672  0.11547043 -0.03785293\n",
      "    -0.11336391 -0.00314016]]\n",
      "\n",
      "  [[ 0.11615828 -0.08284632 -0.0655909  -0.01177603  0.19926038  0.11226904\n",
      "     0.12929447  0.05413377 -0.03856902 -0.03218894 -0.08815847 -0.05781315\n",
      "     0.15364304  0.11205756 -0.01139643 -0.00851163  0.00635307 -0.07778485\n",
      "    -0.03290505 -0.10135616]]]]\n"
     ]
    }
   ],
   "source": [
    "print(weights[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, len(weights)):\n",
    "    print(i)\n",
    "    weights[i] = kernels_cluster(weights[i])\n",
    "model.set_weights(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[[ 0.          0.         -0.07270797  0.          0.         -0.08300267\n",
      "     0.12073883  0.          0.         -0.0710777   0.          0.\n",
      "     0.20875083  0.         -0.07865113 -0.10580776  0.          0.06845199\n",
      "     0.          0.        ]]\n",
      "\n",
      "  [[ 0.          0.06960116  0.08913977  0.18933053  0.          0.\n",
      "     0.12073883  0.          0.          0.          0.09644656  0.13038068\n",
      "     0.20875083  0.08985013 -0.07865113  0.          0.          0.06845199\n",
      "     0.          0.        ]]\n",
      "\n",
      "  [[ 0.         -0.07142154  0.08913977  0.          0.          0.          0.\n",
      "     0.          0.22044756  0.08296047  0.09644656  0.13038068  0.\n",
      "     0.08985013  0.          0.          0.07848413  0.06845199  0.1259973\n",
      "     0.0942094 ]]\n",
      "\n",
      "  [[ 0.         -0.07142154  0.08913977  0.18933053  0.          0.0848879\n",
      "     0.         -0.10517718  0.22044756  0.          0.          0.13038068\n",
      "     0.20875083  0.08985013  0.          0.          0.          0.          0.\n",
      "     0.0942094 ]]\n",
      "\n",
      "  [[-0.07401815  0.          0.08913977  0.18933053  0.          0.0848879\n",
      "    -0.10323096  0.          0.22044756  0.08296047  0.09644656  0.13038068\n",
      "     0.20875083  0.         -0.07865113  0.16474001  0.         -0.06644712\n",
      "     0.1259973  -0.07388642]]]\n",
      "\n",
      "\n",
      " [[[ 0.08457479  0.06960116  0.08913977  0.          0.          0.          0.\n",
      "     0.16205089  0.22044756  0.08296047  0.09644656  0.13038068  0.20875083\n",
      "     0.         -0.07865113  0.16474001  0.07848413  0.          0.1259973\n",
      "     0.        ]]\n",
      "\n",
      "  [[ 0.          0.          0.08913977  0.18933053  0.         -0.08300267\n",
      "     0.          0.          0.22044756  0.08296047  0.09644656  0.\n",
      "     0.20875083  0.         -0.07865113  0.          0.          0.\n",
      "     0.1259973  -0.07388642]]\n",
      "\n",
      "  [[ 0.08457479 -0.07142154 -0.07270797  0.18933053  0.1786865   0.          0.\n",
      "     0.          0.22044756  0.08296047  0.09644656  0.          0.20875083\n",
      "     0.08985013 -0.07865113  0.16474001 -0.07542904  0.          0.1259973\n",
      "     0.        ]]\n",
      "\n",
      "  [[ 0.          0.06960116 -0.07270797  0.18933053  0.          0.          0.\n",
      "     0.16205089  0.22044756  0.08296047  0.          0.          0.20875083\n",
      "     0.08985013 -0.07865113  0.16474001 -0.07542904 -0.06644712  0.1259973\n",
      "    -0.07388642]]\n",
      "\n",
      "  [[ 0.         -0.07142154  0.          0.18933053 -0.09221686  0.          0.\n",
      "     0.          0.          0.08296047  0.09644656  0.          0.20875083\n",
      "     0.08985013  0.          0.16474001  0.          0.          0.1259973\n",
      "    -0.07388642]]]\n",
      "\n",
      "\n",
      " [[[-0.07401815 -0.07142154  0.          0.          0.1786865   0.\n",
      "     0.12073883  0.16205089  0.22044756  0.08296047 -0.08347994  0.13038068\n",
      "     0.20875083  0.          0.          0.          0.07848413  0.06845199\n",
      "     0.1259973   0.        ]]\n",
      "\n",
      "  [[-0.07401815  0.06960116 -0.07270797  0.18933053  0.1786865   0.\n",
      "     0.12073883  0.16205089  0.22044756  0.          0.09644656  0.\n",
      "     0.20875083 -0.09780768 -0.07865113  0.16474001  0.07848413  0.          0.\n",
      "     0.0942094 ]]\n",
      "\n",
      "  [[ 0.08457479  0.06960116  0.08913977  0.          0.1786865  -0.08300267\n",
      "     0.          0.          0.22044756 -0.0710777  -0.08347994  0.13038068\n",
      "     0.20875083  0.          0.07646815  0.          0.07848413  0.06845199\n",
      "     0.          0.0942094 ]]\n",
      "\n",
      "  [[ 0.          0.06960116  0.          0.18933053  0.          0.\n",
      "    -0.10323096  0.16205089  0.22044756  0.         -0.08347994  0.\n",
      "     0.20875083  0.          0.          0.16474001  0.         -0.06644712\n",
      "     0.          0.0942094 ]]\n",
      "\n",
      "  [[ 0.08457479 -0.07142154  0.          0.         -0.09221686  0.          0.\n",
      "     0.16205089  0.          0.         -0.08347994  0.          0.20875083\n",
      "     0.          0.07646815  0.16474001 -0.07542904 -0.06644712  0.1259973\n",
      "     0.        ]]]\n",
      "\n",
      "\n",
      " [[[-0.07401815  0.         -0.07270797 -0.13083923  0.1786865  -0.08300267\n",
      "     0.          0.16205089  0.22044756 -0.0710777   0.          0.13038068\n",
      "     0.         -0.09780768  0.07646815  0.16474001 -0.07542904  0.06845199\n",
      "     0.          0.        ]]\n",
      "\n",
      "  [[-0.07401815  0.          0.08913977 -0.13083923  0.1786865  -0.08300267\n",
      "     0.12073883  0.16205089  0.22044756  0.         -0.08347994  0.          0.\n",
      "     0.          0.          0.16474001  0.07848413  0.          0.\n",
      "     0.0942094 ]]\n",
      "\n",
      "  [[ 0.08457479  0.          0.08913977  0.          0.1786865  -0.08300267\n",
      "     0.12073883  0.16205089  0.          0.          0.          0.\n",
      "     0.20875083  0.         -0.07865113  0.          0.          0.          0.\n",
      "     0.0942094 ]]\n",
      "\n",
      "  [[ 0.08457479  0.06960116 -0.07270797  0.          0.1786865  -0.08300267\n",
      "     0.          0.16205089  0.         -0.0710777  -0.08347994 -0.06729857\n",
      "     0.20875083  0.08985013 -0.07865113  0.16474001  0.07848413 -0.06644712\n",
      "    -0.09772231 -0.07388642]]\n",
      "\n",
      "  [[ 0.         -0.07142154  0.08913977  0.          0.          0.0848879\n",
      "     0.12073883  0.16205089  0.22044756  0.08296047  0.          0.\n",
      "     0.20875083  0.08985013  0.          0.16474001 -0.07542904  0.06845199\n",
      "     0.         -0.07388642]]]\n",
      "\n",
      "\n",
      " [[[-0.07401815 -0.07142154 -0.07270797 -0.13083923  0.          0.0848879\n",
      "     0.          0.          0.         -0.0710777  -0.08347994  0.13038068\n",
      "    -0.13337269 -0.09780768  0.07646815  0.          0.         -0.06644712\n",
      "     0.          0.0942094 ]]\n",
      "\n",
      "  [[-0.07401815  0.         -0.07270797  0.          0.1786865   0.\n",
      "     0.12073883  0.16205089  0.22044756  0.          0.          0.13038068\n",
      "     0.          0.          0.07646815  0.         -0.07542904 -0.06644712\n",
      "    -0.09772231  0.0942094 ]]\n",
      "\n",
      "  [[-0.07401815  0.06960116  0.08913977  0.          0.1786865   0.\n",
      "     0.12073883  0.16205089  0.          0.         -0.08347994  0.          0.\n",
      "    -0.09780768  0.07646815  0.16474001  0.07848413 -0.06644712 -0.09772231\n",
      "     0.        ]]\n",
      "\n",
      "  [[ 0.08457479 -0.07142154  0.08913977 -0.13083923  0.1786865   0.          0.\n",
      "     0.          0.         -0.0710777   0.          0.          0.20875083\n",
      "     0.          0.07646815  0.          0.07848413 -0.06644712 -0.09772231\n",
      "     0.        ]]\n",
      "\n",
      "  [[ 0.08457479 -0.07142154 -0.07270797  0.          0.1786865   0.0848879\n",
      "     0.12073883  0.          0.          0.         -0.08347994  0.\n",
      "     0.20875083  0.08985013  0.          0.          0.         -0.06644712\n",
      "     0.         -0.07388642]]]]\n"
     ]
    }
   ],
   "source": [
    "print(weights[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 10s 1ms/step\n",
      "[INFO] accuracy: 93.66%\n"
     ]
    }
   ],
   "source": [
    "(loss, accuracy) = model.evaluate(\n",
    "        testData, testLabels, batch_size=128, verbose=1)\n",
    "print(\"[INFO] accuracy: {:.2f}%\".format(accuracy * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Saving The Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('model/LeNet_ternary.model', overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Test MNIST Dataset\n",
    "Randomly select one image from test dataset to predict the number."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in np.random.choice(np.arange(0, len(testLabels)), size=(10,)):\n",
    "    # Use the model to classify the digit\n",
    "    probs = model.predict(testData[np.newaxis, i])\n",
    "    prediction = probs.argmax(axis=1)\n",
    "\n",
    "    # Convert the digit data to a color image\n",
    "    image = (testData[i] * 255).astype(\"uint8\")\n",
    "    image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)\n",
    "\n",
    "    # The images are in 28x28 size. Much too small to see properly\n",
    "    # So, we resize them to 280x280 for viewing\n",
    "    image = cv2.resize(image, (280, 280), interpolation=cv2.INTER_LINEAR)\n",
    "\n",
    "    # Add the predicted value on to the image\n",
    "    cv2.putText(image, str(prediction[0]), (20, 40),\n",
    "                cv2.FONT_HERSHEY_DUPLEX, 1.5, (0, 255, 0), 1)\n",
    "\n",
    "    # Show the image and prediction\n",
    "    print(\"[INFO] Predicted: {}, Actual: {}\".format(\n",
    "        prediction[0], np.argmax(testLabels[i])))\n",
    "    cv2.imshow(\"Digit\", image)\n",
    "    cv2.waitKey(0)\n",
    "\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (tf)",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}