Day097_Keras_CNN_vs_DNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "x_train shape: (50000, 32, 32, 3)\n50000 train samples\n10000 test samples\n"
    }
   ],
   "source": [
    "batch_size = 128 # batch 的大小，如果出現 OOM error，請降低這個值\n",
    "num_classes = 10 # 類別的數量，Cifar 10 共有 10 個類別\n",
    "epochs = 10 # 訓練的 epochs 數量\n",
    "\n",
    "# 讀取資料並檢視\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# 對 label 進行 one-hot encoding (y_trian 原本是純數字)\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 首先我們使用一般的 DNN (MLP) 來訓練\n",
    "由於 DNN 只能輸入一維的資料，我們要先將影像進行攤平，若 (50000, 32, 32, 3) 的影像，攤平後會變成 (50000, 32*32*3) = (50000, 3072)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "50000 train samples\n10000 test samples\n"
    }
   ],
   "source": [
    "# 將資料攤平成一維資料\n",
    "x_train = x_train.reshape(50000, 3072) \n",
    "x_test = x_test.reshape(10000, 3072)\n",
    "\n",
    "# 將資料變為 float32 並標準化\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Model: \"sequential\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\ndense (Dense)                (None, 512)               1573376   \n_________________________________________________________________\ndropout (Dropout)            (None, 512)               0         \n_________________________________________________________________\ndense_1 (Dense)              (None, 512)               262656    \n_________________________________________________________________\ndropout_1 (Dropout)          (None, 512)               0         \n_________________________________________________________________\ndense_2 (Dense)              (None, 10)                5130      \n=================================================================\nTotal params: 1,841,162\nTrainable params: 1,841,162\nNon-trainable params: 0\n_________________________________________________________________\nEpoch 1/10\n391/391 [==============================] - 9s 22ms/step - loss: 2.1412 - accuracy: 0.2471 - val_loss: 1.9287 - val_accuracy: 0.2921\nEpoch 2/10\n391/391 [==============================] - 8s 22ms/step - loss: 1.8591 - accuracy: 0.3252 - val_loss: 1.7441 - val_accuracy: 0.3871\nEpoch 3/10\n391/391 [==============================] - 8s 22ms/step - loss: 1.7812 - accuracy: 0.3602 - val_loss: 1.6957 - val_accuracy: 0.3901\nEpoch 4/10\n391/391 [==============================] - 9s 22ms/step - loss: 1.7330 - accuracy: 0.3796 - val_loss: 1.6735 - val_accuracy: 0.3971\nEpoch 5/10\n391/391 [==============================] - 9s 23ms/step - loss: 1.6950 - accuracy: 0.3904 - val_loss: 1.6116 - val_accuracy: 0.4251\nEpoch 6/10\n391/391 [==============================] - 9s 22ms/step - loss: 1.6686 - accuracy: 0.4003 - val_loss: 1.6308 - val_accuracy: 0.4149\nEpoch 7/10\n391/391 [==============================] - 9s 22ms/step - loss: 1.6451 - accuracy: 0.4114 - val_loss: 1.5863 - val_accuracy: 0.4406\nEpoch 8/10\n391/391 [==============================] - 9s 22ms/step - loss: 1.6298 - accuracy: 0.4162 - val_loss: 1.6337 - val_accuracy: 0.4296\nEpoch 9/10\n391/391 [==============================] - 9s 22ms/step - loss: 1.6130 - accuracy: 0.4224 - val_loss: 1.5881 - val_accuracy: 0.4432\nEpoch 10/10\n391/391 [==============================] - 9s 22ms/step - loss: 1.6016 - accuracy: 0.4291 - val_loss: 1.5512 - val_accuracy: 0.4505\nTest loss: 1.5511693954467773\nTest accuracy: 0.4505000114440918\n"
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_shape=(3072,)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 接下來我們使用 CNN 來訓練神經網路\n",
    "CNN 的原理非常適合處理影像類的資料，就讓我們來看看，同樣的訓練條件，CNN 是否顯著優於 DNN 呢?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "x_train shape: (50000, 32, 32, 3)\n50000 train samples\n10000 test samples\n"
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "___________________________________\nactivation_6 (Activation)    (None, 32, 32, 32)        0         \n_________________________________________________________________\nconv2d_5 (Conv2D)            (None, 30, 30, 32)        9248      \n_________________________________________________________________\nactivation_7 (Activation)    (None, 30, 30, 32)        0         \n_________________________________________________________________\nmax_pooling2d_2 (MaxPooling2 (None, 15, 15, 32)        0         \n_________________________________________________________________\ndropout_5 (Dropout)          (None, 15, 15, 32)        0         \n_________________________________________________________________\nconv2d_6 (Conv2D)            (None, 15, 15, 64)        18496     \n_________________________________________________________________\nactivation_8 (Activation)    (None, 15, 15, 64)        0         \n_________________________________________________________________\nconv2d_7 (Conv2D)            (None, 13, 13, 64)        36928     \n_________________________________________________________________\nactivation_9 (Activation)    (None, 13, 13, 64)        0         \n_________________________________________________________________\nmax_pooling2d_3 (MaxPooling2 (None, 6, 6, 64)          0         \n_________________________________________________________________\ndropout_6 (Dropout)          (None, 6, 6, 64)          0         \n_________________________________________________________________\nflatten_1 (Flatten)          (None, 2304)              0         \n_________________________________________________________________\ndense_5 (Dense)              (None, 512)               1180160   \n_________________________________________________________________\nactivation_10 (Activation)   (None, 512)               0         \n_________________________________________________________________\ndropout_7 (Dropout)          (None, 512)               0         \n_________________________________________________________________\ndense_6 (Dense)              (None, 10)                5130      \n_________________________________________________________________\nactivation_11 (Activation)   (None, 10)                0         \n=================================================================\nTotal params: 1,250,858\nTrainable params: 1,250,858\nNon-trainable params: 0\n_________________________________________________________________\nBatch size: 32\noptimizer: Adam\nEpoch 1/10\n1563/1563 [==============================] - 104s 67ms/step - loss: 1.5343 - accuracy: 0.4415 - val_loss: 1.1546 - val_accuracy: 0.5875\nEpoch 2/10\n1563/1563 [==============================] - 104s 67ms/step - loss: 1.1431 - accuracy: 0.5943 - val_loss: 0.9457 - val_accuracy: 0.6639\nEpoch 3/10\n1563/1563 [==============================] - 102s 65ms/step - loss: 0.9847 - accuracy: 0.6524 - val_loss: 0.8608 - val_accuracy: 0.6950\nEpoch 4/10\n1563/1563 [==============================] - 102s 66ms/step - loss: 0.8932 - accuracy: 0.6867 - val_loss: 0.8762 - val_accuracy: 0.6922\nEpoch 5/10\n1563/1563 [==============================] - 104s 67ms/step - loss: 0.8312 - accuracy: 0.7083 - val_loss: 0.7256 - val_accuracy: 0.7476\nEpoch 6/10\n1563/1563 [==============================] - 104s 67ms/step - loss: 0.7799 - accuracy: 0.7275 - val_loss: 0.7305 - val_accuracy: 0.7421\nEpoch 7/10\n1563/1563 [==============================] - 104s 67ms/step - loss: 0.7417 - accuracy: 0.7398 - val_loss: 0.6981 - val_accuracy: 0.7557\nEpoch 8/10\n1563/1563 [==============================] - 104s 66ms/step - loss: 0.7115 - accuracy: 0.7497 - val_loss: 0.7057 - val_accuracy: 0.7496\nEpoch 9/10\n1563/1563 [==============================] - 105s 67ms/step - loss: 0.6877 - accuracy: 0.7593 - val_loss: 0.7097 - val_accuracy: 0.7552\nEpoch 10/10\n1563/1563 [==============================] - 108s 69ms/step - loss: 0.6671 - accuracy: 0.7654 - val_loss: 0.6627 - val_accuracy: 0.7723\nTest loss: 0.6627358794212341\nTest accuracy: 0.7723000049591064\nModel: \"sequential_3\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nconv2d_8 (Conv2D)            (None, 32, 32, 64)        1792      \n_________________________________________________________________\nactivation_12 (Activation)   (None, 32, 32, 64)        0         \n_________________________________________________________________\nconv2d_9 (Conv2D)            (None, 30, 30, 64)        36928     \n_________________________________________________________________\nactivation_13 (Activation)   (None, 30, 30, 64)        0         \n_________________________________________________________________\nmax_pooling2d_4 (MaxPooling2 (None, 15, 15, 64)        0         \n_________________________________________________________________\ndropout_8 (Dropout)          (None, 15, 15, 64)        0         \n_________________________________________________________________\nconv2d_10 (Conv2D)           (None, 15, 15, 128)       73856     \n_________________________________________________________________\nactivation_14 (Activation)   (None, 15, 15, 128)       0         \n_________________________________________________________________\nconv2d_11 (Conv2D)           (None, 13, 13, 128)       147584    \n_________________________________________________________________\nactivation_15 (Activation)   (None, 13, 13, 128)       0         \n_________________________________________________________________\nmax_pooling2d_5 (MaxPooling2 (None, 6, 6, 128)         0         \n_________________________________________________________________\ndropout_9 (Dropout)          (None, 6, 6, 128)         0         \n_________________________________________________________________\nflatten_2 (Flatten)          (None, 4608)              0         \n_________________________________________________________________\ndense_7 (Dense)              (None, 512)               2359808   \n_________________________________________________________________\nactivation_16 (Activation)   (None, 512)               0         \n_________________________________________________________________\ndropout_10 (Dropout)         (None, 512)               0         \n_________________________________________________________________\ndense_8 (Dense)              (None, 10)                5130      \n_________________________________________________________________\nactivation_17 (Activation)   (None, 10)                0         \n=================================================================\nTotal params: 2,625,098\nTrainable params: 2,625,098\nNon-trainable params: 0\n_________________________________________________________________\nBatch size: 64\noptimizer: RMSprop\nEpoch 1/10\n782/782 [==============================] - 209s 267ms/step - loss: 1.5991 - accuracy: 0.4260 - val_loss: 1.1593 - val_accuracy: 0.5968\nEpoch 2/10\n782/782 [==============================] - 209s 268ms/step - loss: 1.0894 - accuracy: 0.6203 - val_loss: 1.0371 - val_accuracy: 0.6390\nEpoch 3/10\n782/782 [==============================] - 209s 267ms/step - loss: 0.9042 - accuracy: 0.6895 - val_loss: 0.8536 - val_accuracy: 0.7097\nEpoch 4/10\n782/782 [==============================] - 210s 269ms/step - loss: 0.8176 - accuracy: 0.7190 - val_loss: 0.8479 - val_accuracy: 0.7143\nEpoch 5/10\n782/782 [==============================] - 209s 267ms/step - loss: 0.7769 - accuracy: 0.7396 - val_loss: 0.9474 - val_accuracy: 0.7001\nEpoch 6/10\n782/782 [==============================] - 209s 267ms/step - loss: 0.7691 - accuracy: 0.7469 - val_loss: 0.7650 - val_accuracy: 0.7552\nEpoch 7/10\n782/782 [==============================] - 209s 267ms/step - loss: 0.7662 - accuracy: 0.7483 - val_loss: 1.0269 - val_accuracy: 0.7087\nEpoch 8/10\n782/782 [==============================] - 210s 268ms/step - loss: 0.7772 - accuracy: 0.7496 - val_loss: 0.8356 - val_accuracy: 0.7600\nEpoch 9/10\n782/782 [==============================] - 208s 265ms/step - loss: 0.7803 - accuracy: 0.7479 - val_loss: 0.7782 - val_accuracy: 0.7662\nEpoch 10/10\n782/782 [==============================] - 209s 267ms/step - loss: 0.7937 - accuracy: 0.7478 - val_loss: 0.9305 - val_accuracy: 0.6975\nTest loss: 0.9304872155189514\nTest accuracy: 0.6974999904632568\nModel: \"sequential_4\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nconv2d_12 (Conv2D)           (None, 32, 32, 64)        1792      \n_________________________________________________________________\nactivation_18 (Activation)   (None, 32, 32, 64)        0         \n_________________________________________________________________\nconv2d_13 (Conv2D)           (None, 30, 30, 64)        36928     \n_________________________________________________________________\nactivation_19 (Activation)   (None, 30, 30, 64)        0         \n_________________________________________________________________\nmax_pooling2d_6 (MaxPooling2 (None, 15, 15, 64)        0         \n_________________________________________________________________\ndropout_11 (Dropout)         (None, 15, 15, 64)        0         \n_________________________________________________________________\nconv2d_14 (Conv2D)           (None, 15, 15, 128)       73856     \n_________________________________________________________________\nactivation_20 (Activation)   (None, 15, 15, 128)       0         \n_________________________________________________________________\nconv2d_15 (Conv2D)           (None, 13, 13, 128)       147584    \n_________________________________________________________________\nactivation_21 (Activation)   (None, 13, 13, 128)       0         \n_________________________________________________________________\nmax_pooling2d_7 (MaxPooling2 (None, 6, 6, 128)         0         \n_________________________________________________________________\ndropout_12 (Dropout)         (None, 6, 6, 128)         0         \n_________________________________________________________________\nflatten_3 (Flatten)          (None, 4608)              0         \n_________________________________________________________________\ndense_9 (Dense)              (None, 512)               2359808   \n_________________________________________________________________\nactivation_22 (Activation)   (None, 512)               0         \n_________________________________________________________________\ndropout_13 (Dropout)         (None, 512)               0         \n_________________________________________________________________\ndense_10 (Dense)             (None, 10)                5130      \n_________________________________________________________________\nactivation_23 (Activation)   (None, 10)                0         \n=================================================================\nTotal params: 2,625,098\nTrainable params: 2,625,098\nNon-trainable params: 0\n_________________________________________________________________\nBatch size: 64\noptimizer: Adam\nEpoch 1/10\n782/782 [==============================] - 204s 261ms/step - loss: 1.5278 - accuracy: 0.4387 - val_loss: 1.1745 - val_accuracy: 0.5723\nEpoch 2/10\n782/782 [==============================] - 205s 263ms/step - loss: 1.0912 - accuracy: 0.6136 - val_loss: 0.9700 - val_accuracy: 0.6568\nEpoch 3/10\n782/782 [==============================] - 206s 264ms/step - loss: 0.9122 - accuracy: 0.6805 - val_loss: 0.7946 - val_accuracy: 0.7205\nEpoch 4/10\n782/782 [==============================] - 206s 264ms/step - loss: 0.8030 - accuracy: 0.7192 - val_loss: 0.8106 - val_accuracy: 0.7243\nEpoch 5/10\n782/782 [==============================] - 205s 263ms/step - loss: 0.7414 - accuracy: 0.7407 - val_loss: 0.7125 - val_accuracy: 0.7535\nEpoch 6/10\n782/782 [==============================] - 206s 263ms/step - loss: 0.6906 - accuracy: 0.7605 - val_loss: 0.6865 - val_accuracy: 0.7621\nEpoch 7/10\n782/782 [==============================] - 206s 263ms/step - loss: 0.6458 - accuracy: 0.7741 - val_loss: 0.6724 - val_accuracy: 0.7686\nEpoch 8/10\n782/782 [==============================] - 208s 266ms/step - loss: 0.6037 - accuracy: 0.7884 - val_loss: 0.6831 - val_accuracy: 0.7723\nEpoch 9/10\n782/782 [==============================] - 206s 264ms/step - loss: 0.5744 - accuracy: 0.7989 - val_loss: 0.6518 - val_accuracy: 0.7791\nEpoch 10/10\n782/782 [==============================] - 214s 273ms/step - loss: 0.5392 - accuracy: 0.8108 - val_loss: 0.6527 - val_accuracy: 0.7842\nTest loss: 0.6526586413383484\nTest accuracy: 0.7842000126838684\nModel: \"sequential_5\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nconv2d_16 (Conv2D)           (None, 32, 32, 128)       3584      \n_________________________________________________________________\nactivation_24 (Activation)   (None, 32, 32, 128)       0         \n_________________________________________________________________\nconv2d_17 (Conv2D)           (None, 30, 30, 128)       147584    \n_________________________________________________________________\nactivation_25 (Activation)   (None, 30, 30, 128)       0         \n_________________________________________________________________\nmax_pooling2d_8 (MaxPooling2 (None, 15, 15, 128)       0         \n_________________________________________________________________\ndropout_14 (Dropout)         (None, 15, 15, 128)       0         \n_________________________________________________________________\nconv2d_18 (Conv2D)           (None, 15, 15, 256)       295168    \n_________________________________________________________________\nactivation_26 (Activation)   (None, 15, 15, 256)       0         \n_________________________________________________________________\nconv2d_19 (Conv2D)           (None, 13, 13, 256)       590080    \n_________________________________________________________________\nactivation_27 (Activation)   (None, 13, 13, 256)       0         \n_________________________________________________________________\nmax_pooling2d_9 (MaxPooling2 (None, 6, 6, 256)         0         \n_________________________________________________________________\ndropout_15 (Dropout)         (None, 6, 6, 256)         0         \n_________________________________________________________________\nflatten_4 (Flatten)          (None, 9216)              0         \n_________________________________________________________________\ndense_11 (Dense)             (None, 512)               4719104   \n_________________________________________________________________\nactivation_28 (Activation)   (None, 512)               0         \n_________________________________________________________________\ndropout_16 (Dropout)         (None, 512)               0         \n_________________________________________________________________\ndense_12 (Dense)             (None, 10)                5130      \n_________________________________________________________________\nactivation_29 (Activation)   (None, 10)                0         \n=================================================================\nTotal params: 5,760,650\nTrainable params: 5,760,650\nNon-trainable params: 0\n_________________________________________________________________\nBatch size: 128\noptimizer: RMSprop\nEpoch 1/10\n391/391 [==============================] - 618s 2s/step - loss: 1.8439 - accuracy: 0.3397 - val_loss: 1.8234 - val_accuracy: 0.4092\nEpoch 2/10\n391/391 [==============================] - 619s 2s/step - loss: 1.2320 - accuracy: 0.5680 - val_loss: 0.9458 - val_accuracy: 0.6752\nEpoch 3/10\n391/391 [==============================] - 629s 2s/step - loss: 0.9537 - accuracy: 0.6712 - val_loss: 0.9401 - val_accuracy: 0.6727\nEpoch 4/10\n391/391 [==============================] - 618s 2s/step - loss: 0.8029 - accuracy: 0.7246 - val_loss: 0.8041 - val_accuracy: 0.7329\nEpoch 5/10\n391/391 [==============================] - 617s 2s/step - loss: 0.7119 - accuracy: 0.7560 - val_loss: 0.7756 - val_accuracy: 0.7486\nEpoch 6/10\n391/391 [==============================] - 618s 2s/step - loss: 0.6533 - accuracy: 0.7798 - val_loss: 0.7286 - val_accuracy: 0.7764\nEpoch 7/10\n391/391 [==============================] - 619s 2s/step - loss: 0.6188 - accuracy: 0.7913 - val_loss: 0.7245 - val_accuracy: 0.7648\nEpoch 8/10\n391/391 [==============================] - 618s 2s/step - loss: 0.5843 - accuracy: 0.8061 - val_loss: 0.6699 - val_accuracy: 0.7917\nEpoch 9/10\n391/391 [==============================] - 619s 2s/step - loss: 0.5624 - accuracy: 0.8152 - val_loss: 0.6484 - val_accuracy: 0.7914\nEpoch 10/10\n391/391 [==============================] - 619s 2s/step - loss: 0.5548 - accuracy: 0.8210 - val_loss: 1.0611 - val_accuracy: 0.7556\nTest loss: 1.0611274242401123\nTest accuracy: 0.7555999755859375\nModel: \"sequential_6\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nconv2d_20 (Conv2D)           (None, 32, 32, 128)       3584      \n_________________________________________________________________\nactivation_30 (Activation)   (None, 32, 32, 128)       0         \n_________________________________________________________________\nconv2d_21 (Conv2D)           (None, 30, 30, 128)       147584    \n_________________________________________________________________\nactivation_31 (Activation)   (None, 30, 30, 128)       0         \n_________________________________________________________________\nmax_pooling2d_10 (MaxPooling (None, 15, 15, 128)       0         \n_________________________________________________________________\ndropout_17 (Dropout)         (None, 15, 15, 128)       0         \n_________________________________________________________________\nconv2d_22 (Conv2D)           (None, 15, 15, 256)       295168    \n_________________________________________________________________\nactivation_32 (Activation)   (None, 15, 15, 256)       0         \n_________________________________________________________________\nconv2d_23 (Conv2D)           (None, 13, 13, 256)       590080    \n_________________________________________________________________\nactivation_33 (Activation)   (None, 13, 13, 256)       0         \n_________________________________________________________________\nmax_pooling2d_11 (MaxPooling (None, 6, 6, 256)         0         \n_________________________________________________________________\ndropout_18 (Dropout)         (None, 6, 6, 256)         0         \n_________________________________________________________________\nflatten_5 (Flatten)          (None, 9216)              0         \n_________________________________________________________________\ndense_13 (Dense)             (None, 512)               4719104   \n_________________________________________________________________\nactivation_34 (Activation)   (None, 512)               0         \n_________________________________________________________________\ndropout_19 (Dropout)         (None, 512)               0         \n_________________________________________________________________\ndense_14 (Dense)             (None, 10)                5130      \n_________________________________________________________________\nactivation_35 (Activation)   (None, 10)                0         \n=================================================================\nTotal params: 5,760,650\nTrainable params: 5,760,650\nNon-trainable params: 0\n_________________________________________________________________\nBatch size: 128\noptimizer: Adam\nEpoch 1/10\n391/391 [==============================] - 616s 2s/step - loss: 1.6088 - accuracy: 0.4103 - val_loss: 1.2130 - val_accuracy: 0.5696\nEpoch 2/10\n391/391 [==============================] - 609s 2s/step - loss: 1.1563 - accuracy: 0.5902 - val_loss: 0.9218 - val_accuracy: 0.6836\nEpoch 3/10\n266/391 [===================>..........] - ETA: 3:08 - loss: 0.9559 - accuracy: 0.6644"
    },
    {
     "output_type": "error",
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-54e9b0b7d185>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     34\u001b[0m                             \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m                             \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m                             validation_data=(x_test, y_test))\n\u001b[0m\u001b[0;32m     37\u001b[0m         \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Test loss:'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m     64\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     67\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m     \u001b[1;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m    846\u001b[0m                 batch_size=batch_size):\n\u001b[0;32m    847\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 848\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    849\u001b[0m               \u001b[1;31m# Catch OutOfRangeError for Datasets of unknown size.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    850\u001b[0m               \u001b[1;31m# This blocks until the batch has finished executing.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    578\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    579\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 580\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    581\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    582\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    609\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    610\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 611\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    612\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    613\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2418\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2419\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2420\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2421\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2422\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs)\u001b[0m\n\u001b[0;32m   1663\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[0;32m   1664\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[1;32m-> 1665\u001b[1;33m         self.captured_inputs)\n\u001b[0m\u001b[0;32m   1666\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1667\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1744\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1746\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 598\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    599\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python37\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[1;32m---> 60\u001b[1;33m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for j in [32,64,128]:\n",
    "    for i in ['RMSprop','Adam']:\n",
    "        model = Sequential()\n",
    "        model.add(Conv2D(j, (3, 3), padding='same',\n",
    "                        input_shape=x_train.shape[1:]))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Conv2D(j, (3, 3)))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Dropout(0.25))\n",
    "\n",
    "        model.add(Conv2D(2*j, (3, 3), padding='same'))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Conv2D(2*j, (3, 3)))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        model.add(Dropout(0.25))\n",
    "\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(512))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Dropout(0.5))\n",
    "        model.add(Dense(num_classes))\n",
    "        model.add(Activation('softmax'))\n",
    "        model.summary()\n",
    "\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                    optimizer=i,\n",
    "                    metrics=['accuracy'])\n",
    "        print('Batch size:',j)\n",
    "        print('optimizer:',i)\n",
    "        history = model.fit(x_train, y_train,\n",
    "                            batch_size=j,\n",
    "                            epochs=epochs,\n",
    "                            verbose=1,\n",
    "                            validation_data=(x_test, y_test))\n",
    "        score = model.evaluate(x_test, y_test, verbose=0)\n",
    "        print('Test loss:', score[0])\n",
    "        print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 同樣運算 10 個 epochs，但 CNN 在 test data 的準確率顯著優於 DNN!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 作業\n",
    "1. 請試著調整各個超參數，並說明那些超參數對於結果有明顯的影響?\n",
    "2. CNN 與 DNN 哪個模型的參數數量比較多? 造成參數的數量不同的原因在哪?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 增加Batch size有助於準確度的上升\n",
    "2. CNN的參數比較多，參數都集中在Dense層，DNN的Conv2D層的參數較少。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}