Day097_Keras_CNN_vs_DNN-HW.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/intel/intelpython3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128 # batch 的大小，如果出現 OOM error，請降低這個值\n",
    "num_classes = 10 # 類別的數量，Cifar 10 共有 10 個類別\n",
    "epochs = 10 # 訓練的 epochs 數量\n",
    "\n",
    "# 讀取資料並檢視\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# 對 label 進行 one-hot encoding (y_trian 原本是純數字)\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 首先我們使用一般的 DNN (MLP) 來訓練\n",
    "由於 DNN 只能輸入一維的資料，我們要先將影像進行攤平，若 (50000, 32, 32, 3) 的影像，攤平後會變成 (50000, 32*32*3) = (50000, 3072)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "# 將資料攤平成一維資料\n",
    "x_train = x_train.reshape(50000, 3072) \n",
    "x_test = x_test.reshape(10000, 3072)\n",
    "\n",
    "# 將資料變為 float32 並標準化\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 1,841,162\n",
      "Trainable params: 1,841,162\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 5s 93us/step - loss: 2.1639 - acc: 0.2466 - val_loss: 1.8197 - val_acc: 0.3513\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.8590 - acc: 0.3265 - val_loss: 1.8009 - val_acc: 0.3321\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.7833 - acc: 0.3575 - val_loss: 1.6762 - val_acc: 0.4060\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.7327 - acc: 0.3780 - val_loss: 1.7022 - val_acc: 0.3977\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.6959 - acc: 0.3896 - val_loss: 1.6314 - val_acc: 0.4194\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 2s 44us/step - loss: 1.6697 - acc: 0.3993 - val_loss: 1.6231 - val_acc: 0.4264\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 2s 44us/step - loss: 1.6483 - acc: 0.4088 - val_loss: 1.6370 - val_acc: 0.4051\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.6299 - acc: 0.4182 - val_loss: 1.5738 - val_acc: 0.4466\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.6143 - acc: 0.4241 - val_loss: 1.5522 - val_acc: 0.4561\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 2s 42us/step - loss: 1.6003 - acc: 0.4278 - val_loss: 1.5236 - val_acc: 0.4728\n",
      "Test loss: 1.5236255004882813\n",
      "Test accuracy: 0.4728\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_shape=(3072,)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 接下來我們使用 CNN 來訓練神經網路\n",
    "CNN 的原理非常適合處理影像類的資料，就讓我們來看看，同樣的訓練條件，CNN 是否顯著優於 DNN 呢?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 15, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 13, 13, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 1,250,858\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 6s 127us/step - loss: 1.7666 - acc: 0.3592 - val_loss: 1.3987 - val_acc: 0.5000\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 4s 80us/step - loss: 1.3330 - acc: 0.5265 - val_loss: 1.1460 - val_acc: 0.5954\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 4s 78us/step - loss: 1.1267 - acc: 0.6023 - val_loss: 1.0026 - val_acc: 0.6478\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 4s 79us/step - loss: 0.9935 - acc: 0.6526 - val_loss: 0.9748 - val_acc: 0.6595\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 4s 79us/step - loss: 0.8920 - acc: 0.6898 - val_loss: 0.8119 - val_acc: 0.7186\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 4s 80us/step - loss: 0.8237 - acc: 0.7140 - val_loss: 0.7926 - val_acc: 0.7251\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 4s 79us/step - loss: 0.7667 - acc: 0.7316 - val_loss: 0.7685 - val_acc: 0.7384\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 4s 79us/step - loss: 0.7274 - acc: 0.7489 - val_loss: 0.9322 - val_acc: 0.6901\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 4s 79us/step - loss: 0.6910 - acc: 0.7588 - val_loss: 0.7872 - val_acc: 0.7317\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 4s 79us/step - loss: 0.6736 - acc: 0.7655 - val_loss: 0.7365 - val_acc: 0.7479\n",
      "Test loss: 0.7365292016983033\n",
      "Test accuracy: 0.7479\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3, 3), padding='same',\n",
    "                 input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(64, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 同樣運算 10 個 epochs，但 CNN 在 test data 的準確率顯著優於 DNN!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 作業\n",
    "1. 請試著調整各個超參數，並說明那些超參數對於結果有明顯的影響?\n",
    "2. CNN 與 DNN 哪個模型的參數數量比較多? 造成參數的數量不同的原因在哪?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}