Transfer Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: KERAS_BACKEND=tensorflow\n"
     ]
    }
   ],
   "source": [
    "%env KERAS_BACKEND=tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras.optimizers import SGD\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**1.輸入出資料處理**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#輸入格式改為向量\n",
    "x_train = x_train.reshape(60000, 784)\n",
    "x_test = x_test.reshape(10000, 784)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "為了後面需要，把訓練/測試資料中的數字 0, 1 資料(資料有0-9)，複製一份出來"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_01 = x_train[y_train <= 1]\n",
    "x_test_01 = x_test[y_test <= 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x26b308c3988>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOP0lEQVR4nO3df6xU9ZnH8c+jFk3kqrhclAAuUDVqGgUcwETTuGm2URPFkrDpjTZsgqF/aKSGxFWMqRKNstlSf2RTpSspu3atTVqFEIQSaEIalHAhVEHc9Ueg3ELgIhokJKLw7B/3sLnFO98Z5pwzZ+B5v5KbmTnPnDkPw/3cMzPfc+Zr7i4AZ79zqm4AQHsQdiAIwg4EQdiBIAg7EMR57dzYyJEjffz48e3cJBDKrl27dPDgQRuqlivsZnabpOclnSvpP9z92dT9x48fr97e3jybBJBQq9Xq1lp+GW9m50r6d0m3S7pOUo+ZXdfq4wEoV5737NMkfeTun7j7MUm/kTSjmLYAFC1P2MdI2jPodl+27G+Y2Vwz6zWz3v7+/hybA5BHnrAP9SHAN469dfcl7l5z91p3d3eOzQHII0/Y+ySNG3R7rKS9+doBUJY8Yd8s6Sozm2BmwyT9UNKKYtoCULSWh97c/Wsze0DSGg0MvS119x2FdQagULnG2d19laRVBfUCoEQcLgsEQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEG2dshmd58SJE8n6li1bkvWFCxcm6wsWLKhbO++8cn/9rr/++rq1888/v9RtdyL27EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBOPswR0/fjxZv+mmm3I9/qpV1U3y++ijj9atPfXUU23spDPkCruZ7ZL0haTjkr5291oRTQEoXhF79n9w94MFPA6AEvGeHQgib9hd0h/MbIuZzR3qDmY218x6zay3v78/5+YAtCpv2G929ymSbpd0v5l999Q7uPsSd6+5e627uzvn5gC0KlfY3X1vdnlA0huSphXRFIDitRx2M7vQzLpOXpf0fUnbi2oMQLHyfBp/maQ3zOzk4/y3u68upKtgjh07lqzv3r07WU+dFz5hwoSWejobLFq0qG7t5ZdfTq67adOmZH3ixIkt9VSllsPu7p9IuqHAXgCUiKE3IAjCDgRB2IEgCDsQBGEHguAU1w6wePHiZP2xxx5L1i+66KK6tdWr06OhU6dOTdbnzZuXrD///PPJepVSX5N96NChltc9U7FnB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgGGcvQKNTVBuNoz/55JO5tn/48OG6tUbj7NOnT0/WZ8+enay//fbbyfrmzZvr1tw9uW6Venp6kvXUv6tTsWcHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAYZy/AM888k6wvXLiwTZ18U6Nx8M8++yxZv+GG9BcIN3r8+fPn160999xzyXUbTav8zjvvJOsrV65M1lM++OCDZH3r1q3J+pQpU1redlnYswNBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIyzZ7766qtk/YUXXqhbe/rpp4tupzBr165N1vfu3ZusjxgxItf2U89No++kHzNmTLJ+9OjRZH3mzJl1a+vXr8/12K+//nqyfkaOs5vZUjM7YGbbBy271MzWmtmH2WW+3wgApWvmZfyvJN12yrJHJK1z96skrctuA+hgDcPu7hsknTpXzgxJy7LryyTdXXBfAArW6gd0l7n7PknKLkfVu6OZzTWzXjPr7e/vb3FzAPIq/dN4d1/i7jV3r3V3d5e9OQB1tBr2/WY2WpKyywPFtQSgDK2GfYWkk98xPFvS8mLaAVCWhuPsZvaapFsljTSzPkk/lfSspN+a2RxJf5E0q8wm22Hbtm3J+sMPP9ymToo1bdq0ZL3st1YXXHBB3doVV1yR67G7urqS9dRYeN5/944dO5L1zz//PFm/5JJLcm2/FQ3D7u71vi3/ewX3AqBEHC4LBEHYgSAIOxAEYQeCIOxAEJzimml0imsnGzWq7tHKWr48fQhEat0zXZn/p2+99Vay/vHHHyfrN954Y5HtNIU9OxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EwTh75pZbbknWzaxNnZy+WbPqn2F8No+jN3LXXXdV3UJHYc8OBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0Ewzn4GmDNnTrK+aNGiNnWCMxl7diAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgnH2DjB16tRk/aWXXkrWzznnzPybvX///mS90XTTBw8eTNa//PLL0+6pWQ899FCyPnny5NK23aqGvyVmttTMDpjZ9kHLnjCzv5rZtuznjnLbBJBXM7uEX0m6bYjlP3f3SdnPqmLbAlC0hmF39w2SDrWhFwAlyvNm7wEzezd7mT+i3p3MbK6Z9ZpZb39/f47NAcij1bD/QtK3JU2StE/Sz+rd0d2XuHvN3Wvd3d0tbg5AXi2F3d33u/txdz8h6ZeS0h+bAqhcS2E3s9GDbv5A0vZ69wXQGRqOs5vZa5JulTTSzPok/VTSrWY2SZJL2iXpxyX2eNZr9J30nTyOvmbNmmR948aNLa/b19fXUk9F6OrqStbvu+++ZL0T/88aht3de4ZY/EoJvQAoUef9+QFQCsIOBEHYgSAIOxAEYQeC4BTXjLsn62VO2XzkyJFkfffu3aVt+/HHH0/WN2zYkKx/+umnyfrRo0dPu6dOsH79+mT9mmuuaVMnxWHPDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM6emTdvXrL+4osvlrbt999/P1mfOHFiaduO6s4770zWr7zyyjZ10j7s2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMbZM/fcc0+yXuY4O1ozatSoZH369Ol1a6+++mpy3eHDh7fUUydjzw4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQTDOnhk3blyyPnXq1Lq1zZs3F90OJI0dOzZZX716dbJ+7bXXFtnOGa/hnt3MxpnZH81sp5ntMLN52fJLzWytmX2YXY4ov10ArWrmZfzXkua7+7WSbpJ0v5ldJ+kRSevc/SpJ67LbADpUw7C7+z5335pd/0LSTkljJM2QtCy72zJJd5fVJID8TusDOjMbL2mypE2SLnP3fdLAHwRJQx6obGZzzazXzHr7+/vzdQugZU2H3cyGS/qdpJ+4++Fm13P3Je5ec/dad3d3Kz0CKEBTYTezb2kg6L92999ni/eb2eisPlrSgXJaBFCEhkNvNjBX8SuSdrr74kGlFZJmS3o2u1xeSodtcvnllyfrK1eurFur1WrJdffs2dNST2eDYcOG1a1dfPHFyXXXrFmTrJ+J0yZXqZlx9psl/UjSe2a2LVu2QAMh/62ZzZH0F0mzymkRQBEaht3d/yTJ6pS/V2w7AMrC4bJAEIQdCIKwA0EQdiAIwg4EwSmuTRo5cmTdWqNTXBtND3wmnyI7c+bMlus9PT1Ft4ME9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7AVo9A08b775ZrK+cePGZH3WrNbPHr733nuT9QcffLDlx5akq6++Olnv6urK9fgoDnt2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCcfY2aPSd9I3OCT9+/HiR7SAo9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EETDsJvZODP7o5ntNLMdZjYvW/6Emf3VzLZlP3eU3y6AVjVzUM3Xkua7+1Yz65K0xczWZrWfu/u/ldcegKI0Mz/7Pkn7sutfmNlOSWPKbgxAsU7rPbuZjZc0WdKmbNEDZvaumS01sxF11plrZr1m1tvf35+rWQCtazrsZjZc0u8k/cTdD0v6haRvS5qkgT3/z4Zaz92XuHvN3WuNvqsNQHmaCruZfUsDQf+1u/9ektx9v7sfd/cTkn4paVp5bQLIq5lP403SK5J2uvviQctHD7rbDyRtL749AEVp5tP4myX9SNJ7ZrYtW7ZAUo+ZTZLkknZJ+nEpHQIoRDOfxv9Jkg1RWlV8OwDKwhF0QBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIMzd27cxs35JuwctGinpYNsaOD2d2lun9iXRW6uK7O3v3X3I739ra9i/sXGzXnevVdZAQqf21ql9SfTWqnb1xst4IAjCDgRRddiXVLz9lE7trVP7kuitVW3prdL37ADap+o9O4A2IexAEJWE3cxuM7P/MbOPzOyRKnqox8x2mdl72TTUvRX3stTMDpjZ9kHLLjWztWb2YXY55Bx7FfXWEdN4J6YZr/S5q3r687a/ZzezcyX9r6R/lNQnabOkHnd/v62N1GFmuyTV3L3yAzDM7LuSjkj6T3f/TrbsXyUdcvdnsz+UI9z9XzqktyckHal6Gu9stqLRg6cZl3S3pH9Whc9doq9/Uhuetyr27NMkfeTun7j7MUm/kTSjgj46nrtvkHTolMUzJC3Lri/TwC9L29XprSO4+z5335pd/0LSyWnGK33uEn21RRVhHyNpz6Dbfeqs+d5d0h/MbIuZza26mSFc5u77pIFfHkmjKu7nVA2n8W6nU6YZ75jnrpXpz/OqIuxDTSXVSeN/N7v7FEm3S7o/e7mK5jQ1jXe7DDHNeEdodfrzvKoIe5+kcYNuj5W0t4I+huTue7PLA5LeUOdNRb3/5Ay62eWBivv5f500jfdQ04yrA567Kqc/ryLsmyVdZWYTzGyYpB9KWlFBH99gZhdmH5zIzC6U9H113lTUKyTNzq7PlrS8wl7+RqdM411vmnFV/NxVPv25u7f9R9IdGvhE/mNJj1XRQ52+Jkr6c/azo+reJL2mgZd1X2ngFdEcSX8naZ2kD7PLSzuot/+S9J6kdzUQrNEV9XaLBt4avitpW/ZzR9XPXaKvtjxvHC4LBMERdEAQhB0IgrADQRB2IAjCDgRB2IEgCDsQxP8B4fE9wnSQ/z0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(x_test_01[6].reshape(28,28),cmap='Greys') #只選出0,1資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12665"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(y_train[y_train <= 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12665"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train_01) #只取y_train <= 1對應的 x 資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_10 = np_utils.to_categorical(y_train, 10)\n",
    "y_test_10 = np_utils.to_categorical(y_test, 10)\n",
    "\n",
    "y_train_01 = y_train[y_train <= 1]\n",
    "y_train_01 = np_utils.to_categorical(y_train_01, 2) #轉換為2維\n",
    "y_test_01 = y_test[y_test <= 1]\n",
    "y_test_01 = np_utils.to_categorical(y_test_01, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((12665, 784), (2115, 784))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_01.shape, x_test_01.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((12665, 2), (2115, 2))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_01.shape, y_test_01.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3. 回顧一下 Sequential API**\n",
    "\n",
    "1. 使用 2 個 hidden layers\n",
    "2. 每個 hidden layer 用 500 個神經元\n",
    "3. Activation Function 唯一指名 sigmoid\n",
    "\n",
    "的神經網路，建立指令是透過建立 Sequential() 和 .add 的方式逐層建立"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 500)               392500    \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5010      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 648,010\n",
      "Trainable params: 648,010\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Construct a sandbox to put layers inside\n",
    "model = Sequential()\n",
    "\n",
    "# Put fully-connected layers (Dense) inside \n",
    "model.add(Dense(500, input_dim=784))\n",
    "model.add(Activation('sigmoid'))\n",
    "model.add(Dense(500))\n",
    "model.add(Activation('sigmoid'))\n",
    "model.add(Dense(10))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "觀察 model.layers，發現 model 其實就是一堆神經網路層疊起來。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.layers.core.Dense at 0x26b29c5ccc8>,\n",
       " <keras.layers.core.Activation at 0x26b2597c808>,\n",
       " <keras.layers.core.Dense at 0x26b25a0c4c8>,\n",
       " <keras.layers.core.Activation at 0x26b23a7e708>,\n",
       " <keras.layers.core.Dense at 0x26b28efaf88>,\n",
       " <keras.layers.core.Activation at 0x26b28fbcc88>]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers #用list方式來建構神經網路"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**4. 以 list 的形式使用 Sequential API**\n",
    "\n",
    "神經網路其實就是將隱藏層逐層堆疊在一起的 list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_layer = [Dense(500, input_dim = 784), Activation('sigmoid')]\n",
    "\n",
    "second_layer = [Dense(500), Activation('sigmoid')]\n",
    "\n",
    "third_layer = [Dense(10), Activation('softmax') ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.layers.core.Dense at 0x26b25e39488>,\n",
       " <keras.layers.core.Activation at 0x26b2611eb48>,\n",
       " <keras.layers.core.Dense at 0x26b25e39a08>,\n",
       " <keras.layers.core.Activation at 0x26b26137788>,\n",
       " <keras.layers.core.Dense at 0x26b25f6b5c8>,\n",
       " <keras.layers.core.Activation at 0x26b25f601c8>]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_layer + second_layer + third_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential(first_layer + second_layer + third_layer) \n",
    "#將寫成 list 的隱藏層 + 起來送進 Sequential 中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 500)               392500    \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 10)                5010      \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 648,010\n",
      "Trainable params: 648,010\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**5. 題目：建立可以辨識 0 或 1 的模型，除了最後一層，想沿用前兩層的網路設定及結構之神經網路建構**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_15 (Dense)             (None, 500)               392500    \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 10)                5010      \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 648,010\n",
      "Trainable params: 648,010\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "all_expect_last = [Dense(500, input_dim =784),\n",
    "                  Activation('sigmoid'),\n",
    "                  Dense(500),\n",
    "                  Activation('sigmoid')] #前兩層\n",
    "\n",
    "output_layer = [Dense(10),\n",
    "               Activation('softmax')] #最後一層\n",
    "\n",
    "model_0_9 = Sequential(all_expect_last + output_layer )\n",
    "model_0_9.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_0_9.load_weights('NN_handwriting_weights.h5') #權重與之前建置的模型一樣，無錯誤"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定義新的out_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_15 (Dense)             (None, 500)               392500    \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 2)                 1002      \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 644,002\n",
      "Trainable params: 644,002\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "new_output_layer = [Dense(2),\n",
    "                   Activation('softmax')]\n",
    "\n",
    "model_0_1 = Sequential(all_expect_last + new_output_layer)\n",
    "model_0_1.summary() #前兩層權重一致"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "改變layer可不可以被訓練的方法 \n",
    "\n",
    "**layer.trainable = False** \n",
    "\n",
    "True就可被訓練"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layer in all_expect_last:\n",
    "    layer.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_15 (Dense)             (None, 500)               392500    \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 2)                 1002      \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 644,002\n",
      "Trainable params: 1,002\n",
      "Non-trainable params: 643,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_0_1.summary() #Trainable params只剩1002，前四層被凍結"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_0_1.compile(loss='mse', optimizer=SGD(lr=0.1), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**6. 訓練轉移學習的神經網路**\n",
    "\n",
    "1. batch_size=100 ,  100 筆調一次參數\n",
    "2. 12665 筆資料一共要訓練幾次 (epochs), 可訓練個 5 次試試 (因為只剩 0 或 1的資料了，訓練太多容易 over-fitting)\n",
    "\n",
    "(訓練資料只剩下 1/5，且可訓練權重數量從 64萬變成 1千)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((12665, 784), (12665, 2))"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_01.shape, y_train_01.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\USER\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "Epoch 1/5\n",
      "12665/12665 [==============================] - 1s 61us/step - loss: 0.0206 - accuracy: 0.9751\n",
      "Epoch 2/5\n",
      "12665/12665 [==============================] - 1s 45us/step - loss: 0.0036 - accuracy: 0.9976\n",
      "Epoch 3/5\n",
      "12665/12665 [==============================] - 1s 58us/step - loss: 0.0029 - accuracy: 0.9974\n",
      "Epoch 4/5\n",
      "12665/12665 [==============================] - 1s 46us/step - loss: 0.0026 - accuracy: 0.9977\n",
      "Epoch 5/5\n",
      "12665/12665 [==============================] - 1s 47us/step - loss: 0.0024 - accuracy: 0.9976\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x26b32409548>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_0_1.fit(x_train_01, y_train_01, batch_size=100, epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2115/2115 [==============================] - 0s 171us/step\n"
     ]
    }
   ],
   "source": [
    "score = model_0_1.evaluate(x_test_01, y_test_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "測試資料的 loss: 0.0013454807988983353\n",
      "測試資料正確率: 0.9990543723106384\n"
     ]
    }
   ],
   "source": [
    "print('測試資料的 loss:', score[0])\n",
    "print('測試資料正確率:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keras 亦提供許多被證實有良好表現且訓練好 (pre-trained) 的模型\n",
    "\n",
    "Xception/\n",
    "VGG16/\n",
    "VGG19/\n",
    "ResNet50/\n",
    "InceptionV3/\n",
    "InceptionResNetV2/\n",
    "MobileNet/\n",
    "DenseNet/\n",
    "NASNet\n",
    "\n",
    "https://keras.io/applications/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}