julia_033_hw.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Julia 深度學習：卷積神經網路模型簡介\n",
    "\n",
    "## 作業 033：訓練 CNN 學習門牌號碼資料集\n",
    "\n",
    "訓練一個 CNN 模型來學習門牌號碼資料集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Flux\n",
    "using Flux.Data: DataLoader\n",
    "using Flux: @epochs, onecold, onehotbatch, throttle, logitcrossentropy\n",
    "using MLDatasets\n",
    "using Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "using CuArrays"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 讀取資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Float32[0.14901961 0.15294118 … 0.19607843 0.1882353; 0.15294118 0.15294118 … 0.2 0.1882353; … ; 0.16470589 0.16862746 … 0.1764706 0.17254902; 0.15294118 0.15294118 … 0.16470589 0.16470589]\n",
       "\n",
       "Float32[0.40392157 0.40784314 … 0.45882353 0.4509804; 0.40784314 0.40784314 … 0.4627451 0.4509804; … ; 0.40392157 0.39607844 … 0.45490196 0.4509804; 0.38039216 0.38039216 … 0.44313726 0.44313726]\n",
       "\n",
       "Float32[0.23529412 0.23921569 … 0.29803923 0.2901961; 0.23921569 0.23921569 … 0.3019608 0.2901961; … ; 0.24313726 0.24705882 … 0.28235295 0.2784314; 0.22352941 0.22352941 … 0.27058825 0.2784314]\n",
       "\n",
       "Float32[0.5058824 0.5254902 … 0.5411765 0.5137255; 0.49803922 0.52156866 … 0.50980395 0.47843137; … ; 0.48235294 0.49411765 … 0.39607844 0.43529412; 0.48235294 0.49019608 … 0.4392157 0.48235294]\n",
       "\n",
       "Float32[0.5568628 0.5882353 … 0.59607846 0.5686275; 0.56078434 0.58431375 … 0.5647059 0.53333336; … ; 0.5254902 0.5372549 … 0.41960785 0.4627451; 0.5294118 0.5372549 … 0.4627451 0.50980395]\n",
       "\n",
       "Float32[0.6 0.627451 … 0.64705884 0.61960787; 0.59607846 0.61960787 … 0.6156863 0.58431375; … ; 0.6117647 0.6156863 … 0.5254902 0.5647059; 0.6156863 0.61960787 … 0.5647059 0.6117647]\n",
       "\n",
       "Float32[0.5882353 0.5882353 … 0.5764706 0.62352943; 0.5882353 0.5921569 … 0.5764706 0.62352943; … ; 0.5882353 0.6 … 0.54509807 0.59607846; 0.5764706 0.5882353 … 0.5411765 0.59607846]\n",
       "\n",
       "Float32[0.627451 0.627451 … 0.64705884 0.69411767; 0.6392157 0.6392157 … 0.6431373 0.69411767; … ; 0.67058825 0.67058825 … 0.60784316 0.65882355; 0.6627451 0.6627451 … 0.60784316 0.65882355]\n",
       "\n",
       "Float32[0.6627451 0.6627451 … 0.7019608 0.7411765; 0.6666667 0.6627451 … 0.7019608 0.74509805; … ; 0.70980394 0.7058824 … 0.6666667 0.7137255; 0.7058824 0.7019608 … 0.67058825 0.7176471]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[0.050980393 0.050980393 … 0.043137256 0.047058824; 0.047058824 0.047058824 … 0.047058824 0.050980393; … ; 0.10980392 0.11764706 … 0.13725491 0.13725491; 0.105882354 0.11372549 … 0.14509805 0.15294118]\n",
       "\n",
       "Float32[0.05882353 0.05490196 … 0.047058824 0.050980393; 0.050980393 0.050980393 … 0.050980393 0.05490196; … ; 0.1254902 0.13725491 … 0.14117648 0.14117648; 0.12941177 0.13725491 … 0.14901961 0.15294118]\n",
       "\n",
       "Float32[0.05490196 0.05882353 … 0.05490196 0.05882353; 0.05882353 0.05882353 … 0.05882353 0.0627451; … ; 0.12941177 0.13333334 … 0.16470589 0.16862746; 0.12156863 0.12941177 … 0.1764706 0.18431373]\n",
       "\n",
       "Float32[0.23529412 0.23529412 … 0.2627451 0.25882354; 0.23921569 0.23921569 … 0.2627451 0.25490198; … ; 0.4745098 0.49411765 … 0.43529412 0.41960785; 0.4745098 0.49803922 … 0.4392157 0.41568628]\n",
       "\n",
       "Float32[0.21960784 0.22352941 … 0.21960784 0.20784314; 0.22745098 0.23529412 … 0.22352941 0.20784314; … ; 0.5019608 0.52156866 … 0.4392157 0.40784314; 0.5019608 0.5254902 … 0.44313726 0.40392157]\n",
       "\n",
       "Float32[0.2784314 0.29411766 … 0.26666668 0.24705882; 0.2901961 0.30588236 … 0.27058825 0.24705882; … ; 0.5411765 0.56078434 … 0.4862745 0.45882353; 0.5411765 0.5647059 … 0.49019608 0.45490196]\n",
       "\n",
       "Float32[0.40392157 0.43529412 … 0.47058824 0.43529412; 0.40392157 0.43137255 … 0.46666667 0.43529412; … ; 0.41568628 0.4509804 … 0.37254903 0.32156864; 0.4 0.44313726 … 0.36862746 0.32156864]\n",
       "\n",
       "Float32[0.45490196 0.49411765 … 0.4509804 0.40784314; 0.45882353 0.49411765 … 0.4509804 0.4117647; … ; 0.41960785 0.4627451 … 0.35686275 0.3019608; 0.40392157 0.44705883 … 0.3529412 0.29803923]\n",
       "\n",
       "Float32[0.5058824 0.54509807 … 0.52156866 0.47843137; 0.50980395 0.54509807 … 0.5176471 0.47843137; … ; 0.4 0.4392157 … 0.3529412 0.29411766; 0.38039216 0.41960785 … 0.3372549 0.2901961], [5, 2, 1, 10, 6, 1, 9, 1, 1, 8  …  4, 6, 5, 7, 1, 8, 4, 8, 1, 6])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X, train_y = SVHN2.traindata(Float32, 1:20000)\n",
    "test_X,  test_y  = SVHN2.testdata(Float32, 1:2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10×2000 Flux.OneHotMatrix{Array{Flux.OneHotVector,1}}:\n",
       " 0  0  1  0  0  1  0  1  1  0  0  0  0  …  0  0  0  0  0  0  1  0  0  0  1  0\n",
       " 0  1  0  0  0  0  0  0  0  0  0  0  0     0  0  0  0  0  0  0  0  0  0  0  0\n",
       " 0  0  0  0  0  0  0  0  0  0  1  0  0     0  0  0  0  0  0  0  0  0  0  0  0\n",
       " 0  0  0  0  0  0  0  0  0  0  0  0  0     0  0  1  0  0  0  0  0  1  0  0  0\n",
       " 1  0  0  0  0  0  0  0  0  0  0  0  1     0  0  0  0  1  0  0  0  0  0  0  0\n",
       " 0  0  0  0  1  0  0  0  0  0  0  1  0  …  0  0  0  1  0  0  0  0  0  0  0  1\n",
       " 0  0  0  0  0  0  0  0  0  0  0  0  0     1  0  0  0  0  1  0  0  0  0  0  0\n",
       " 0  0  0  0  0  0  0  0  0  1  0  0  0     0  1  0  0  0  0  0  1  0  1  0  0\n",
       " 0  0  0  0  0  0  1  0  0  0  0  0  0     0  0  0  0  0  0  0  0  0  0  0  0\n",
       " 0  0  0  1  0  0  0  0  0  0  0  0  0     0  0  0  0  0  0  0  0  0  0  0  0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y = onehotbatch(train_y, 1:10)\n",
    "test_y = onehotbatch(test_y, 1:10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataLoader((Float32[0.14901961 0.15294118 … 0.19607843 0.1882353; 0.15294118 0.15294118 … 0.2 0.1882353; … ; 0.16470589 0.16862746 … 0.1764706 0.17254902; 0.15294118 0.15294118 … 0.16470589 0.16470589]\n",
       "\n",
       "Float32[0.40392157 0.40784314 … 0.45882353 0.4509804; 0.40784314 0.40784314 … 0.4627451 0.4509804; … ; 0.40392157 0.39607844 … 0.45490196 0.4509804; 0.38039216 0.38039216 … 0.44313726 0.44313726]\n",
       "\n",
       "Float32[0.23529412 0.23921569 … 0.29803923 0.2901961; 0.23921569 0.23921569 … 0.3019608 0.2901961; … ; 0.24313726 0.24705882 … 0.28235295 0.2784314; 0.22352941 0.22352941 … 0.27058825 0.2784314]\n",
       "\n",
       "Float32[0.5058824 0.5254902 … 0.5411765 0.5137255; 0.49803922 0.52156866 … 0.50980395 0.47843137; … ; 0.48235294 0.49411765 … 0.39607844 0.43529412; 0.48235294 0.49019608 … 0.4392157 0.48235294]\n",
       "\n",
       "Float32[0.5568628 0.5882353 … 0.59607846 0.5686275; 0.56078434 0.58431375 … 0.5647059 0.53333336; … ; 0.5254902 0.5372549 … 0.41960785 0.4627451; 0.5294118 0.5372549 … 0.4627451 0.50980395]\n",
       "\n",
       "Float32[0.6 0.627451 … 0.64705884 0.61960787; 0.59607846 0.61960787 … 0.6156863 0.58431375; … ; 0.6117647 0.6156863 … 0.5254902 0.5647059; 0.6156863 0.61960787 … 0.5647059 0.6117647]\n",
       "\n",
       "Float32[0.5882353 0.5882353 … 0.5764706 0.62352943; 0.5882353 0.5921569 … 0.5764706 0.62352943; … ; 0.5882353 0.6 … 0.54509807 0.59607846; 0.5764706 0.5882353 … 0.5411765 0.59607846]\n",
       "\n",
       "Float32[0.627451 0.627451 … 0.64705884 0.69411767; 0.6392157 0.6392157 … 0.6431373 0.69411767; … ; 0.67058825 0.67058825 … 0.60784316 0.65882355; 0.6627451 0.6627451 … 0.60784316 0.65882355]\n",
       "\n",
       "Float32[0.6627451 0.6627451 … 0.7019608 0.7411765; 0.6666667 0.6627451 … 0.7019608 0.74509805; … ; 0.70980394 0.7058824 … 0.6666667 0.7137255; 0.7058824 0.7019608 … 0.67058825 0.7176471]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[0.050980393 0.050980393 … 0.043137256 0.047058824; 0.047058824 0.047058824 … 0.047058824 0.050980393; … ; 0.10980392 0.11764706 … 0.13725491 0.13725491; 0.105882354 0.11372549 … 0.14509805 0.15294118]\n",
       "\n",
       "Float32[0.05882353 0.05490196 … 0.047058824 0.050980393; 0.050980393 0.050980393 … 0.050980393 0.05490196; … ; 0.1254902 0.13725491 … 0.14117648 0.14117648; 0.12941177 0.13725491 … 0.14901961 0.15294118]\n",
       "\n",
       "Float32[0.05490196 0.05882353 … 0.05490196 0.05882353; 0.05882353 0.05882353 … 0.05882353 0.0627451; … ; 0.12941177 0.13333334 … 0.16470589 0.16862746; 0.12156863 0.12941177 … 0.1764706 0.18431373]\n",
       "\n",
       "Float32[0.23529412 0.23529412 … 0.2627451 0.25882354; 0.23921569 0.23921569 … 0.2627451 0.25490198; … ; 0.4745098 0.49411765 … 0.43529412 0.41960785; 0.4745098 0.49803922 … 0.4392157 0.41568628]\n",
       "\n",
       "Float32[0.21960784 0.22352941 … 0.21960784 0.20784314; 0.22745098 0.23529412 … 0.22352941 0.20784314; … ; 0.5019608 0.52156866 … 0.4392157 0.40784314; 0.5019608 0.5254902 … 0.44313726 0.40392157]\n",
       "\n",
       "Float32[0.2784314 0.29411766 … 0.26666668 0.24705882; 0.2901961 0.30588236 … 0.27058825 0.24705882; … ; 0.5411765 0.56078434 … 0.4862745 0.45882353; 0.5411765 0.5647059 … 0.49019608 0.45490196]\n",
       "\n",
       "Float32[0.40392157 0.43529412 … 0.47058824 0.43529412; 0.40392157 0.43137255 … 0.46666667 0.43529412; … ; 0.41568628 0.4509804 … 0.37254903 0.32156864; 0.4 0.44313726 … 0.36862746 0.32156864]\n",
       "\n",
       "Float32[0.45490196 0.49411765 … 0.4509804 0.40784314; 0.45882353 0.49411765 … 0.4509804 0.4117647; … ; 0.41960785 0.4627451 … 0.35686275 0.3019608; 0.40392157 0.44705883 … 0.3529412 0.29803923]\n",
       "\n",
       "Float32[0.5058824 0.54509807 … 0.52156866 0.47843137; 0.50980395 0.54509807 … 0.5176471 0.47843137; … ; 0.4 0.4392157 … 0.3529412 0.29411766; 0.38039216 0.41960785 … 0.3372549 0.2901961], Bool[0 0 … 1 0; 0 1 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]), 1024, 2000, true, 2000, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10  …  1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000], false)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batchsize = 1024\n",
    "train = DataLoader(train_X, train_y, batchsize=batchsize, shuffle=true)\n",
    "test = DataLoader(test_X, test_y, batchsize=batchsize)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Chain(Conv((3, 3), 3=>16, relu), MaxPool((2, 2), pad = (0, 0, 0, 0), stride = (2, 2)), Conv((3, 3), 16=>32, relu), MaxPool((2, 2), pad = (0, 0, 0, 0), stride = (2, 2)), Conv((3, 3), 32=>32, relu), MaxPool((2, 2), pad = (0, 0, 0, 0), stride = (2, 2)), flatten, Dense(512, 10), softmax)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Chain(\n",
    "    Conv((3, 3), 3=>16, pad=(1,1), relu),\n",
    "    MaxPool((2,2)),\n",
    "    Conv((3, 3), 16=>32, pad=(1,1), relu),\n",
    "    MaxPool((2,2)),\n",
    "    Conv((3, 3), 32=>32, pad=(1,1), relu),\n",
    "    MaxPool((2,2)),\n",
    "    flatten,\n",
    "    Dense(512,10),\n",
    "    softmax)                # write your model here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10×2000 Flux.OneHotMatrix{CuArray{Flux.OneHotVector,1,Nothing}}:\n",
       " 0  0  1  0  0  1  0  1  1  0  0  0  0  …  0  0  0  0  0  0  1  0  0  0  1  0\n",
       " 0  1  0  0  0  0  0  0  0  0  0  0  0     0  0  0  0  0  0  0  0  0  0  0  0\n",
       " 0  0  0  0  0  0  0  0  0  0  1  0  0     0  0  0  0  0  0  0  0  0  0  0  0\n",
       " 0  0  0  0  0  0  0  0  0  0  0  0  0     0  0  1  0  0  0  0  0  1  0  0  0\n",
       " 1  0  0  0  0  0  0  0  0  0  0  0  1     0  0  0  0  1  0  0  0  0  0  0  0\n",
       " 0  0  0  0  1  0  0  0  0  0  0  1  0  …  0  0  0  1  0  0  0  0  0  0  0  1\n",
       " 0  0  0  0  0  0  0  0  0  0  0  0  0     1  0  0  0  0  1  0  0  0  0  0  0\n",
       " 0  0  0  0  0  0  0  0  0  1  0  0  0     0  1  0  0  0  0  0  1  0  1  0  0\n",
       " 0  0  0  0  0  0  1  0  0  0  0  0  0     0  0  0  0  0  0  0  0  0  0  0  0\n",
       " 0  0  0  1  0  0  0  0  0  0  0  0  0     0  0  0  0  0  0  0  0  0  0  0  0"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = model |> gpu\n",
    "train_X = train_X |> gpu\n",
    "train_y = train_y |> gpu\n",
    "test_X = test_X |> gpu\n",
    "test_y = test_y |> gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataLoader((Float32[0.14901961 0.15294118 … 0.19607843 0.1882353; 0.15294118 0.15294118 … 0.2 0.1882353; … ; 0.16470589 0.16862746 … 0.1764706 0.17254902; 0.15294118 0.15294118 … 0.16470589 0.16470589]\n",
       "\n",
       "Float32[0.40392157 0.40784314 … 0.45882353 0.4509804; 0.40784314 0.40784314 … 0.4627451 0.4509804; … ; 0.40392157 0.39607844 … 0.45490196 0.4509804; 0.38039216 0.38039216 … 0.44313726 0.44313726]\n",
       "\n",
       "Float32[0.23529412 0.23921569 … 0.29803923 0.2901961; 0.23921569 0.23921569 … 0.3019608 0.2901961; … ; 0.24313726 0.24705882 … 0.28235295 0.2784314; 0.22352941 0.22352941 … 0.27058825 0.2784314]\n",
       "\n",
       "Float32[0.5058824 0.5254902 … 0.5411765 0.5137255; 0.49803922 0.52156866 … 0.50980395 0.47843137; … ; 0.48235294 0.49411765 … 0.39607844 0.43529412; 0.48235294 0.49019608 … 0.4392157 0.48235294]\n",
       "\n",
       "Float32[0.5568628 0.5882353 … 0.59607846 0.5686275; 0.56078434 0.58431375 … 0.5647059 0.53333336; … ; 0.5254902 0.5372549 … 0.41960785 0.4627451; 0.5294118 0.5372549 … 0.4627451 0.50980395]\n",
       "\n",
       "Float32[0.6 0.627451 … 0.64705884 0.61960787; 0.59607846 0.61960787 … 0.6156863 0.58431375; … ; 0.6117647 0.6156863 … 0.5254902 0.5647059; 0.6156863 0.61960787 … 0.5647059 0.6117647]\n",
       "\n",
       "Float32[0.5882353 0.5882353 … 0.5764706 0.62352943; 0.5882353 0.5921569 … 0.5764706 0.62352943; … ; 0.5882353 0.6 … 0.54509807 0.59607846; 0.5764706 0.5882353 … 0.5411765 0.59607846]\n",
       "\n",
       "Float32[0.627451 0.627451 … 0.64705884 0.69411767; 0.6392157 0.6392157 … 0.6431373 0.69411767; … ; 0.67058825 0.67058825 … 0.60784316 0.65882355; 0.6627451 0.6627451 … 0.60784316 0.65882355]\n",
       "\n",
       "Float32[0.6627451 0.6627451 … 0.7019608 0.7411765; 0.6666667 0.6627451 … 0.7019608 0.74509805; … ; 0.70980394 0.7058824 … 0.6666667 0.7137255; 0.7058824 0.7019608 … 0.67058825 0.7176471]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[0.050980393 0.050980393 … 0.043137256 0.047058824; 0.047058824 0.047058824 … 0.047058824 0.050980393; … ; 0.10980392 0.11764706 … 0.13725491 0.13725491; 0.105882354 0.11372549 … 0.14509805 0.15294118]\n",
       "\n",
       "Float32[0.05882353 0.05490196 … 0.047058824 0.050980393; 0.050980393 0.050980393 … 0.050980393 0.05490196; … ; 0.1254902 0.13725491 … 0.14117648 0.14117648; 0.12941177 0.13725491 … 0.14901961 0.15294118]\n",
       "\n",
       "Float32[0.05490196 0.05882353 … 0.05490196 0.05882353; 0.05882353 0.05882353 … 0.05882353 0.0627451; … ; 0.12941177 0.13333334 … 0.16470589 0.16862746; 0.12156863 0.12941177 … 0.1764706 0.18431373]\n",
       "\n",
       "Float32[0.23529412 0.23529412 … 0.2627451 0.25882354; 0.23921569 0.23921569 … 0.2627451 0.25490198; … ; 0.4745098 0.49411765 … 0.43529412 0.41960785; 0.4745098 0.49803922 … 0.4392157 0.41568628]\n",
       "\n",
       "Float32[0.21960784 0.22352941 … 0.21960784 0.20784314; 0.22745098 0.23529412 … 0.22352941 0.20784314; … ; 0.5019608 0.52156866 … 0.4392157 0.40784314; 0.5019608 0.5254902 … 0.44313726 0.40392157]\n",
       "\n",
       "Float32[0.2784314 0.29411766 … 0.26666668 0.24705882; 0.2901961 0.30588236 … 0.27058825 0.24705882; … ; 0.5411765 0.56078434 … 0.4862745 0.45882353; 0.5411765 0.5647059 … 0.49019608 0.45490196]\n",
       "\n",
       "Float32[0.40392157 0.43529412 … 0.47058824 0.43529412; 0.40392157 0.43137255 … 0.46666667 0.43529412; … ; 0.41568628 0.4509804 … 0.37254903 0.32156864; 0.4 0.44313726 … 0.36862746 0.32156864]\n",
       "\n",
       "Float32[0.45490196 0.49411765 … 0.4509804 0.40784314; 0.45882353 0.49411765 … 0.4509804 0.4117647; … ; 0.41960785 0.4627451 … 0.35686275 0.3019608; 0.40392157 0.44705883 … 0.3529412 0.29803923]\n",
       "\n",
       "Float32[0.5058824 0.54509807 … 0.52156866 0.47843137; 0.50980395 0.54509807 … 0.5176471 0.47843137; … ; 0.4 0.4392157 … 0.3529412 0.29411766; 0.38039216 0.41960785 … 0.3372549 0.2901961], Bool[0 0 … 1 0; 0 1 … 0 0; … ; 0 0 … 0 0; 0 0 … 0 0]), 64, 2000, true, 2000, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10  …  1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000], false)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batchsize=64\n",
    "train = DataLoader(train_X, train_y, batchsize=batchsize, shuffle=true)\n",
    "test = DataLoader(test_X, test_y, batchsize=batchsize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "loss (generic function with 1 method)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss(x, y) = logitcrossentropy(model(x), y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_loss (generic function with 1 method)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function test_loss()\n",
    "    l = 0f0\n",
    "    for (x, y) in test\n",
    "        l += loss(x, y)\n",
    "    end\n",
    "    l/length(test)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "evalcb (generic function with 1 method)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evalcb() = @show(test_loss())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 2.2928054f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 1\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 2.1301587f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 2\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 2.1325555f0\n",
      "test_loss() = 2.0787897f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 3\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 2.0809517f0\n",
      "test_loss() = 2.0089386f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 4\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 2.0065923f0\n",
      "test_loss() = 1.9842044f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 5\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 2.0017412f0\n",
      "test_loss() = 1.976253f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 6\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 1.9754242f0\n",
      "test_loss() = 1.965752f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 7\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 1.9764447f0\n",
      "test_loss() = 1.9667717f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 8\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 1.9632901f0\n",
      "test_loss() = 1.9626763f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 9\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 1.9603214f0\n",
      "test_loss() = 1.9613765f0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Epoch 10\n",
      "└ @ Main C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\optimise\\train.jl:121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_loss() = 1.9594471f0\n",
      "test_loss() = 1.9626478f0\n"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "@epochs epochs Flux.train!(loss, params(model), train, ADAM(), cb=throttle(evalcb, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "accuracy (generic function with 1 method)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy(x, y) = mean(onecold(model(x)) .== onecold(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "Out of GPU memory trying to allocate 62.500 MiB\nEffective GPU memory usage: 99.72% (1.994 GiB/2.000 GiB)\nCuArrays allocator usage: 490.768 MiB\nBinnedPool usage: 490.768 MiB (490.768 MiB allocated, 0 bytes cached)\n",
     "output_type": "error",
     "traceback": [
      "Out of GPU memory trying to allocate 62.500 MiB\nEffective GPU memory usage: 99.72% (1.994 GiB/2.000 GiB)\nCuArrays allocator usage: 490.768 MiB\nBinnedPool usage: 490.768 MiB (490.768 MiB allocated, 0 bytes cached)\n",
      "",
      "Stacktrace:",
      " [1] alloc at C:\\Users\\zero0\\.julia\\packages\\CuArrays\\YFdj7\\src\\memory.jl:202 [inlined]",
      " [2] CuArray{Float32,4,P} where P(::UndefInitializer, ::NTuple{4,Int64}) at C:\\Users\\zero0\\.julia\\packages\\CuArrays\\YFdj7\\src\\array.jl:107",
      " [3] CuArray at C:\\Users\\zero0\\.julia\\packages\\CuArrays\\YFdj7\\src\\array.jl:115 [inlined]",
      " [4] similar at .\\abstractarray.jl:675 [inlined]",
      " [5] similar at .\\abstractarray.jl:674 [inlined]",
      " [6] similar at C:\\Users\\zero0\\.julia\\packages\\CuArrays\\YFdj7\\src\\broadcast.jl:11 [inlined]",
      " [7] copy at .\\broadcast.jl:862 [inlined]",
      " [8] materialize(::Base.Broadcast.Broadcasted{CuArrays.CuArrayStyle{4},Nothing,typeof(relu),Tuple{Base.Broadcast.Broadcasted{CuArrays.CuArrayStyle{4},Nothing,typeof(+),Tuple{CuArray{Float32,4,Nothing},CuArray{Float32,4,CuArray{Float32,1,Nothing}}}}}}) at .\\broadcast.jl:837",
      " [9] (::Conv{2,2,typeof(relu),CuArray{Float32,4,Nothing},CuArray{Float32,1,Nothing}})(::CuArray{Float32,4,Nothing}) at C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\layers\\conv.jl:61",
      " [10] applychain(::Tuple{Conv{2,2,typeof(relu),CuArray{Float32,4,Nothing},CuArray{Float32,1,Nothing}},MaxPool{2,4},Conv{2,2,typeof(relu),CuArray{Float32,4,Nothing},CuArray{Float32,1,Nothing}},MaxPool{2,4},typeof(flatten),Dense{typeof(identity),CuArray{Float32,2,Nothing},CuArray{Float32,1,Nothing}},typeof(softmax)}, ::CuArray{Float32,4,Nothing}) at C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\layers\\basic.jl:36 (repeats 3 times)",
      " [11] (::Chain{Tuple{Conv{2,2,typeof(relu),CuArray{Float32,4,Nothing},CuArray{Float32,1,Nothing}},MaxPool{2,4},Conv{2,2,typeof(relu),CuArray{Float32,4,Nothing},CuArray{Float32,1,Nothing}},MaxPool{2,4},Conv{2,2,typeof(relu),CuArray{Float32,4,Nothing},CuArray{Float32,1,Nothing}},MaxPool{2,4},typeof(flatten),Dense{typeof(identity),CuArray{Float32,2,Nothing},CuArray{Float32,1,Nothing}},typeof(softmax)}})(::CuArray{Float32,4,Nothing}) at C:\\Users\\zero0\\.julia\\packages\\Flux\\Fj3bt\\src\\layers\\basic.jl:38",
      " [12] accuracy(::CuArray{Float32,4,Nothing}, ::Flux.OneHotMatrix{CuArray{Flux.OneHotVector,1,Nothing}}) at .\\In[36]:1",
      " [13] top-level scope at In[40]:2",
      " [14] include_string(::Function, ::Module, ::String, ::String) at .\\loading.jl:1091",
      " [15] execute_code(::String, ::String) at C:\\Users\\zero0\\.julia\\packages\\IJulia\\rWZ9e\\src\\execute_request.jl:27",
      " [16] execute_request(::ZMQ.Socket, ::IJulia.Msg) at C:\\Users\\zero0\\.julia\\packages\\IJulia\\rWZ9e\\src\\execute_request.jl:86",
      " [17] #invokelatest#1 at .\\essentials.jl:710 [inlined]",
      " [18] invokelatest at .\\essentials.jl:709 [inlined]",
      " [19] eventloop(::ZMQ.Socket) at C:\\Users\\zero0\\.julia\\packages\\IJulia\\rWZ9e\\src\\eventloop.jl:8",
      " [20] (::IJulia.var\"#15#18\")() at .\\task.jl:356"
     ]
    }
   ],
   "source": [
    "using Printf\n",
    "@printf(\"Accuracy: %.2f%%\\n\", accuracy(test_X, test_y)*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.5.0",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}