kaggle_mnist_digit_recognizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "kaggle-mnist-digit-recognizer.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "aY8FX8b6_zsU",
        "EXsOmNOqAWaU"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/JacksonIsaac/colab_notebooks/blob/master/kaggle_mnist_digit_recognizer.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "bwMnKOtFk0Kn",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Kaggle Notebook**\n",
        "\n",
        "For *Digit Recognizer* competition: \n",
        "\n",
        "https://www.kaggle.com/c/digit-recognizer"
      ]
    },
    {
      "metadata": {
        "id": "mJ_m9u5pl47J",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 445
        },
        "outputId": "f37073a1-607c-4a0c-c1e3-55fec979abdd"
      },
      "cell_type": "code",
      "source": [
        "!pip install kaggle"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting kaggle\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c6/78/832b9a9ec6b3baf8ec566e1f0a695f2fd08d2c94a6797257a106304bfc3c/kaggle-1.4.7.1.tar.gz (52kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: urllib3<1.23.0,>=1.15 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.22)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.11.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle) (2018.8.24)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.5.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.18.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.25.0)\n",
            "Collecting python-slugify (from kaggle)\n",
            "  Downloading https://files.pythonhosted.org/packages/00/ad/c778a6df614b6217c30fe80045b365bfa08b5dd3cb02e8b37a6d25126781/python-slugify-1.2.6.tar.gz\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (2.6)\n",
            "Collecting Unidecode>=0.04.16 (from python-slugify->kaggle)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/59/ef/67085e30e8bbcdd76e2f0a4ad8151c13a2c5bce77c85f8cad6e1f16fb141/Unidecode-1.0.22-py2.py3-none-any.whl (235kB)\n",
            "\u001b[K    100% |████████████████████████████████| 235kB 4.6MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: kaggle, python-slugify\n",
            "  Running setup.py bdist_wheel for kaggle ... \u001b[?25l-\b \b\\\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/44/2c/df/22a6eeb780c36c28190faef6252b739fdc47145fd87a6642d4\n",
            "  Running setup.py bdist_wheel for python-slugify ... \u001b[?25l-\b \b\\\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/e3/65/da/2045deea3098ed7471eca0e2460cfbd3fdfe8c1d6fa6fcac92\n",
            "Successfully built kaggle python-slugify\n",
            "Installing collected packages: Unidecode, python-slugify, kaggle\n",
            "Successfully installed Unidecode-1.0.22 kaggle-1.4.7.1 python-slugify-1.2.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "glB40LzevXQn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5ede0928-84d2-46b3-bb57-2a2856b0c324"
      },
      "cell_type": "code",
      "source": [
        "## Load Kaggle config JSON\n",
        "from googleapiclient.discovery import build\n",
        "import io, os\n",
        "from googleapiclient.http import MediaIoBaseDownload\n",
        "from google.colab import auth\n",
        "\n",
        "auth.authenticate_user()\n",
        "\n",
        "drive_service = build('drive', 'v3')\n",
        "results = drive_service.files().list(\n",
        "        q=\"name = 'kaggle.json'\", fields=\"files(id)\").execute()\n",
        "kaggle_api_key = results.get('files', [])\n",
        "\n",
        "filename = \"/content/.kaggle/kaggle.json\"\n",
        "os.makedirs(os.path.dirname(filename), exist_ok=True)\n",
        "\n",
        "request = drive_service.files().get_media(fileId=kaggle_api_key[0]['id'])\n",
        "fh = io.FileIO(filename, 'wb')\n",
        "downloader = MediaIoBaseDownload(fh, request)\n",
        "done = False\n",
        "while done is False:\n",
        "    status, done = downloader.next_chunk()\n",
        "    print(\"Download %d%%.\" % int(status.progress() * 100))\n",
        "os.chmod(filename, 600)\n",
        "\n",
        "!mkdir ~/.kaggle\n",
        "!cp /content/.kaggle/kaggle.json ~/.kaggle/kaggle.json"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Download 100%.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "eWpo0ipKtXRB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "6f8a25f9-d251-40d6-89e4-d8c5cea9f1f9"
      },
      "cell_type": "code",
      "source": [
        "!kaggle competitions download -c digit-recognizer -p data/"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading train.csv to data\n",
            " 89%|█████████████████████████████████▋    | 65.0M/73.2M [00:00<00:00, 94.0MB/s]\n",
            "100%|███████████████████████████████████████| 73.2M/73.2M [00:00<00:00, 165MB/s]\n",
            "Downloading test.csv to data\n",
            " 84%|████████████████████████████████▊      | 41.0M/48.8M [00:00<00:00, 104MB/s]\n",
            "100%|███████████████████████████████████████| 48.8M/48.8M [00:00<00:00, 141MB/s]\n",
            "Downloading sample_submission.csv to data\n",
            "  0%|                                                | 0.00/235k [00:00<?, ?B/s]\n",
            "100%|████████████████████████████████████████| 235k/235k [00:00<00:00, 62.0MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "GOd3iDIv87jV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!ls data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "RsSa_NYAtxzX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "outputId": "8beabf09-b9d4-4f7e-e3fe-cff0445a6d1a"
      },
      "cell_type": "code",
      "source": [
        "!kaggle competitions list"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ref                                              deadline             category            reward  teamCount  userHasEntered  \r\n",
            "-----------------------------------------------  -------------------  ---------------  ---------  ---------  --------------  \r\n",
            "digit-recognizer                                 2030-01-01 00:00:00  Getting Started  Knowledge       2360            True  \r\n",
            "titanic                                          2030-01-01 00:00:00  Getting Started  Knowledge       9847           False  \r\n",
            "house-prices-advanced-regression-techniques      2030-01-01 00:00:00  Getting Started  Knowledge       4254            True  \r\n",
            "imagenet-object-localization-challenge           2029-12-31 07:00:00  Research         Knowledge         24           False  \r\n",
            "competitive-data-science-predict-future-sales    2019-01-01 23:59:00  Playground           Kudos       1343           False  \r\n",
            "rsna-pneumonia-detection-challenge               2018-10-24 23:59:00  Featured           $30,000        515           False  \r\n",
            "tgs-salt-identification-challenge                2018-10-19 23:59:00  Featured          $100,000       2006           False  \r\n",
            "airbus-ship-detection                            2018-10-04 23:59:00  Featured           $60,000        672            True  \r\n",
            "new-york-city-taxi-fare-prediction               2018-09-25 23:59:00  Playground       Knowledge        778            True  \r\n",
            "forest-cover-type-kernels-only                   2018-09-24 23:59:00  Playground       Knowledge        278           False  \r\n",
            "demand-forecasting-kernels-only                  2018-09-24 23:59:00  Playground       Knowledge        342           False  \r\n",
            "whats-cooking-kernels-only                       2018-09-24 23:59:00  Playground       Knowledge        396           False  \r\n",
            "flavours-of-physics-kernels-only                 2018-09-24 23:59:00  Playground       Knowledge         46           False  \r\n",
            "movie-review-sentiment-analysis-kernels-only     2018-09-24 23:59:00  Playground       Knowledge        297           False  \r\n",
            "costa-rican-household-poverty-prediction         2018-09-19 23:59:00  Playground            Swag        472           False  \r\n",
            "google-ai-open-images-object-detection-track     2018-08-30 23:59:00  Featured           $30,000        454           False  \r\n",
            "google-ai-open-images-visual-relationship-track  2018-08-30 23:59:00  Featured           $20,000        232           False  \r\n",
            "home-credit-default-risk                         2018-08-29 23:59:00  Featured           $70,000       7198           False  \r\n",
            "santander-value-prediction-challenge             2018-08-20 23:59:00  Featured           $60,000       4484            True  \r\n",
            "trackml-particle-identification                  2018-08-13 23:59:00  Featured           $25,000        656           False  \r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "R6EdekUG83Xx",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Import packages and utilities"
      ]
    },
    {
      "metadata": {
        "id": "CJWKQhYI7PH8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%matplotlib inline\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gy-i2OZQ9ygW",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Loading training and test dataset"
      ]
    },
    {
      "metadata": {
        "id": "XI07Z9y28Jre",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "input = pd.read_csv('./data/train.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hSXXWwmq8NuN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 253
        },
        "outputId": "b2c18d73-ddc5-4e88-8018-7921d054e061"
      },
      "cell_type": "code",
      "source": [
        "input.head()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>pixel0</th>\n",
              "      <th>pixel1</th>\n",
              "      <th>pixel2</th>\n",
              "      <th>pixel3</th>\n",
              "      <th>pixel4</th>\n",
              "      <th>pixel5</th>\n",
              "      <th>pixel6</th>\n",
              "      <th>pixel7</th>\n",
              "      <th>pixel8</th>\n",
              "      <th>...</th>\n",
              "      <th>pixel774</th>\n",
              "      <th>pixel775</th>\n",
              "      <th>pixel776</th>\n",
              "      <th>pixel777</th>\n",
              "      <th>pixel778</th>\n",
              "      <th>pixel779</th>\n",
              "      <th>pixel780</th>\n",
              "      <th>pixel781</th>\n",
              "      <th>pixel782</th>\n",
              "      <th>pixel783</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 785 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n",
              "0      1       0       0       0       0       0       0       0       0   \n",
              "1      0       0       0       0       0       0       0       0       0   \n",
              "2      1       0       0       0       0       0       0       0       0   \n",
              "3      4       0       0       0       0       0       0       0       0   \n",
              "4      0       0       0       0       0       0       0       0       0   \n",
              "\n",
              "   pixel8    ...     pixel774  pixel775  pixel776  pixel777  pixel778  \\\n",
              "0       0    ...            0         0         0         0         0   \n",
              "1       0    ...            0         0         0         0         0   \n",
              "2       0    ...            0         0         0         0         0   \n",
              "3       0    ...            0         0         0         0         0   \n",
              "4       0    ...            0         0         0         0         0   \n",
              "\n",
              "   pixel779  pixel780  pixel781  pixel782  pixel783  \n",
              "0         0         0         0         0         0  \n",
              "1         0         0         0         0         0  \n",
              "2         0         0         0         0         0  \n",
              "3         0         0         0         0         0  \n",
              "4         0         0         0         0         0  \n",
              "\n",
              "[5 rows x 785 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "metadata": {
        "id": "vxvrA2ID8QvN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "1f95f7fa-d601-4042-8a41-1192b9d5f68d"
      },
      "cell_type": "code",
      "source": [
        "y = input.iloc[:, 0].values ## First column is the label\n",
        "x = input.iloc[:, 1:].values\n",
        "\n",
        "print(x.shape)\n",
        "print(y.shape)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(42000, 784)\n",
            "(42000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "XHv-GB5o8zu6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ec882455-6e17-4aa9-aa8f-bfdc2dd5d890"
      },
      "cell_type": "code",
      "source": [
        "X_test = pd.read_csv('./data/test.csv')\n",
        "\n",
        "print(X_test.shape)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(28000, 784)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "EA1msFug93xN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Exploring y label and categorization"
      ]
    },
    {
      "metadata": {
        "id": "GcI-sLN29g1Q",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "3f679986-d671-4d32-80a5-4411f48a74e3"
      },
      "cell_type": "code",
      "source": [
        "print(np.unique(y))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0 1 2 3 4 5 6 7 8 9]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "xuQpEUF39o4P",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#from sklearn.preprocessing import OneHotEncoder"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-RFDTFwHCNXu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#enc = OneHotEncoder()\n",
        "#enc.fit(x.split() for x in y)\n",
        "#y_encoded = enc.transform(x.split() for x in y).toarray()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "um64PYzyDMVy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "33869bc3-9bc0-4ebd-9c6f-530bfce04761"
      },
      "cell_type": "code",
      "source": [
        "from keras.utils import np_utils"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "q7m6DJYBDfqh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "deebcc4d-fa2f-4e59-ec91-17a8d5857a1d"
      },
      "cell_type": "code",
      "source": [
        "y_encoded = np_utils.to_categorical(y)\n",
        "print(y_encoded.shape)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(42000, 10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "oO1YkSBye78M",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x_4d = x.reshape(x.shape[0], 28, 28, 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Cm-H3Uj9e6jm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "10eff3cb-77d8-4747-ffd8-1b2048c7272b"
      },
      "cell_type": "code",
      "source": [
        "x_4d.shape"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(42000, 28, 28, 1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "metadata": {
        "id": "pLvr1FtDDyzn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mean_px = x_4d.mean().astype(np.float32)\n",
        "std_px = x_4d.std().astype(np.float32)\n",
        "\n",
        "def normalize_data(X):\n",
        "    return (X - mean_px) / std_px"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Zpdyqw_sezwG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#x_norm = normalize_data(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SGuOIb3ne3lv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#x_norm.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "V4R2NmwpmhiY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 4318
        },
        "outputId": "0dd29047-cdc1-405d-c161-7b74090d9065"
      },
      "cell_type": "code",
      "source": [
        "x_norm = normalize_data(x_4d)\n",
        "\n",
        "x_norm[:5,:]"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[[[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        ...,\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]]],\n",
              "\n",
              "\n",
              "       [[[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        ...,\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]]],\n",
              "\n",
              "\n",
              "       [[[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        ...,\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]]],\n",
              "\n",
              "\n",
              "       [[[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        ...,\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]]],\n",
              "\n",
              "\n",
              "       [[[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        ...,\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]],\n",
              "\n",
              "        [[-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         ...,\n",
              "         [-0.42462978],\n",
              "         [-0.42462978],\n",
              "         [-0.42462978]]]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "metadata": {
        "id": "RjKjQt8gm4Ml",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "fe46d62c-f9df-47ad-d23f-7712471bdd04"
      },
      "cell_type": "code",
      "source": [
        "x_temp = x_norm.reshape(x_norm.shape[0], 28, 28)\n",
        "for i in range(1, 5):\n",
        "    plt.subplot(330 + (i+1))\n",
        "    plt.imshow(x_temp[i], cmap=plt.get_cmap('gray'))\n",
        "    plt.title(y[i]);"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUwAAAC7CAYAAAAOjqdKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAF0JJREFUeJzt3XuQFOW5x/Hv4CXAoiwSRbQ2aiXy\nWOWiJl4iFQhw5K4npETDSVkGFQOVeL9UJEUlByUVFaXQGASvgaDWUVKJCBIgXkpiODF4S5Y6+Aoq\neAGBoICgILB7/pjp3pndnZmenZ7p6Znfp4qi5+3e7nf3WV6e7n4viZaWFkREJL8uUVdARCQu1GCK\niASkBlNEJCA1mCIiAanBFBEJSA2miEhAh0ZdAZHOMLP/AO4GegAbgcudcx9GWysplpkdBtwB3Ag0\nVFpMlWFK7JhZHfA/wJXOuX7AYmButLWSkCwCdkddiWzUYEoc/QfwrnPu9dTnR4ERZnZEhHWScEx3\nzv131JXIRg2mxFE/4B3vg3NuN7Ad+EZkNZJQOOf+N+o65KIGU+KoO7C3TdkXQF0EdZEaogZT4mgP\n0LVNWXcq+NmXVAc1mBJHb5F2+21mPYFewLrIaiQ1QQ2mxNGLwAlmNjD1+QZgiXNuT4R1khqQ0PRu\nEkdmNgS4l+Rzy/XAZc65jyOtlBTFzPoAL3kfSb7YOwCc55z7KLKKpVGDKSISkG7JRUQCUoMpIhKQ\nxpJLBjObBZwLtADXOedWR1wlCYliWzxlmOIzs8HAyc65AcBE4DcRV0lCotiGQw2mpDsPeBrAObcW\n6GVmR0ZbJQmJYhsCNZiS7lhgW9rnbamyrBKJRMuaNWtaEolErP6EVeeSRiNcBcU2rnENq97Zfi56\nhim5JPId0NTURGNjI3HsnhbHOocoZ2zjHFcoXWzVYEq6TWRmHccBm3N9Qf/+/WlpaSGRyNu2VpSw\n6hyjBqWg2MY1rhBObLPFVbfkkm4FcBGAmX0L2OSc+yzaKklIFNsQaKSPZDCzO4DvAs3AVc65f+Y6\nPpFItMQxEwkxw4zNN15IbOMaVwgtw+zwBGowpShx/YdViw1mIeIaVyhtg6lbchGRgNRgiogEpAZT\nRCQgdSuSinXUUUcB0KNHDwCuuuqqdsd8+9vfBuD+++/3y3bt2gXA8uXLgVh1/alJhxxyiL89Y8YM\nAJqbmwGYMmWKv+/gwYPlrVgHlGGKiASkt+RSlLDeph5xROuS4qNHjwbgscceA+DQQ/PfCK1b17qc\nT0NDAwDz588H4M477wRgw4YN/jF6S55bOd+Sd+vWzd/esydzlZHu3bv723v3tl0otGN6Sy4iUgHU\nYIqIBKRbcilKsbdu9fX1ACxYsMAvO//880Opm2fLli0AjB071i975ZVXqK+vZ+fOnUWdW7fkxdMt\nuYhIFVKGKUUpNhMZNWoUAEuXLg2zWnklEgl+8pOfMHfu3KLOowyzeLkyzPSuZHPmzAl0PmWYIiIV\nQB3XJRIDBw4E4JZbbunU11933XUAbNq0CYCbb77Z3+d1Zs/nrrvuYvv27QAsXLiwU/WQ0kp/7hw0\nwywlZZgiIgEpw6xhZtYILAJmOed+a2YNwALgEJKzcV/qnNsXZR2lcIpr6ajBrFFmVgfcBzyfVnwb\nMNs5t9DMfg1cAZTkPuj6668HYPDgwVmPefXVV4FkF6C2XnzxRQDWrFkDwLJly/x93hh07zb7nHPO\n6fD8dXV1/OAHP8g4Nu6ijmu10y157doHjCG51otnCPBMansxMKzMdZLiKa4lpAyzRjnnDgAHzCy9\nuC7tVm0r0DfMa6Z39ejSJfv/1ZdcckmyAlu3AvD8889nPdaT3h3F2/ayzrPOOivrNU855RQALrjg\nAr9syZIlea9XqaKIay1RgynZBOrI1tTUBIQ7hdoTTzwR2rlySSQSNDY2ArB48eKyXLMCRBbXzhg5\ncqS/XUhdtMyulMNuM+vmnPsCOJ7M27oOFbIc6+mnn+5vv/HGG1mPO+GEEwD44IMP8tc4gHHjxgGZ\nzykTiUTGP6qHHnrI3548eXLgc0fdoARU0rgW6/DDD/e3vf+4hg8fDsCKFSv8fd4gh3y0zK6Uy3PA\nuNT2OGBZjmMlPhTXkCjDrFFmdiYwEzgR2G9mFwGXAPPMbDKwEZgf5jVPOumkrPu8WdIB9u/fH+Zl\nWbVqVbtr9OzZM9RrVIoo4lqsL7/80t+eN28e0JphVho1mDXKOfcaybenbVXmb6oEoriWlm7JRUQC\nUoYpZbNjx46s+/7xj3/4259++mmo1928eTOQOSPSD3/4w4xj0t/Geouu7d69O9R6SMfSlyAZMGBA\nhDXJTxmmiEhAmg9TihJk3sQjjzwSgLffftsvO+aYY7IeH3a3Ik/6TO5LlizJ2nWkd+/eQLBMV/Nh\nFi/XfJgRdivSfJgiIsXQM0wpOe8ZVa6sshw++uijSK8v8acMU0QkIDWYIiIB6ZZcSs7rTvT444/7\nZd6MRCJxogxTRCQgZZhScs3NzQD85S9/8ctyZZjerELDhiXnuS22A3l9fT0A8+dnH0Kdvtxurg72\nUtuUYYqIBKQMU8pm0aJF/vabb74JwBlnnNHuOG8NnhdeeAFoXYrXW8cnqKOPPhqAu+++G0jO8djW\nF198AcCdd97pl2kwh2SjDFNEJCBlmDXMzGYAg0j+HtwOrKaEy7Hu3LnT37722msBmDMnuXjhqaee\n2u54by2eW2+9Feh4qKI3x2X6rN1du3YFWp9ZdpRZerwJOTZu3Bjwu6h85Y5rse67776oqxCYMswa\nZWZDgUbn3ABgFHAPrcuxDgLWk1yOVWJEcS0tNZi1ayVwcWp7B1CHlmOtBoprCemWvEY55w4C3tQw\nE4GlwMhyLcf68ssvAzB9+nQAHnnkEX9fXV1dxrEDBw4E4PXXX293nm3btgHQvXv3rF+fS/rCaNUg\n6rh2RkNDg79djtmRiqEGs8aZ2ViS/7BGAOvSdsViOdZiJvRIJBI8+eSTAP7f1SLucfVomV2pGGY2\nEpgKjHLO7TSzyJZjveGGG/ztmTNnFnWubNJfOtXX1zN+/HieffZZoP08jEFF3aB0pJLiGsTy5cv9\n7baLn2mZXakIZtYTuAu4wDn3SapYy7HGnOJaWsowa9d44KvAU2bmlU0AHo5iOdaHH37Y3/ayjKAZ\nRT5e9jh+/Hi/bPny5Tz11FOhnL/CVFRcq40azBrlnHsQeLCDXVqONcYU19LSLbmISEBaBE2KUorF\nsryROt5sRSNGjADg6quvTr8u0PpwPv363sgRb4TQgQMHgMyXPmHVWYugFW/w4MH+dtv5AoYMGeJv\nr1y5MtD5tAiaiEgFUIYpRSlnJhImZZi5xTWuoAxTRKQiqMEUEQlIDaaISEBqMEVEAlKDKSISkBpM\nEZGAamJopJmdDywBTnLObYi4OhKCbt26cfTRR9OlSxf279/Pli1b/A7qIqVS9RmmmXUH7gA+yXes\nxEMikaBv375s2bKFDRs2sGfPnqLmxRQJqhYyzGkkF4D6acT1qEpeB99yDoAws/8Epu7du/fc1Oce\nwCf9+vXr7Zz7LOh5NGgjuyjiGqZS1buqM0wz609ylpZZUddFQtUPeMf74JzbDWwHvhFZjaQmVG2D\naWYJYC5wjXNuf9T1kVB1B/a2KfuC5IJfIiVTtQ0mMAn4P+fcy1FXREK3B+japqw7sDuCukgNqeYG\ncyww1sw+NrOPgQZgdWrdZom3t0i7/U4ty9CLzMW+REJXM7MVmdkGYIi6FcWfmXUD3gMucs69bGbT\ngP7OuXG5v1KkONWcYUqVSq1++F/AbDNbD5wLXBVtraQW1EyGKeEzs1kkG6sW4Drn3OqIq5SVmc0A\nBpHsSnc7sJpkd7NDgM3Apc65fdHVsHLEKa5Q3tgqw5ROMbPBwMnOuQHAROA3EVcpq9Rz68ZUXUcB\n9wC3AbOdc4OA9cAVEVaxYsQprlD+2KrBlM46D3gawDm3FuhlZkdGW6WsVgIXp7Z3kOx+NAR4JlW2\nGBhW/mpVpDjFFcoc206P9Ilb2i7BFBDXY4HX0j5vS5XtKm0NC+ecO0iyKxIks6alwMi027StQN8o\n6laBYhNXKH9sO5Vhxi1tl2CKjGvFL/5iZmNJfl9Xt9lV8XWPUCx+NuWKbWdvyeOWtkswhcR1E8nM\nw3McyQfsFcnMRgJTgdHOuZ3A7lT3JIDjSX4/ErO4Qnlj29lb8oLS9kQi0QLQ1NRE//79O3nJaIRR\n5xitLFhIXFcAtyYSiblNTU28/fbbALtisMrg9kQi4cX18/T6er+nhYhRbINaAdwKPGBm3wI2FTKh\nSbmlBi3cBQxzznkzkj0HjAMeS/29LKzrhTVbUc5fmqamJhobG4F4zn4SxzqHJGtcnXOrzOy1pqam\nYY2NjbH8GcWxzqXmxdXMVgHNVH7/1vHAV4GnzMwrmwA8bGaTgY3A/LAu1tkGs6C03cvQ4rjOcUhr\nHIdUm5IrKK7OuSmJROKWWo2rd55q45ybEnUdgnLOPQg82MGu4aW4XmefYa4ALgKIQ9ougSmuIjl0\neqSPmd0BfJdU2u6c+2fWi6SeDdVqJhKn51yFxBWSsa3VuKbOE69vXIpSlqGRajCr9x+VGszqja20\np5E+IiIBqcEUEQlIDaaISEC1sGqkxIz3bPHYY5M9nH760+SCn337tg4JnjhxYtav/93vfgfAtGnT\nAPjwww8BaG5uDr2uUluUYYqIBFTRb8nXr18PwNq1awEYN651BYIvv/wylLp165YccjpsWHIGqMWL\nF2fs11vy3MJ6S961a+uaZhMmTABgzpw5RZ3Tc9NNNwFw7733+mUHDx7UW3IpmDJMEZGA1GCKiARU\n0S99hgwZAsC6dcnVU+vq6vx9Yd2SH3XUUQD84he/ANrfkktpeTFdtWqVXxb2jFYzZ84Ewvudkdql\nDFNEJKCKfunj2bUrOR3jk08+6Zf9+Mc/DqVuxx9/PAAffPABAEOHDvX3vfTSS3rpk0exL31OOOEE\nAN57770wq9Wh1JydAJgZkyZN4tFHHwWSL4E6o5pjK+0pwxQRCaiin2F6/vjHPwJw1lln+WWHH344\nEP5zqS5d9H9IOfTp0weAJUuW5D12//79QOYdxqBBgzKO8Tq5A3zlK1/p8Dz9+vXL+PzAAw+wcuVK\nAJxzAWottU6tg4hIQIEyTDNrBBYBs5xzvzWzBmABcAjJGbkvTVvWUmJCcRUpTN4G08zqgPuA59OK\nbwNmO+cWmtmvgSuAcIZldMB7IfCjH/3IL+vZsycA27ZtK+rc+/Yl24OdO3cWdZ64iTquN954IwCn\nnnpq1mM+/vhjACZPngzk7vI1YsQIf3v27NkAfP3rX89bj0WLFgEwffp0v+zxxx/P+3VSm4Lcku8D\nxpC5VOUQ4JnU9mJgWLjVkjJQXEUKlDfDdM4dAA6krcgGUJd2q7YV6NvuC0P0+uuvl+zc//73vwFY\ns2ZNya5RiaKI62GHHeZvf+9738t7/DvvvAMEG0ywYsUKf9vrqP7zn/8cgIaGhqxf570I8gYuAP6L\nIK+rmYgnjLfkefuhlWKZ3a1bt4ZynrZeeOGFdmXVuDJgAIH6FzY1NQGl+RkNHDiwZOcGMvqOpv/H\n8f7775fkehJ/nW0wd5tZN+fcF8DxZN7WtVPsMrve86lly1rXY/e6pRT7DNPz17/+FYDVq1f7ZTfe\neGOtLbNbUFwhGdugP6Obb77Z354xY0bW47yuYt7sVM8++2zec3fkuOOOA+BPf/oTAGeffba/L5FI\nZI2LNxTXe7564MCBrNeIUWwlBJ3tVvQc4M21Ng5YluNYiQ/FVSSHvEMjzexMYCZwIrAf+Ai4BJgH\ndAU2Apc75/ZnvUiRQyPPPfdcoDULhNbsIewM88QTT/TLGhoaqnZoZBhxhcKGRqb/ruX6vfvb3/4G\ntO+c3lltM02Ac845J2926HWA9zrOd6QSYyulE+Slz2sk3562NTz02kjZKK4ihdNIHxGRgGIxlvzv\nf/87kNnN41e/+hUAV199NZD7tikI78XClClT/LIjjjiiqHNK58ybNy/U823alHx39f3vfz+jbOvW\nrRxzzDFZv86bSclbKkVEGaaISECxyDA96XNgel2MZs2aBcBbb71V1Lm9LMQbcgmtL5ukOmzevDnj\n8969e3Me7w3F/eUvf1myOkm8KMMUEQkoVhnm88+3zhPx6aefAnDPPfcAMGrUqKLO7T3D/Pzzz4s6\nj8THvHnzlD1KQZRhiogEpAZTRCSgWN2SdySseSx37NgBwL/+9S+/7IYbbgCge/fugG7Xq02PHj1y\n7l+7dm2ZaiJxoQxTRCSg2GaYTz/9NABnnnkmAIcemvxWOppZxhtLfNppp/llXpeh888/H2idqzH9\nGI83r2L6nIkSP23n4LzmmmtyHv+HP/yhlNWRGFKGKSISUGwzzN///vcAXHnllUBr9uc9iwQYPXo0\nAN/5zneA1qV5oXVW7WnTpgGwfft2IHP43M9+9jMAVq1aFXr9JTvv5/7iiy8C8O677xZ1Pm8GKu9u\nwpM+A3w6L/PMNQ+m1CZlmCIiAeWdDxPAzGYAg0hmpLcDqylgOdZi58PsiDeE8ZVXXgGgV69e7Y5Z\nunRpxjGvvvqqvy99O523xgskh1smEgnGjBkDwJ///OdO1bVS50wsNq5Q2HyYb7zxhr99+umn5z3e\nG/Z600035T023de+9jUArr32WgAmTJgAQO/evf1j2s64/sgjj/jb3iqVzc3Nea9VqbGV0sibYZrZ\nUKDROTcAGAXcQ+tyrIOA9SSXY5UYUVxFChfklnwlcHFqewdQh5ZjrQaKq0iBgsy4fhDYk/o4EVgK\njCznMrsd8Tqsn3LKKaGe11t2t9pFEdehQ4f6297qnGeccUbW471b6mHDku323Llzsx572WWX+dsn\nn3wyAPX19Xnr5C2vPHXqVL8syK241KbAb8nNbCzJf1gjgHVpuyJZZrecvGeh1aiYuEJpl9n1+tZ6\nfWPvv//+UM+fSCT8FU23bNkS6rmlOgV6S25mI4GpwGjn3E5Sy7GmdgdaZtd7KZBIJCr6T+/evf0/\nXiMwZswYxowZ0+lzVqpi4wqtSygX+jO58MILufDCC2lpaYnkDyQb+z59+tCnT5+qi62URpCXPj2B\nu4ALnHOfpIq1HGvMKa4ihQtySz4e+CrwlJl5ZROAh81sMsnlWOeXpnrl99lnn/nbb775Jt/85jcz\nlt6tIpHG1RvaeumllwKwYMGCUl3Kn41/+vTpftkTTzzB2Wefzb59OXtNiWQI8tLnQeDBDnZpOdYY\nU1xFCqeRPiIiAQUa6VP0RUow0qccVqxYwfDhw/1lXy+//PJOnaeaR4MUMtIny9cDmSO1rr/+egDG\njh0LtL5YysWbWwDg/fffB1rns1y4cCGQOTY8rN/Fao6ttKcMU0QkIGWYbaTPaLR69WpOO+00Jk2a\nBMBDDz3UqXNWcxZSbIYZFWWY0hnKMEVEAlKGmUcYda7mLEQZZvXGVtpThikiEpAaTBGRgNRgiogE\npAZTRCQgNZgiIgGpwRQRCags3YpERKqBMkwRkYDUYIqIBKQGU0QkIDWYIiIBqcEUEQlIDaaISECB\n1yUvhpnNAs4FWoDrnHOry3HdzjCzGcAgkj+b24HVwALgEGAzcKlzTitnobhK7Sl5hmlmg4GTnXMD\ngInAb0p9zc4ys6FAY6quo4B7gNuA2c65QcB64IoIq1gxFFepReW4JT8PeBrAObcW6GVmR5bhup2x\nErg4tb0DqAOGAM+kyhYDw8pfrYqkuErNKcct+bHAa2mft6XKdpXh2gVxzh0E9qQ+TgSWAiPTbtW2\nAn2jqFsFUlyl5pTlGWYbFT9DtZmNJfkPawSwLm1Xxdc9QhX/s1FcpVjluCXfRDLz8BxH8iF7RTKz\nkcBUYLRzbiew28y6pXYfT/L7EcVValA5GswVwEUAZvYtYJNz7rMyXLdgZtYTuAu4wDn3Sar4OWBc\nanscsCyKulUgxVVqTllmKzKzO4DvAs3AVc65f5b8op1gZpOAacDbacUTgIeBrsBG4HLn3P7y167y\nKK5SazS9m4hIQBrpIyISkBpMEZGA1GCKiASkBlNEJCA1mCIiAanBFBEJSA2miEhAajBFRAL6f4dr\nqWdt/j0BAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f3268975f60>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "6dVsDVivkLeQ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Preparing Training Dataset"
      ]
    },
    {
      "metadata": {
        "id": "hLpQg3d3kOZN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "random_seed = 23\n",
        "\n",
        "X_train, X_val, Y_train, Y_val = train_test_split(\n",
        "    x_norm, y_encoded, test_size = 0.1, random_state = random_seed)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gIr0UaDM9MTy",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Data Generator"
      ]
    },
    {
      "metadata": {
        "id": "-qfJoz__oiPA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.preprocessing import image\n",
        "#gen = image.ImageDataGenerator()\n",
        "\n",
        "gen = image.ImageDataGenerator(\n",
        "    rotation_range=10,\n",
        "    zoom_range=0.1,\n",
        "    width_shift_range=0.1,\n",
        "    height_shift_range=0.1\n",
        ")\n",
        "\n",
        "batches = gen.flow(X_train, Y_train, batch_size=64)\n",
        "val_batches = gen.flow(X_val, Y_val, batch_size=64)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IDrijhP4qE-I",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "300f7a3f-cea6-492b-c44d-61126383f148"
      },
      "cell_type": "code",
      "source": [
        "print(batches.n)"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "37800\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "StCgKgDinL1c",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Training input Dataset"
      ]
    },
    {
      "metadata": {
        "id": "x8dX15a0j0as",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras import layers\n",
        "from keras import optimizers\n",
        "from keras import activations\n",
        "from keras import models"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aY8FX8b6_zsU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Using Simple CNN"
      ]
    },
    {
      "metadata": {
        "id": "6WwzNCd7lWZ_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def create_CNN():\n",
        "    model = models.Sequential([\n",
        "        layers.core.Lambda(normalize_data, input_shape=(28, 28, 1)),\n",
        "        layers.Conv2D(32, (3, 3), activation='relu'),\n",
        "        layers.Conv2D(32, (3, 3), activation='relu'),\n",
        "        layers.MaxPooling2D(),\n",
        "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
        "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
        "        layers.MaxPooling2D(),\n",
        "        layers.core.Flatten(),\n",
        "        layers.core.Dense(512, activation='relu'),\n",
        "        layers.core.Dense(10, activation='softmax')\n",
        "    ])\n",
        "    \n",
        "    model.compile(optimizers.Adam(),\n",
        "                 loss='categorical_crossentropy',\n",
        "                 metrics=['accuracy'])\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "t5_ao1q7oRG_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = create_CNN()\n",
        "\n",
        "epochs = 3\n",
        "\n",
        "model.optimizer.lr=0.0001"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "EXsOmNOqAWaU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Using Batch Normalization"
      ]
    },
    {
      "metadata": {
        "id": "X37eX2oDAaRL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def create_CNN_batch_norm():\n",
        "    model = models.Sequential([\n",
        "        #layers.core.Lambda(normalize_data, input_shape=(28, 28, 1)),\n",
        "        layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n",
        "        layers.BatchNormalization(),\n",
        "        layers.Conv2D(32, (3, 3), activation='relu'),\n",
        "        layers.BatchNormalization(),\n",
        "        layers.Conv2D(32, (5, 5), strides=2, activation='relu'),\n",
        "        layers.BatchNormalization(),\n",
        "        #layers.MaxPooling2D(),\n",
        "        layers.Dropout(0.3),\n",
        "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
        "        layers.BatchNormalization(),\n",
        "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
        "        layers.BatchNormalization(),\n",
        "        layers.Conv2D(64, (5, 5), strides=2, activation='relu'),\n",
        "        layers.BatchNormalization(),\n",
        "        #layers.MaxPooling2D(),\n",
        "        layers.core.Flatten(),\n",
        "        layers.Dropout(0.3),\n",
        "        layers.core.Dense(512, activation='relu'),\n",
        "        layers.core.Dense(10, activation='softmax')\n",
        "    ])\n",
        "    \n",
        "    model.compile(optimizers.Adam(),\n",
        "                 loss='categorical_crossentropy',\n",
        "                 metrics=['accuracy'])\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "iYDrOcr7yKao",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = create_CNN_batch_norm()\n",
        "model.optimizer.lr=0.01"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Hxr4fuIfx79y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 748
        },
        "outputId": "bed55f69-cc27-4be3-eeb8-91bbd1752bd4"
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "lambda_7 (Lambda)            (None, 28, 28, 1)         0         \n",
            "_________________________________________________________________\n",
            "conv2d_33 (Conv2D)           (None, 26, 26, 32)        320       \n",
            "_________________________________________________________________\n",
            "batch_normalization_23 (Batc (None, 26, 26, 32)        128       \n",
            "_________________________________________________________________\n",
            "conv2d_34 (Conv2D)           (None, 24, 24, 32)        9248      \n",
            "_________________________________________________________________\n",
            "batch_normalization_24 (Batc (None, 24, 24, 32)        128       \n",
            "_________________________________________________________________\n",
            "conv2d_35 (Conv2D)           (None, 10, 10, 32)        25632     \n",
            "_________________________________________________________________\n",
            "batch_normalization_25 (Batc (None, 10, 10, 32)        128       \n",
            "_________________________________________________________________\n",
            "dropout_9 (Dropout)          (None, 10, 10, 32)        0         \n",
            "_________________________________________________________________\n",
            "conv2d_36 (Conv2D)           (None, 8, 8, 64)          18496     \n",
            "_________________________________________________________________\n",
            "batch_normalization_26 (Batc (None, 8, 8, 64)          256       \n",
            "_________________________________________________________________\n",
            "conv2d_37 (Conv2D)           (None, 6, 6, 64)          36928     \n",
            "_________________________________________________________________\n",
            "batch_normalization_27 (Batc (None, 6, 6, 64)          256       \n",
            "_________________________________________________________________\n",
            "conv2d_38 (Conv2D)           (None, 1, 1, 64)          102464    \n",
            "_________________________________________________________________\n",
            "batch_normalization_28 (Batc (None, 1, 1, 64)          256       \n",
            "_________________________________________________________________\n",
            "flatten_7 (Flatten)          (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dropout_10 (Dropout)         (None, 64)                0         \n",
            "_________________________________________________________________\n",
            "dense_13 (Dense)             (None, 512)               33280     \n",
            "_________________________________________________________________\n",
            "dense_14 (Dense)             (None, 10)                5130      \n",
            "=================================================================\n",
            "Total params: 232,650\n",
            "Trainable params: 232,074\n",
            "Non-trainable params: 576\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ozU5HJuy74Gg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras import callbacks"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "48tjxaOd8K0d",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "annealer = callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZrvyCdc1_pRY",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Using MaxPooling with CNN"
      ]
    },
    {
      "metadata": {
        "id": "ionIJ11B_uuj",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def create_CNN_max_pool_norm():\n",
        "    model = models.Sequential([\n",
        "        #layers.core.Lambda(normalize_data, input_shape=(28, 28, 1)),\n",
        "        layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n",
        "        layers.BatchNormalization(),\n",
        "        layers.Conv2D(32, (3, 3), activation='relu'),\n",
        "        layers.MaxPooling2D(),\n",
        "        layers.Dropout(0.3),\n",
        "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
        "        layers.BatchNormalization(),\n",
        "        layers.Conv2D(64, (3, 3), activation='relu'),\n",
        "        layers.MaxPooling2D(),\n",
        "        layers.Conv2D(128, (3, 3), activation='relu'),\n",
        "        layers.Dropout(0.3),\n",
        "        layers.core.Flatten(),\n",
        "        layers.BatchNormalization(),\n",
        "        layers.core.Dense(512, activation='relu'),\n",
        "        layers.core.Dense(10, activation='softmax')\n",
        "    ])\n",
        "    \n",
        "    model.compile(optimizers.Adam(),\n",
        "                 loss='categorical_crossentropy',\n",
        "                 metrics=['accuracy'])\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "RtpPZmRW_-S-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model = create_CNN_max_pool_norm()\n",
        "model.optimizer.lr = 0.01"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oGaFYK8t_-1u",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Training model"
      ]
    },
    {
      "metadata": {
        "id": "-gVHMw4eqN4T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "3ff4aee7-e2e9-4779-a809-9ea71347d0cf"
      },
      "cell_type": "code",
      "source": [
        "gen = image.ImageDataGenerator(\n",
        "    rotation_range=10,\n",
        "    shear_range=0.1,\n",
        "    zoom_range=0.2,\n",
        "    width_shift_range=0.1,\n",
        "    height_shift_range=0.1\n",
        ")\n",
        "\n",
        "batches = gen.flow(X_train, Y_train, batch_size=64)\n",
        "val_batches = gen.flow(X_val, Y_val, batch_size=64)\n",
        "\n",
        "res = model.fit_generator(generator=batches,\n",
        "                         steps_per_epoch=batches.n,\n",
        "                         epochs=1,\n",
        "                         validation_data=val_batches,\n",
        "                         validation_steps=val_batches.n)"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/1\n",
            "37800/37800 [==============================] - 1727s 46ms/step - loss: 0.0714 - acc: 0.9818 - val_loss: 0.0404 - val_acc: 0.9920\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "1CBYFTMcSn8a",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Train again with complete data set after accuracy testing on validation set"
      ]
    },
    {
      "metadata": {
        "id": "gtKEO_aYSvdA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "outputId": "1ea183d7-57ac-419d-a2d1-36cc7f4c5fdf"
      },
      "cell_type": "code",
      "source": [
        "epochs = 3\n",
        "model.optimizer.lr=0.01\n",
        "\n",
        "gen = image.ImageDataGenerator()\n",
        "\n",
        "batches = gen.flow(x_norm, y_encoded, batch_size=64)\n",
        "\n",
        "res = model.fit_generator(generator=batches,\n",
        "                         steps_per_epoch=batches.n,\n",
        "                         epochs=epochs)"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "42000/42000 [==============================] - 1304s 31ms/step - loss: 0.0358 - acc: 0.9939\n",
            "Epoch 2/3\n",
            "42000/42000 [==============================] - 1302s 31ms/step - loss: 0.0781 - acc: 0.9876\n",
            "Epoch 3/3\n",
            "30641/42000 [====================>.........] - ETA: 6:13 - loss: 0.1161 - acc: 0.9823Buffered data was truncated after reaching the output size limit."
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "0FZeevY6eddm",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "print(res)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Xt-8aPuNSwHZ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Save network weights and predict output for test data set"
      ]
    },
    {
      "metadata": {
        "id": "3cx3LSXirYf_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model.save_weights('model_batch_norm_96_5.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "TGaZ5OBgEHYG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "8714b659-8431-4eae-8ee7-0221cac78f5c"
      },
      "cell_type": "code",
      "source": [
        "!ls -l"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 976\r\n",
            "-rw-r--r-- 1 root root   2514 Sep  6 04:08 adc.json\r\n",
            "drwxr-xr-x 2 root root   4096 Sep  6 04:08 data\r\n",
            "-rw-r--r-- 1 root root 984832 Sep  6 05:07 model_batch_norm.h5\r\n",
            "drwxr-xr-x 2 root root   4096 Aug 31 20:42 sample_data\r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "XmAQqw57EIqF",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#results = model.predict(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FTk_CmSpEQyO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X_test = X_test.reshape(X_test.shape[0], 28, 28,1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "U7QjOhlnEnFI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "predictions = model.predict_classes(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wCpxMtDqe0K0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "278ea8c4-3af5-4a8e-daed-dcab1f7cead1"
      },
      "cell_type": "code",
      "source": [
        "print(predictions.shape)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(28000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "w6ln_IasErDt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "results = pd.DataFrame({'ImageId': list(range(1, len(predictions)+1)),\n",
        "    'Label': predictions\n",
        "})\n",
        "\n",
        "results.to_csv('submission.csv', index=False, header=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5hf4mZuMKw9o",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Submit results using Kaggle CLI"
      ]
    },
    {
      "metadata": {
        "id": "f-xHZAYWK1hb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "4016a6f8-ee6e-483f-d1b1-0c9f2872a9ff"
      },
      "cell_type": "code",
      "source": [
        "!kaggle competitions submit -c digit-recognizer -f submission.csv -m \"Submission with Batch Normalization CNN\""
      ],
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Successfully submitted to Digit Recognizer"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "xtTUrQE_Hfqy",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Write output results to Drive folder"
      ]
    },
    {
      "metadata": {
        "id": "4g0HXBiZFI8p",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2451
        },
        "outputId": "296c551d-f7c5-4c47-a10e-34cd96eb4c86"
      },
      "cell_type": "code",
      "source": [
        "# Install a Drive FUSE wrapper.\n",
        "# https://github.com/astrada/google-drive-ocamlfuse\n",
        "!apt-get update\n",
        "!apt-get install -y -qq software-properties-common python-software-properties module-init-tools\n",
        "!add-apt-repository -y ppa:alessandro-strada/ppa 2>&1 > /dev/null\n",
        "!apt-get update -qq 2>&1 > /dev/null\n",
        "!apt-get -y install -qq google-drive-ocamlfuse fuse\n",
        "\n",
        "!pip install -U -q PyDrive\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "# Generate creds for the Drive FUSE library.\n",
        "from oauth2client.client import GoogleCredentials\n",
        "creds = GoogleCredentials.get_application_default()\n",
        "import getpass\n",
        "!google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret} < /dev/null 2>&1 | grep URL\n",
        "vcode = getpass.getpass()\n",
        "!echo {vcode} | google-drive-ocamlfuse -headless -id={creds.client_id} -secret={creds.client_secret}"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Hit:1 http://security.ubuntu.com/ubuntu artful-security InRelease\n",
            "Hit:2 http://archive.ubuntu.com/ubuntu artful InRelease\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu artful-updates InRelease\n",
            "Hit:4 http://archive.ubuntu.com/ubuntu artful-backports InRelease\n",
            "Reading package lists... Done\n",
            "Preconfiguring packages ...\n",
            "Selecting previously unselected package cron.\n",
            "(Reading database ... 18408 files and directories currently installed.)\n",
            "Preparing to unpack .../00-cron_3.0pl1-128ubuntu5_amd64.deb ...\n",
            "Unpacking cron (3.0pl1-128ubuntu5) ...\n",
            "Selecting previously unselected package libapparmor1:amd64.\n",
            "Preparing to unpack .../01-libapparmor1_2.11.0-2ubuntu17.1_amd64.deb ...\n",
            "Unpacking libapparmor1:amd64 (2.11.0-2ubuntu17.1) ...\n",
            "Selecting previously unselected package libdbus-1-3:amd64.\n",
            "Preparing to unpack .../02-libdbus-1-3_1.10.22-1ubuntu1_amd64.deb ...\n",
            "Unpacking libdbus-1-3:amd64 (1.10.22-1ubuntu1) ...\n",
            "Selecting previously unselected package dbus.\n",
            "Preparing to unpack .../03-dbus_1.10.22-1ubuntu1_amd64.deb ...\n",
            "Unpacking dbus (1.10.22-1ubuntu1) ...\n",
            "Selecting previously unselected package dirmngr.\n",
            "Preparing to unpack .../04-dirmngr_2.1.15-1ubuntu8.1_amd64.deb ...\n",
            "Unpacking dirmngr (2.1.15-1ubuntu8.1) ...\n",
            "Selecting previously unselected package distro-info-data.\n",
            "Preparing to unpack .../05-distro-info-data_0.36ubuntu0.2_all.deb ...\n",
            "Unpacking distro-info-data (0.36ubuntu0.2) ...\n",
            "Selecting previously unselected package libkmod2:amd64.\n",
            "Preparing to unpack .../06-libkmod2_24-1ubuntu2_amd64.deb ...\n",
            "Unpacking libkmod2:amd64 (24-1ubuntu2) ...\n",
            "Selecting previously unselected package kmod.\n",
            "Preparing to unpack .../07-kmod_24-1ubuntu2_amd64.deb ...\n",
            "Unpacking kmod (24-1ubuntu2) ...\n",
            "Selecting previously unselected package lsb-release.\n",
            "Preparing to unpack .../08-lsb-release_9.20160110ubuntu5_all.deb ...\n",
            "Unpacking lsb-release (9.20160110ubuntu5) ...\n",
            "Selecting previously unselected package libgirepository-1.0-1:amd64.\n",
            "Preparing to unpack .../09-libgirepository-1.0-1_1.54.1-1_amd64.deb ...\n",
            "Unpacking libgirepository-1.0-1:amd64 (1.54.1-1) ...\n",
            "Selecting previously unselected package gir1.2-glib-2.0:amd64.\n",
            "Preparing to unpack .../10-gir1.2-glib-2.0_1.54.1-1_amd64.deb ...\n",
            "Unpacking gir1.2-glib-2.0:amd64 (1.54.1-1) ...\n",
            "Selecting previously unselected package iso-codes.\n",
            "Preparing to unpack .../11-iso-codes_3.75-1_all.deb ...\n",
            "Unpacking iso-codes (3.75-1) ...\n",
            "Selecting previously unselected package libdbus-glib-1-2:amd64.\n",
            "Preparing to unpack .../12-libdbus-glib-1-2_0.108-2_amd64.deb ...\n",
            "Unpacking libdbus-glib-1-2:amd64 (0.108-2) ...\n",
            "Selecting previously unselected package python-apt-common.\n",
            "Preparing to unpack .../13-python-apt-common_1.4.0~beta3build2_all.deb ...\n",
            "Unpacking python-apt-common (1.4.0~beta3build2) ...\n",
            "Selecting previously unselected package python3-apt.\n",
            "Preparing to unpack .../14-python3-apt_1.4.0~beta3build2_amd64.deb ...\n",
            "Unpacking python3-apt (1.4.0~beta3build2) ...\n",
            "Selecting previously unselected package python3-dbus.\n",
            "Preparing to unpack .../15-python3-dbus_1.2.4-1build3_amd64.deb ...\n",
            "Unpacking python3-dbus (1.2.4-1build3) ...\n",
            "Selecting previously unselected package python3-gi.\n",
            "Preparing to unpack .../16-python3-gi_3.24.1-2build1_amd64.deb ...\n",
            "Unpacking python3-gi (3.24.1-2build1) ...\n",
            "Selecting previously unselected package module-init-tools.\n",
            "Preparing to unpack .../17-module-init-tools_24-1ubuntu2_all.deb ...\n",
            "Unpacking module-init-tools (24-1ubuntu2) ...\n",
            "Selecting previously unselected package python-apt.\n",
            "Preparing to unpack .../18-python-apt_1.4.0~beta3build2_amd64.deb ...\n",
            "Unpacking python-apt (1.4.0~beta3build2) ...\n",
            "Selecting previously unselected package python-pycurl.\n",
            "Preparing to unpack .../19-python-pycurl_7.43.0-2build2_amd64.deb ...\n",
            "Unpacking python-pycurl (7.43.0-2build2) ...\n",
            "Selecting previously unselected package python-software-properties.\n",
            "Preparing to unpack .../20-python-software-properties_0.96.24.17_all.deb ...\n",
            "Unpacking python-software-properties (0.96.24.17) ...\n",
            "Selecting previously unselected package python3-software-properties.\n",
            "Preparing to unpack .../21-python3-software-properties_0.96.24.17_all.deb ...\n",
            "Unpacking python3-software-properties (0.96.24.17) ...\n",
            "Selecting previously unselected package software-properties-common.\n",
            "Preparing to unpack .../22-software-properties-common_0.96.24.17_all.deb ...\n",
            "Unpacking software-properties-common (0.96.24.17) ...\n",
            "Selecting previously unselected package unattended-upgrades.\n",
            "Preparing to unpack .../23-unattended-upgrades_0.98ubuntu1.1_all.deb ...\n",
            "Unpacking unattended-upgrades (0.98ubuntu1.1) ...\n",
            "Setting up python-apt-common (1.4.0~beta3build2) ...\n",
            "Setting up python3-apt (1.4.0~beta3build2) ...\n",
            "Setting up iso-codes (3.75-1) ...\n",
            "Setting up distro-info-data (0.36ubuntu0.2) ...\n",
            "Setting up python-pycurl (7.43.0-2build2) ...\n",
            "Setting up lsb-release (9.20160110ubuntu5) ...\n",
            "Setting up libgirepository-1.0-1:amd64 (1.54.1-1) ...\n",
            "Setting up libkmod2:amd64 (24-1ubuntu2) ...\n",
            "Setting up gir1.2-glib-2.0:amd64 (1.54.1-1) ...\n",
            "Processing triggers for libc-bin (2.26-0ubuntu2.1) ...\n",
            "Setting up libapparmor1:amd64 (2.11.0-2ubuntu17.1) ...\n",
            "Setting up unattended-upgrades (0.98ubuntu1.1) ...\n",
            "\n",
            "Creating config file /etc/apt/apt.conf.d/20auto-upgrades with new version\n",
            "\n",
            "Creating config file /etc/apt/apt.conf.d/50unattended-upgrades with new version\n",
            "invoke-rc.d: could not determine current runlevel\n",
            "invoke-rc.d: policy-rc.d denied execution of start.\n",
            "Setting up dirmngr (2.1.15-1ubuntu8.1) ...\n",
            "Setting up cron (3.0pl1-128ubuntu5) ...\n",
            "Adding group `crontab' (GID 102) ...\n",
            "Done.\n",
            "update-rc.d: warning: start and stop actions are no longer supported; falling back to defaults\n",
            "update-rc.d: warning: stop runlevel arguments (1) do not match cron Default-Stop values (none)\n",
            "invoke-rc.d: could not determine current runlevel\n",
            "invoke-rc.d: policy-rc.d denied execution of start.\n",
            "Setting up libdbus-1-3:amd64 (1.10.22-1ubuntu1) ...\n",
            "Setting up kmod (24-1ubuntu2) ...\n",
            "Setting up libdbus-glib-1-2:amd64 (0.108-2) ...\n",
            "Setting up python3-gi (3.24.1-2build1) ...\n",
            "Setting up module-init-tools (24-1ubuntu2) ...\n",
            "Setting up python3-software-properties (0.96.24.17) ...\n",
            "Setting up dbus (1.10.22-1ubuntu1) ...\n",
            "Setting up python-apt (1.4.0~beta3build2) ...\n",
            "Setting up python3-dbus (1.2.4-1build3) ...\n",
            "Setting up python-software-properties (0.96.24.17) ...\n",
            "Setting up software-properties-common (0.96.24.17) ...\n",
            "Processing triggers for libc-bin (2.26-0ubuntu2.1) ...\n",
            "Processing triggers for dbus (1.10.22-1ubuntu1) ...\n",
            "gpg: keybox '/tmp/tmpi05jussq/pubring.gpg' created\n",
            "gpg: /tmp/tmpi05jussq/trustdb.gpg: trustdb created\n",
            "gpg: key AD5F235DF639B041: public key \"Launchpad PPA for Alessandro Strada\" imported\n",
            "gpg: Total number processed: 1\n",
            "gpg:               imported: 1\n",
            "Warning: apt-key output should not be parsed (stdout is not a terminal)\n",
            "Selecting previously unselected package libfuse2:amd64.\n",
            "(Reading database ... 19816 files and directories currently installed.)\n",
            "Preparing to unpack .../libfuse2_2.9.7-1ubuntu1_amd64.deb ...\n",
            "Unpacking libfuse2:amd64 (2.9.7-1ubuntu1) ...\n",
            "Selecting previously unselected package fuse.\n",
            "Preparing to unpack .../fuse_2.9.7-1ubuntu1_amd64.deb ...\n",
            "Unpacking fuse (2.9.7-1ubuntu1) ...\n",
            "Selecting previously unselected package google-drive-ocamlfuse.\n",
            "Preparing to unpack .../google-drive-ocamlfuse_0.6.21-0ubuntu2_amd64.deb ...\n",
            "Unpacking google-drive-ocamlfuse (0.6.21-0ubuntu2) ...\n",
            "Setting up libfuse2:amd64 (2.9.7-1ubuntu1) ...\n",
            "Processing triggers for libc-bin (2.26-0ubuntu2.1) ...\n",
            "Setting up fuse (2.9.7-1ubuntu1) ...\n",
            "Setting up google-drive-ocamlfuse (0.6.21-0ubuntu2) ...\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "··········\n",
            "Please, open the following URL in a web browser: https://accounts.google.com/o/oauth2/auth?client_id=32555940559.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive&response_type=code&access_type=offline&approval_prompt=force\n",
            "Please enter the verification code: Access token retrieved correctly.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RtoyP6LcF0gu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!mkdir -p kaggle-drive\n",
        "!google-drive-ocamlfuse kaggle-drive"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cmYI6XKBFcSc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!cp model_batch_norm_96_5.h5 /content/kaggle-drive/kaggle_competitions/digit_recognizer\n",
        "!cp submission.csv /content/kaggle-drive/kaggle_competitions/digit_recognizer/submission_batch_norm_96_5.csv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jPPhiFSWF6TX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}