MNIST_liza_kiseleva_colab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XV-THAovnJXs"
   },
   "outputs": [],
   "source": [
    "from matrix_square_root_power import *\n",
    "from shampoo_optimizer import *\n",
    "from tensorflow.keras import datasets, layers, models\n",
    "import tensorflow as tf\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.layers import Dense, Conv2D, BatchNormalization, Activation, Dropout\n",
    "from keras.layers import AveragePooling2D, Input, Flatten, GlobalAveragePooling2D\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "from keras.models import Model, Sequential\n",
    "from keras.datasets import cifar10\n",
    "from keras import backend\n",
    "from keras.utils import np_utils\n",
    "import numpy\n",
    "from keras.datasets import cifar10\n",
    "from keras.layers import Flatten\n",
    "from keras.constraints import maxnorm\n",
    "from keras.optimizers import SGD\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "F9JoJ_9hnJX7"
   },
   "outputs": [],
   "source": [
    "from keras.datasets import mnist\n",
    "#download mnist data and split into train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2K7Twx8JnJYo"
   },
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(60000,28,28,1)\n",
    "X_test = X_test.reshape(10000,28,28,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jsEFBsSrnJY0"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Conv2D, Flatten\n",
    "#create model\n",
    "model = Sequential()\n",
    "#add model layers\n",
    "model.add(Conv2D(64, kernel_size=3, activation=\"relu\", input_shape=(28,28,1)))\n",
    "model.add(Conv2D(32, kernel_size=3, activation=\"relu\"))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tTjoKP2VnJY-"
   },
   "outputs": [],
   "source": [
    "X_train = tf.convert_to_tensor(X_train, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bpEBC-Q8nJZs"
   },
   "outputs": [],
   "source": [
    "def loss(model, x, y, loss_object):\n",
    "    y_ = model(x)\n",
    "\n",
    "    return loss_object(y_true=y, y_pred=y_)\n",
    "\n",
    "def grad(model, inputs, targets, loss_object):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss_value = loss(model, inputs, targets, loss_object)\n",
    "        return loss_value, tape.gradient(loss_value, model.trainable_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 353
    },
    "colab_type": "code",
    "id": "Lu2xKde8nJZ4",
    "outputId": "c99be6fd-b4cb-48dd-e6b5-d905c2379019"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 000: Loss: 1.826, Accuracy: 65.042%\n",
      "Epoch 001: Loss: 1.703, Accuracy: 76.833%\n",
      "Epoch 002: Loss: 1.692, Accuracy: 77.571%\n",
      "Epoch 003: Loss: 1.686, Accuracy: 77.995%\n",
      "Epoch 004: Loss: 1.682, Accuracy: 78.349%\n",
      "Epoch 005: Loss: 1.679, Accuracy: 78.617%\n",
      "Epoch 006: Loss: 1.676, Accuracy: 78.829%\n",
      "Epoch 007: Loss: 1.674, Accuracy: 78.971%\n",
      "Epoch 008: Loss: 1.672, Accuracy: 79.132%\n",
      "Epoch 009: Loss: 1.671, Accuracy: 79.235%\n",
      "Epoch 010: Loss: 1.669, Accuracy: 79.372%\n",
      "Epoch 011: Loss: 1.662, Accuracy: 80.162%\n",
      "Epoch 012: Loss: 1.587, Accuracy: 88.014%\n",
      "Epoch 013: Loss: 1.579, Accuracy: 88.674%\n",
      "Epoch 014: Loss: 1.576, Accuracy: 88.879%\n",
      "Epoch 015: Loss: 1.573, Accuracy: 89.071%\n",
      "Epoch 016: Loss: 1.572, Accuracy: 89.208%\n",
      "Epoch 017: Loss: 1.570, Accuracy: 89.293%\n",
      "Epoch 018: Loss: 1.569, Accuracy: 89.378%\n",
      "Epoch 019: Loss: 1.568, Accuracy: 89.456%\n"
     ]
    }
   ],
   "source": [
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.001)\n",
    "# optimizer = ShampooOptimizer(learning_rate=0.0001)\n",
    "\n",
    "adam_train_loss_results = []\n",
    "adam_train_accuracy_results = []\n",
    "\n",
    "epochs = 20\n",
    "batch_size = 128\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    epoch_loss_avg = tf.keras.metrics.Mean()\n",
    "    epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n",
    "\n",
    "    # Training loop - using batches of 128\n",
    "    for i in range(X_train.shape[0] // batch_size):\n",
    "        x = X_train[i*batch_size:i*batch_size+batch_size]\n",
    "        y = y_train[i*batch_size:i*batch_size+batch_size]\n",
    "        # Optimize the model\n",
    "        loss_value, grads = grad(model, x, y, loss_object)\n",
    "        optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
    "\n",
    "        # Track progress\n",
    "        epoch_loss_avg(loss_value)  # Add current batch loss\n",
    "        # Compare predicted label to actual label\n",
    "        epoch_accuracy(y, model(x))\n",
    "\n",
    "        # End epoch\n",
    "    adam_train_loss_results.append(epoch_loss_avg.result())\n",
    "    adam_train_accuracy_results.append(epoch_accuracy.result())\n",
    "\n",
    "    print(\"Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}\".format(epoch,\n",
    "                                                                epoch_loss_avg.result(),\n",
    "                                                                epoch_accuracy.result()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 134
    },
    "colab_type": "code",
    "id": "4qovxVkNn_0m",
    "outputId": "68781129-85fc-4d6a-f1dc-31c11dc4c27c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MNIST, Adagrad, lr = 10e-4, loss = [1.8257794 1.7025895 1.6923213 1.686369  1.6823964 1.6788954 1.676232\n",
      " 1.6741681 1.672348  1.670665  1.6692    1.6620312 1.5872949 1.5789883\n",
      " 1.5756624 1.5734642 1.5718085 1.5703863 1.5692716 1.5683758]\n",
      "MNIST, Adagrad, lr = 10e-4, loss = [0.650424   0.7683293  0.7757078  0.77994794 0.7834869  0.78617454\n",
      " 0.7882946  0.78971356 0.7913161  0.79235107 0.79371995 0.8016159\n",
      " 0.88014156 0.88673544 0.8887887  0.89070845 0.8920773  0.89292866\n",
      " 0.89378005 0.8945646 ]\n"
     ]
    }
   ],
   "source": [
    "print(\"MNIST, Adagrad, lr = 10e-4, loss = {0}\".format(np.array(adam_train_loss_results)))\n",
    "print(\"MNIST, Adagrad, lr = 10e-4, loss = {0}\".format(np.array(adam_train_accuracy_results)))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "MNIST.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}