CIFAR10_liza_kiseleva_colab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wN_s-OjEe6zd"
   },
   "outputs": [],
   "source": [
    "from matrix_square_root_power import *\n",
    "from shampoo_optimizer import *\n",
    "from tensorflow.keras import datasets, layers, models\n",
    "import tensorflow as tf\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.layers import Dense, Conv2D, BatchNormalization, Activation, Dropout\n",
    "from keras.layers import AveragePooling2D, Input, Flatten, GlobalAveragePooling2D\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "from keras.models import Model, Sequential\n",
    "from keras.datasets import cifar10\n",
    "from keras import backend\n",
    "from keras.utils import np_utils\n",
    "import numpy\n",
    "from keras.datasets import cifar10\n",
    "from keras.layers import Flatten\n",
    "from keras.constraints import maxnorm\n",
    "from keras.optimizers import SGD\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ycjGLueCe6zs"
   },
   "outputs": [],
   "source": [
    "(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NgmbhUpNe6z6"
   },
   "outputs": [],
   "source": [
    "train_images = train_images.astype('float32')\n",
    "test_images = test_images.astype('float32')\n",
    "train_images = train_images / 255.0\n",
    "test_images = test_images / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cXeGmPN6e60J"
   },
   "outputs": [],
   "source": [
    "def loss(model, x, y, loss_object):\n",
    "    y_ = model(x)\n",
    "\n",
    "    return loss_object(y_true=y, y_pred=y_)\n",
    "\n",
    "def grad(model, inputs, targets, loss_object):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss_value = loss(model, inputs, targets, loss_object)\n",
    "        return loss_value, tape.gradient(loss_value, model.trainable_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0aUdWPz7fjo0"
   },
   "outputs": [],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(64, activation='relu'))\n",
    "model.add(layers.Dense(10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zm7NHUjpe60W"
   },
   "outputs": [],
   "source": [
    "train_images = tf.convert_to_tensor(train_images, dtype=tf.float32)\n",
    "test_images = tf.convert_to_tensor(test_images, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adam, lr = 0.0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 857
    },
    "colab_type": "code",
    "id": "D6d2X-6ze60f",
    "outputId": "ec717aa3-31a9-41e3-b759-3a0cc89566fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 000: Loss: 2.223, Accuracy: 23.586%\n",
      "Epoch 001: Loss: 2.141, Accuracy: 32.258%\n",
      "Epoch 002: Loss: 2.105, Accuracy: 36.086%\n",
      "Epoch 003: Loss: 2.078, Accuracy: 39.083%\n",
      "Epoch 004: Loss: 2.057, Accuracy: 41.220%\n",
      "Epoch 005: Loss: 2.042, Accuracy: 42.833%\n",
      "Epoch 006: Loss: 2.031, Accuracy: 43.788%\n",
      "Epoch 007: Loss: 2.022, Accuracy: 44.677%\n",
      "Epoch 008: Loss: 2.014, Accuracy: 45.425%\n",
      "Epoch 009: Loss: 2.004, Accuracy: 46.687%\n",
      "Epoch 010: Loss: 1.993, Accuracy: 47.738%\n",
      "Epoch 011: Loss: 1.986, Accuracy: 48.622%\n",
      "Epoch 012: Loss: 1.979, Accuracy: 49.237%\n",
      "Epoch 013: Loss: 1.973, Accuracy: 49.766%\n",
      "Epoch 014: Loss: 1.968, Accuracy: 50.246%\n",
      "Epoch 015: Loss: 1.964, Accuracy: 50.709%\n",
      "Epoch 016: Loss: 1.959, Accuracy: 51.192%\n",
      "Epoch 017: Loss: 1.955, Accuracy: 51.625%\n",
      "Epoch 018: Loss: 1.950, Accuracy: 52.025%\n",
      "Epoch 019: Loss: 1.946, Accuracy: 52.396%\n",
      "Epoch 020: Loss: 1.943, Accuracy: 52.794%\n",
      "Epoch 021: Loss: 1.939, Accuracy: 53.235%\n",
      "Epoch 022: Loss: 1.935, Accuracy: 53.632%\n",
      "Epoch 023: Loss: 1.932, Accuracy: 53.938%\n",
      "Epoch 024: Loss: 1.929, Accuracy: 54.385%\n",
      "Epoch 025: Loss: 1.926, Accuracy: 54.694%\n",
      "Epoch 026: Loss: 1.922, Accuracy: 54.978%\n",
      "Epoch 027: Loss: 1.920, Accuracy: 55.266%\n",
      "Epoch 028: Loss: 1.917, Accuracy: 55.545%\n",
      "Epoch 029: Loss: 1.911, Accuracy: 56.080%\n",
      "Epoch 030: Loss: 1.890, Accuracy: 58.438%\n",
      "Epoch 031: Loss: 1.883, Accuracy: 59.004%\n",
      "Epoch 032: Loss: 1.879, Accuracy: 59.503%\n",
      "Epoch 033: Loss: 1.875, Accuracy: 59.954%\n",
      "Epoch 034: Loss: 1.871, Accuracy: 60.395%\n",
      "Epoch 035: Loss: 1.868, Accuracy: 60.827%\n",
      "Epoch 036: Loss: 1.864, Accuracy: 61.114%\n",
      "Epoch 037: Loss: 1.861, Accuracy: 61.436%\n",
      "Epoch 038: Loss: 1.859, Accuracy: 61.713%\n",
      "Epoch 039: Loss: 1.856, Accuracy: 62.023%\n",
      "Epoch 040: Loss: 1.854, Accuracy: 62.220%\n",
      "Epoch 041: Loss: 1.851, Accuracy: 62.382%\n",
      "Epoch 042: Loss: 1.849, Accuracy: 62.594%\n",
      "Epoch 043: Loss: 1.847, Accuracy: 62.883%\n",
      "Epoch 044: Loss: 1.844, Accuracy: 63.071%\n",
      "Epoch 045: Loss: 1.842, Accuracy: 63.403%\n",
      "Epoch 046: Loss: 1.840, Accuracy: 63.578%\n",
      "Epoch 047: Loss: 1.839, Accuracy: 63.682%\n",
      "Epoch 048: Loss: 1.837, Accuracy: 63.842%\n",
      "Epoch 049: Loss: 1.835, Accuracy: 64.056%\n"
     ]
    }
   ],
   "source": [
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
    "# optimizer = ShampooOptimizer(learning_rate=0.0001)\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)\n",
    "\n",
    "shampoo_train_loss_results = []\n",
    "shampoo_train_accuracy_results = []\n",
    "\n",
    "epochs = 50\n",
    "batch_size = 128\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    epoch_loss_avg = tf.keras.metrics.Mean()\n",
    "    epoch_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()\n",
    "\n",
    "    # Training loop - using batches of 128\n",
    "    for i in range(train_images.shape[0] // batch_size):\n",
    "        x = train_images[i*batch_size:i*batch_size+batch_size]\n",
    "        y = train_labels[i*batch_size:i*batch_size+batch_size]\n",
    "        # Optimize the model\n",
    "        loss_value, grads = grad(model, x, y, loss_object)\n",
    "        optimizer.apply_gradients(zip(grads, model.trainable_weights))\n",
    "\n",
    "        # Track progress\n",
    "        epoch_loss_avg(loss_value)  # Add current batch loss\n",
    "        # Compare predicted label to actual label\n",
    "        epoch_accuracy(y, model(x))\n",
    "\n",
    "        # End epoch\n",
    "    shampoo_train_loss_results.append(epoch_loss_avg.result())\n",
    "    shampoo_train_accuracy_results.append(epoch_accuracy.result())\n",
    "\n",
    "    print(\"Epoch {:03d}: Loss: {:.3f}, Accuracy: {:.3%}\".format(epoch,\n",
    "                                                                epoch_loss_avg.result(),\n",
    "                                                                epoch_accuracy.result()))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "CIFAR10.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}