usage_tf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 17134,
     "status": "ok",
     "timestamp": 1651027155476,
     "user": {
      "displayName": "김형진",
      "userId": "05530574334109307920"
     },
     "user_tz": -540
    },
    "id": "TAtrzndb7Vun",
    "outputId": "2a73483e-7802-4aa8-fc67-c416eb3facb4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
      "170500096/170498071 [==============================] - 5s 0us/step\n",
      "170508288/170498071 [==============================] - 5s 0us/step\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "import ssl\n",
    "ssl._create_default_https_context = ssl._create_unverified_context\n",
    "\n",
    "def pipe(data, batch_size = 128, shuffle = False):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices(data)\n",
    "    if shuffle:\n",
    "        dataset = dataset.shuffle(buffer_size = batch_size * 10)\n",
    "    dataset = dataset.batch(batch_size)\n",
    "    #dataset = dataset.prefetch((batch_size * 2) + 1)\n",
    "    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "    return dataset\n",
    "\n",
    "(tr_x, tr_y), (te_x, te_y) = tf.keras.datasets.cifar10.load_data()\n",
    "\n",
    "tr_x = tr_x * 1/255\n",
    "te_x = te_x * 1/255\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "tr_data = pipe((tr_x, tr_y), batch_size = batch_size, shuffle = True)\n",
    "te_data = pipe((te_x, te_y), batch_size = batch_size, shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1756289,
     "status": "ok",
     "timestamp": 1651028911758,
     "user": {
      "displayName": "김형진",
      "userId": "05530574334109307920"
     },
     "user_tz": -540
    },
    "id": "8voh8tnX-sDU",
    "outputId": "cc064269-52e3-4b1a-c4c1-c4110fd86e50"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 - model learn\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/resnet/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
      "94773248/94765736 [==============================] - 0s 0us/step\n",
      "94781440/94765736 [==============================] - 0s 0us/step\n",
      "79/79 [==============================] - 3s 25ms/step - loss: 0.9806 - sparse_categorical_accuracy: 0.7928\n",
      "2 - model learn\n",
      "79/79 [==============================] - 2s 19ms/step - loss: 0.8892 - sparse_categorical_accuracy: 0.7996\n",
      "3 - model learn\n",
      "79/79 [==============================] - 3s 19ms/step - loss: 0.9653 - sparse_categorical_accuracy: 0.7863\n",
      "4 - model learn\n",
      "79/79 [==============================] - 2s 19ms/step - loss: 0.9529 - sparse_categorical_accuracy: 0.7896\n",
      "5 - model learn\n",
      "79/79 [==============================] - 2s 19ms/step - loss: 0.9828 - sparse_categorical_accuracy: 0.7956\n",
      "finish\n"
     ]
    }
   ],
   "source": [
    "def get_model(shape = (32, 32, 3), weights = \"imagenet\"):\n",
    "    model = tf.keras.applications.ResNet50(input_shape = shape, include_top = False, weights = weights)\n",
    "    flatten = tf.keras.layers.GlobalAveragePooling2D()(model.output)\n",
    "    drop_out = tf.keras.layers.Dropout(0.5)(flatten)\n",
    "    dense = tf.keras.layers.Dense(2048, activation = \"relu\")(drop_out)\n",
    "    prediction = tf.keras.layers.Dense(10, activation = \"softmax\", name = \"prediction\")(dense)\n",
    "    model = tf.keras.Model(model.input, prediction)\n",
    "    return model\n",
    "\n",
    "result = {}\n",
    "for index in range(5):\n",
    "    print(\"{0} - model learn\".format(index + 1))\n",
    "    model = get_model()\n",
    "    loss = tf.keras.losses.sparse_categorical_crossentropy\n",
    "    opt = tf.keras.optimizers.Adam(1e-4)\n",
    "    metric = [tf.keras.metrics.sparse_categorical_accuracy]\n",
    "    model.compile(loss = loss, optimizer = opt, metrics = metric)\n",
    "    model.fit(tr_data, epochs = 10, verbose = 0)\n",
    "\n",
    "    loss, score = model.evaluate(te_data, verbose = 1)\n",
    "    path = \"model{0}.h5\".format(index + 1)\n",
    "\n",
    "    result[path] = score\n",
    "    model.save_weights(path)\n",
    "print(\"finish\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 110166,
     "status": "ok",
     "timestamp": 1651030130612,
     "user": {
      "displayName": "김형진",
      "userId": "05530574334109307920"
     },
     "user_tz": -540
    },
    "id": "MsTu5aaRHY9d",
    "outputId": "ea1a530c-d696-475c-fc9b-9e166fcc1dff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Original Performance]\n",
      "[model1.h5] accuracy:0.7928\n",
      "[model2.h5] accuracy:0.7996\n",
      "[model3.h5] accuracy:0.7863\n",
      "[model4.h5] accuracy:0.7896\n",
      "[model5.h5] accuracy:0.7956\n",
      "\n",
      "[Greedy Soup (uniform weight update) Performance]\n",
      "[model1.h5] step: 79 - time: 7.37s - sparse_categorical_accuracy: 0.7928\n",
      "[model2.h5] step: 79 - time: 7.77s - sparse_categorical_accuracy: 0.8137\n",
      "[model3.h5] step: 79 - time: 7.43s - sparse_categorical_accuracy: 0.8167\n",
      "[model4.h5] step: 79 - time: 7.85s - sparse_categorical_accuracy: 0.8154\n",
      "[model5.h5] step: 79 - time: 7.47s - sparse_categorical_accuracy: 0.8139\n",
      "model soup best score : 0.8167\n",
      "79/79 [==============================] - 2s 20ms/step - loss: 0.8823 - sparse_categorical_accuracy: 0.8167\n",
      "\n",
      "[Greedy Soup (greedy weight update) Performance]\n",
      "[model1.h5] step: 79 - time: 7.34s - sparse_categorical_accuracy: 0.7928\n",
      "[model2.h5] step: 79 - time: 7.37s - sparse_categorical_accuracy: 0.8137\n",
      "[model3.h5] step: 79 - time: 7.72s - sparse_categorical_accuracy: 0.8177\n",
      "[model4.h5] step: 79 - time: 8.09s - sparse_categorical_accuracy: 0.8114\n",
      "[model5.h5] step: 79 - time: 7.49s - sparse_categorical_accuracy: 0.8158\n",
      "model soup best score : 0.8177\n",
      "79/79 [==============================] - 2s 20ms/step - loss: 0.7608 - sparse_categorical_accuracy: 0.8177\n",
      "\n",
      "[Uniform Soup Performance]\n",
      "79/79 [==============================] - 2s 20ms/step - loss: 1.3767 - sparse_categorical_accuracy: 0.8122\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import model_soup\n",
    "\n",
    "print(\"[Original Performance]\")\n",
    "for k, v in result.items():\n",
    "    print(\"[{0}] accuracy:{1:.4f}\".format(os.path.basename(k), v))\n",
    "\n",
    "print(\"\\n[Greedy Soup (uniform weight update) Performance]\") #original paper style\n",
    "greedy_model = model_soup.tf.greedy_soup(model, list(result.keys()), te_data, metric = tf.keras.metrics.sparse_categorical_accuracy, compare = np.greater_equal, update_greedy = False)\n",
    "loss, score = greedy_model.evaluate(te_data)\n",
    "\n",
    "print(\"\\n[Greedy Soup (greedy weight update) Performance]\")\n",
    "greedy_model = model_soup.tf.greedy_soup(model, list(result.keys()), te_data, metric = tf.keras.metrics.sparse_categorical_accuracy, compare = np.greater_equal, update_greedy = True)\n",
    "loss, score = greedy_model.evaluate(te_data)\n",
    "\n",
    "print(\"\\n[Uniform Soup Performance]\")\n",
    "uniform_model = model_soup.tf.uniform_soup(model, list(result.keys()))\n",
    "loss, score = uniform_model.evaluate(te_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "L7DeWl0h7Qkq"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "usage_tf.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}