NAS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, json\n",
    "from scipy import stats\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create models from search spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('random_arcs_mobilenetv2.pickle', 'rb') as infile:\n",
    "    obj = pickle.load(infile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1, 16, 4, 1],\n",
       " [6, 32, 1, 2],\n",
       " [6, 64, 4, 2],\n",
       " [6, 128, 4, 2],\n",
       " [6, 184, 2, 1],\n",
       " [6, 320, 1, 2],\n",
       " [6, 440, 2, 1],\n",
       " [0, 536, 2, 0]]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# architecture config\n",
    "\n",
    "arc = obj[0]\n",
    "\n",
    "arc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mobilenetv2 import MobileNetV2\n",
    "\n",
    "model_s = MobileNetV2(num_classes = 100, first_stride = 1,\\\n",
    "                        inverted_residual_setting = arc[:-1], last_channel = arc[-1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('random_arcs_shufflenetv2.pickle', 'rb') as infile:\n",
    "    obj = pickle.load(infile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'out_channels': [98, 158, 510, 524], 'num_blocks': [6, 3, 5]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# architecture config\n",
    "\n",
    "arc = obj[0]\n",
    "\n",
    "arc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ShuffleNetv2 import ShuffleV2\n",
    "\n",
    "model_s = ShuffleV2(num_classes=100, config = arc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "benchmark = pickle.load(open('benchmark.pickle', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['cifar100', 'imagenet'])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# datasets\n",
    "\n",
    "benchmark.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['mobilenet', 'shufflenet'])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# search spaces\n",
    "\n",
    "benchmark['cifar100'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['attention', 'nokd', 'correlation', 'crd', 'hint', 'kd', 'nst', 'pkt', 'similarity', 'vid'])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# KD methods\n",
    "\n",
    "benchmark['cifar100']['mobilenet'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([27, 9, 3])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# parts of train dataset\n",
    "\n",
    "benchmark['cifar100']['mobilenet']['nst'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'test_acc': 0.41689998626708985,\n",
       "  'valid_acc': 0.4229999923706055,\n",
       "  'wall_time': 286.5539231300354},\n",
       " 1: {'test_acc': 0.4271999740600586,\n",
       "  'valid_acc': 0.4212999725341797,\n",
       "  'wall_time': 264.8843615055084},\n",
       " 2: {'test_acc': 0.43200000762939456,\n",
       "  'valid_acc': 0.4318000030517578,\n",
       "  'wall_time': 357.8804075717926},\n",
       " 3: {'test_acc': 0.4515999984741211,\n",
       "  'valid_acc': 0.45219997406005863,\n",
       "  'wall_time': 298.00210762023926},\n",
       " 4: {'test_acc': 0.4225,\n",
       "  'valid_acc': 0.4239999771118164,\n",
       "  'wall_time': 318.9312756061554},\n",
       " 5: {'test_acc': 0.4555999755859375,\n",
       "  'valid_acc': 0.45099998474121095,\n",
       "  'wall_time': 435.14361596107483},\n",
       " 6: {'test_acc': 0.455099983215332,\n",
       "  'valid_acc': 0.4520999908447266,\n",
       "  'wall_time': 305.1986322402954},\n",
       " 7: {'test_acc': 0.42779998779296874,\n",
       "  'valid_acc': 0.42619998931884767,\n",
       "  'wall_time': 297.0506205558777},\n",
       " 8: {'test_acc': 0.4562999725341797,\n",
       "  'valid_acc': 0.4544999694824219,\n",
       "  'wall_time': 398.249636888504},\n",
       " 9: {'test_acc': 0.3755999755859375,\n",
       "  'valid_acc': 0.36369998931884767,\n",
       "  'wall_time': 321.2383692264557},\n",
       " 10: {'test_acc': 0.41919998168945316,\n",
       "  'valid_acc': 0.41250000000000003,\n",
       "  'wall_time': 327.2572214603424},\n",
       " 11: {'test_acc': 0.4353999710083008,\n",
       "  'valid_acc': 0.43329998016357424,\n",
       "  'wall_time': 317.9316620826721},\n",
       " 12: {'test_acc': 0.43599998474121093,\n",
       "  'valid_acc': 0.42700000762939455,\n",
       "  'wall_time': 266.13237380981445},\n",
       " 13: {'test_acc': 0.3955999755859375,\n",
       "  'valid_acc': 0.3880999755859375,\n",
       "  'wall_time': 250.14973187446594},\n",
       " 14: {'test_acc': 0.48529998779296873,\n",
       "  'valid_acc': 0.47619998931884766,\n",
       "  'wall_time': 612.7750191688538},\n",
       " 15: {'test_acc': 0.43700000762939456,\n",
       "  'valid_acc': 0.43340000152587893,\n",
       "  'wall_time': 275.16892075538635},\n",
       " 16: {'test_acc': 0.4291999816894531,\n",
       "  'valid_acc': 0.4243000030517578,\n",
       "  'wall_time': 384.6304929256439},\n",
       " 17: {'test_acc': 0.4341999816894531,\n",
       "  'valid_acc': 0.4234999847412109,\n",
       "  'wall_time': 327.01022934913635},\n",
       " 18: {'test_acc': 0.44209999084472656,\n",
       "  'valid_acc': 0.44270000457763675,\n",
       "  'wall_time': 299.90756249427795},\n",
       " 19: {'test_acc': 0.41669998168945316,\n",
       "  'valid_acc': 0.4071999740600586,\n",
       "  'wall_time': 279.0899889469147},\n",
       " 20: {'test_acc': 0.41959999084472654,\n",
       "  'valid_acc': 0.4087999725341797,\n",
       "  'wall_time': 287.7612216472626},\n",
       " 21: {'test_acc': 0.43279998779296874,\n",
       "  'valid_acc': 0.42829998016357423,\n",
       "  'wall_time': 316.68421173095703},\n",
       " 22: {'test_acc': 0.39439998626708983,\n",
       "  'valid_acc': 0.38020000457763675,\n",
       "  'wall_time': 291.39700412750244},\n",
       " 23: {'test_acc': 0.44610000610351563,\n",
       "  'valid_acc': 0.44149997711181643,\n",
       "  'wall_time': 447.0146083831787},\n",
       " 24: {'test_acc': 0.3989999771118164,\n",
       "  'valid_acc': 0.3880999755859375,\n",
       "  'wall_time': 259.69505047798157},\n",
       " 25: {'test_acc': 0.3927000045776367,\n",
       "  'valid_acc': 0.3870000076293945,\n",
       "  'wall_time': 322.6545536518097},\n",
       " 26: {'test_acc': 0.45939998626708983,\n",
       "  'valid_acc': 0.4512999725341797,\n",
       "  'wall_time': 324.0005798339844},\n",
       " 27: {'test_acc': 0.42379997253417967,\n",
       "  'valid_acc': 0.4184000015258789,\n",
       "  'wall_time': 331.0129210948944},\n",
       " 28: {'test_acc': 0.4127999877929688,\n",
       "  'valid_acc': 0.4102999877929688,\n",
       "  'wall_time': 300.27982997894287},\n",
       " 29: {'test_acc': 0.3909000015258789,\n",
       "  'valid_acc': 0.3868000030517578,\n",
       "  'wall_time': 318.5951828956604},\n",
       " 30: {'test_acc': 0.44779998779296876,\n",
       "  'valid_acc': 0.43959999084472656,\n",
       "  'wall_time': 461.8324279785156},\n",
       " 31: {'test_acc': 0.3718000030517578,\n",
       "  'valid_acc': 0.37270000457763675,\n",
       "  'wall_time': 352.83617186546326},\n",
       " 32: {'test_acc': 0.4447999954223633,\n",
       "  'valid_acc': 0.42709999084472655,\n",
       "  'wall_time': 408.3046872615814},\n",
       " 33: {'test_acc': 0.46439998626708984,\n",
       "  'valid_acc': 0.46680000305175784,\n",
       "  'wall_time': 369.01591634750366},\n",
       " 34: {'test_acc': 0.41110000610351566,\n",
       "  'valid_acc': 0.3975,\n",
       "  'wall_time': 305.97215723991394},\n",
       " 35: {'test_acc': 0.4234999847412109,\n",
       "  'valid_acc': 0.4159000015258789,\n",
       "  'wall_time': 312.526474237442},\n",
       " 36: {'test_acc': 0.4331999969482422,\n",
       "  'valid_acc': 0.4272999954223633,\n",
       "  'wall_time': 364.40617752075195},\n",
       " 37: {'test_acc': 0.4065999984741211,\n",
       "  'valid_acc': 0.40419998168945315,\n",
       "  'wall_time': 255.63009929656982},\n",
       " 38: {'test_acc': 0.4156999969482422,\n",
       "  'valid_acc': 0.4072999954223633,\n",
       "  'wall_time': 314.8539664745331},\n",
       " 39: {'test_acc': 0.4602000045776367,\n",
       "  'valid_acc': 0.4562999725341797,\n",
       "  'wall_time': 349.29395484924316},\n",
       " 40: {'test_acc': 0.44860000610351564,\n",
       "  'valid_acc': 0.4495000076293945,\n",
       "  'wall_time': 355.44460797309875},\n",
       " 41: {'test_acc': 0.4243000030517578,\n",
       "  'valid_acc': 0.42,\n",
       "  'wall_time': 304.46356892585754},\n",
       " 42: {'test_acc': 0.39869998931884765,\n",
       "  'valid_acc': 0.3995000076293945,\n",
       "  'wall_time': 286.5955979824066},\n",
       " 43: {'test_acc': 0.4559000015258789,\n",
       "  'valid_acc': 0.4473999786376953,\n",
       "  'wall_time': 349.3416292667389},\n",
       " 44: {'test_acc': 0.40049999237060546,\n",
       "  'valid_acc': 0.3990999984741211,\n",
       "  'wall_time': 281.53082728385925},\n",
       " 45: {'test_acc': 0.4322999954223633,\n",
       "  'valid_acc': 0.4303999710083008,\n",
       "  'wall_time': 281.6090016365051},\n",
       " 46: {'test_acc': 0.4564999771118164,\n",
       "  'valid_acc': 0.4525,\n",
       "  'wall_time': 487.39508628845215},\n",
       " 47: {'test_acc': 0.4172999954223633,\n",
       "  'valid_acc': 0.4145999908447266,\n",
       "  'wall_time': 263.63876008987427},\n",
       " 48: {'test_acc': 0.4157999801635742,\n",
       "  'valid_acc': 0.4082999801635742,\n",
       "  'wall_time': 318.87702894210815},\n",
       " 49: {'test_acc': 0.4437999725341797,\n",
       "  'valid_acc': 0.43689998626708987,\n",
       "  'wall_time': 386.06458258628845},\n",
       " 50: {'test_acc': 0.4321999740600586,\n",
       "  'valid_acc': 0.4305999755859375,\n",
       "  'wall_time': 348.5429365634918},\n",
       " 51: {'test_acc': 0.4084000015258789,\n",
       "  'valid_acc': 0.3981999969482422,\n",
       "  'wall_time': 302.89235377311707},\n",
       " 52: {'test_acc': 0.4134000015258789,\n",
       "  'valid_acc': 0.40669998168945315,\n",
       "  'wall_time': 259.74769949913025},\n",
       " 53: {'test_acc': 0.4834000015258789,\n",
       "  'valid_acc': 0.4812999725341797,\n",
       "  'wall_time': 655.6030640602112},\n",
       " 54: {'test_acc': 0.41259998321533203,\n",
       "  'valid_acc': 0.40619998931884765,\n",
       "  'wall_time': 296.1298038959503},\n",
       " 55: {'test_acc': 0.4187999725341797,\n",
       "  'valid_acc': 0.4115999984741211,\n",
       "  'wall_time': 309.9375102519989},\n",
       " 56: {'test_acc': 0.4429999923706055,\n",
       "  'valid_acc': 0.4363999938964844,\n",
       "  'wall_time': 290.14500761032104},\n",
       " 57: {'test_acc': 0.4339999771118164,\n",
       "  'valid_acc': 0.43329998016357424,\n",
       "  'wall_time': 378.72653913497925},\n",
       " 58: {'test_acc': 0.42759998321533205,\n",
       "  'valid_acc': 0.4303999710083008,\n",
       "  'wall_time': 284.0435073375702},\n",
       " 59: {'test_acc': 0.4387999725341797,\n",
       "  'valid_acc': 0.4354999923706055,\n",
       "  'wall_time': 319.506023645401},\n",
       " 60: {'test_acc': 0.4254999923706055,\n",
       "  'valid_acc': 0.41450000762939454,\n",
       "  'wall_time': 296.7007715702057},\n",
       " 61: {'test_acc': 0.4306999969482422,\n",
       "  'valid_acc': 0.4330999755859375,\n",
       "  'wall_time': 318.1882288455963},\n",
       " 62: {'test_acc': 0.4353999710083008,\n",
       "  'valid_acc': 0.4372999954223633,\n",
       "  'wall_time': 317.12397146224976},\n",
       " 63: {'test_acc': 0.4368000030517578,\n",
       "  'valid_acc': 0.4246999740600586,\n",
       "  'wall_time': 308.97561836242676},\n",
       " 64: {'test_acc': 0.43439998626708987,\n",
       "  'valid_acc': 0.43959999084472656,\n",
       "  'wall_time': 468.35458040237427},\n",
       " 65: {'test_acc': 0.4230999755859375,\n",
       "  'valid_acc': 0.4145999908447266,\n",
       "  'wall_time': 302.547908782959},\n",
       " 66: {'test_acc': 0.45680000305175783,\n",
       "  'valid_acc': 0.45860000610351564,\n",
       "  'wall_time': 505.9682614803314},\n",
       " 67: {'test_acc': 0.445,\n",
       "  'valid_acc': 0.4455999755859375,\n",
       "  'wall_time': 301.4510908126831},\n",
       " 68: {'test_acc': 0.4597999954223633,\n",
       "  'valid_acc': 0.45789997100830077,\n",
       "  'wall_time': 400.9382264614105},\n",
       " 69: {'test_acc': 0.4663999938964844,\n",
       "  'valid_acc': 0.45989997863769533,\n",
       "  'wall_time': 440.7160303592682},\n",
       " 70: {'test_acc': 0.4495000076293945,\n",
       "  'valid_acc': 0.44919998168945313,\n",
       "  'wall_time': 325.81901383399963},\n",
       " 71: {'test_acc': 0.37950000762939456,\n",
       "  'valid_acc': 0.37840000152587894,\n",
       "  'wall_time': 283.3300428390503},\n",
       " 72: {'test_acc': 0.43829998016357424,\n",
       "  'valid_acc': 0.42829998016357423,\n",
       "  'wall_time': 356.6618399620056},\n",
       " 73: {'test_acc': 0.42759998321533205,\n",
       "  'valid_acc': 0.42579998016357423,\n",
       "  'wall_time': 280.5092086791992},\n",
       " 74: {'test_acc': 0.4356999969482422,\n",
       "  'valid_acc': 0.4293000030517578,\n",
       "  'wall_time': 304.10149359703064},\n",
       " 75: {'test_acc': 0.4471999740600586,\n",
       "  'valid_acc': 0.44459999084472657,\n",
       "  'wall_time': 400.45698046684265},\n",
       " 76: {'test_acc': 0.45169998168945313,\n",
       "  'valid_acc': 0.4495000076293945,\n",
       "  'wall_time': 347.3728129863739},\n",
       " 77: {'test_acc': 0.4112999725341797,\n",
       "  'valid_acc': 0.40169998168945314,\n",
       "  'wall_time': 281.4130663871765},\n",
       " 78: {'test_acc': 0.4336999893188477,\n",
       "  'valid_acc': 0.43189998626708986,\n",
       "  'wall_time': 316.0343248844147},\n",
       " 79: {'test_acc': 0.45529998779296876,\n",
       "  'valid_acc': 0.45439998626708983,\n",
       "  'wall_time': 459.55058550834656},\n",
       " 80: {'test_acc': 0.38340000152587894,\n",
       "  'valid_acc': 0.3818999862670899,\n",
       "  'wall_time': 307.30062079429626},\n",
       " 81: {'test_acc': 0.3930999755859375,\n",
       "  'valid_acc': 0.39099998474121095,\n",
       "  'wall_time': 232.74533009529114},\n",
       " 82: {'test_acc': 0.4386999893188477,\n",
       "  'valid_acc': 0.4341999816894531,\n",
       "  'wall_time': 303.26314997673035},\n",
       " 83: {'test_acc': 0.43200000762939456,\n",
       "  'valid_acc': 0.4231999969482422,\n",
       "  'wall_time': 292.08738255500793},\n",
       " 84: {'test_acc': 0.4403999710083008,\n",
       "  'valid_acc': 0.44020000457763675,\n",
       "  'wall_time': 300.11958265304565},\n",
       " 85: {'test_acc': 0.4190999984741211,\n",
       "  'valid_acc': 0.41309997558593753,\n",
       "  'wall_time': 298.07866740226746},\n",
       " 86: {'test_acc': 0.44649997711181644,\n",
       "  'valid_acc': 0.4397999954223633,\n",
       "  'wall_time': 310.4106824398041},\n",
       " 87: {'test_acc': 0.46419998168945314,\n",
       "  'valid_acc': 0.46099998474121096,\n",
       "  'wall_time': 325.09605145454407},\n",
       " 88: {'test_acc': 0.45049999237060545,\n",
       "  'valid_acc': 0.4441999816894531,\n",
       "  'wall_time': 329.19744896888733},\n",
       " 89: {'test_acc': 0.4493000030517578,\n",
       "  'valid_acc': 0.4456999969482422,\n",
       "  'wall_time': 312.2850043773651},\n",
       " 90: {'test_acc': 0.4803999710083008,\n",
       "  'valid_acc': 0.48389999389648436,\n",
       "  'wall_time': 642.2862105369568},\n",
       " 91: {'test_acc': 0.4739999771118164,\n",
       "  'valid_acc': 0.4595999908447266,\n",
       "  'wall_time': 397.3292808532715},\n",
       " 92: {'test_acc': 0.4177999877929688,\n",
       "  'valid_acc': 0.4177999877929688,\n",
       "  'wall_time': 278.3622479438782},\n",
       " 93: {'test_acc': 0.3982999801635742,\n",
       "  'valid_acc': 0.39279998779296876,\n",
       "  'wall_time': 293.60141253471375},\n",
       " 94: {'test_acc': 0.41250000000000003,\n",
       "  'valid_acc': 0.4122999954223633,\n",
       "  'wall_time': 375.1176199913025},\n",
       " 95: {'test_acc': 0.40489997863769533,\n",
       "  'valid_acc': 0.39849998474121096,\n",
       "  'wall_time': 299.110830783844},\n",
       " 96: {'test_acc': 0.4064999771118164,\n",
       "  'valid_acc': 0.4032999801635742,\n",
       "  'wall_time': 268.7792468070984},\n",
       " 97: {'test_acc': 0.44450000762939457,\n",
       "  'valid_acc': 0.44860000610351564,\n",
       "  'wall_time': 287.01158928871155},\n",
       " 98: {'test_acc': 0.4280999755859375,\n",
       "  'valid_acc': 0.4236000061035156,\n",
       "  'wall_time': 330.7720868587494},\n",
       " 99: {'test_acc': 0.4715999984741211,\n",
       "  'valid_acc': 0.4727999877929688,\n",
       "  'wall_time': 449.14019107818604}}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# metrics for each architecture from the search space\n",
    "\n",
    "benchmark['cifar100']['mobilenet']['nst'][9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read high-fidelity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_data = benchmark['cifar100']['mobilenet']['nokd'][1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test accuracy histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANkAAADQCAYAAACKscS+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOxUlEQVR4nO3df5BdZX3H8feHxBBpgklIRAyEJTXQYtqCDVbBaflRpxEcDU7AZESDMGRGpxC0VcPQ1o6tM+nUUVQcmVQhoBFBICMKojEBWlpANiEJPwJJDEEi0URSQGEoBL/94zxLTja7e+/u3uecuzef18zOnvvcc8/55uR+9vy4z3muIgIzy+egugsw63QOmVlmDplZZg6ZWWYOmVlmDplZZqPrLqAZkydPjq6urrrLMNvPmjVrfhMRUwaaZ0SErKuri+7u7rrLMNuPpCcbzePDRbPMHDKzzLKFTNLVknZKerjU9u+SHpO0QdIKSRNyrd+sXeTcky0DZvdqWwnMjIg/BTYBl2Vcv1lbyBayiPhPYHevtp9ExJ708D7gyFzrN2sXdV5dvAC4ob8nJS0EFgJMmzatqpqsga7Ft/XZvm3JWRVXMnLUcuFD0uXAHmB5f/NExNKImBURs6ZMGfBjCLO2VvmeTNIC4L3AGeGb2ewAUGnIJM0GPgP8VUS8WOW6zeqS8xL+9cC9wHGStku6ELgSGA+slLRO0lW51m/WLrLtySJifh/N38y1PrN25R4fZpk5ZGaZOWRmmTlkZpk5ZGaZOWRmmTlkZpmNiOEHrP3113G4PwdSh2Lvycwyc8jMMnPIzDJzyMwyc8jMMnPIzDJzyMwyc8jMMnPIzDKregThSZJWStqcfk/MtX6zdlH1CMKLgVURMQNYlR6bdbRKRxAG3g9cm6avBebkWr9Zu6i6g/DhEbEDICJ2SHpjfzN6BOF6DbbDb6uW34kdh9v2wodHELZOUXXIfi3pCID0e2fF6zerXNUhuxVYkKYXAN+veP1mlat6BOElwLslbQbenR6bdbSqRxAGOCPXOs3aUdte+DDrFA6ZWWYOmVlmDplZZg6ZWWYOmVlmDplZZg6ZWWYOmVlmDplZZg6ZWWYOmVlmDplZZg6ZWWYOmVlmDplZZg6ZWWa1hEzSJyQ9IulhSddLGltHHWZVqDxkkqYClwCzImImMAqYV3UdZlWp63BxNPB6SaOBQ4Cna6rDLLvKQxYRvwS+APwC2AE8FxE/6T2fpIWSuiV179q1q+oyzVqmjsPFiRRj4h8DvBn4A0nn9Z7PIwhbp6jjcPGvgSciYldEvALcApxcQx1mlagjZL8A3iHpEEmiGIdxYw11mFWijnOy+4GbgLXAQ6mGpVXXYVaVqr86CYCI+Czw2TrWbVY19/gwy8whM8vMITPLrKmQSTqlmTYz21+ze7KvNtlmZr0MeHVR0jspPiieIumTpacOpejYa2YNNLqEPwYYl+YbX2p/HpibqyizTjJgyCLibuBuScsi4smKarIDWNfi2/ps37bkrIoraZ1mP4w+WNJSoKv8mog4PUdRZp2k2ZB9D7gK+Abwar5yzDpPsyHbExFfz1qJWYdq9hL+DyR9XNIRkib1/GStzKxDNLsnW5B+f6rUFsD01pZj1nmaCllEHJO7ELNO1VTIJH2kr/aIuK615Zh1nmYPF08qTY+luJt5LeCQmTXQ7OHixeXHkt4AfCtLRWYdZqi3urwIzBjqSiVNkHSTpMckbUx9JM06UrPnZD+guJoIRcfgPwZuHMZ6vwzcERFzJY2hGODUrCM1e072hdL0HuDJiNg+lBVKOhT4S+B8gIh4GXh5KMsyGwmaPSe7W9Lh7L0AsnkY65wO7AKukfRnwBpgUUS8UJ5J0kJgIcC0adOGsTobSH8dckeKkdChuNk7o88FfgacA5wL3C9pqLe6jAbeBnw9Ik4EXgAW957JIwhbp2j2cPFy4KSI2AkgaQrwU4rxEwdrO7A9jb9IWsZ+ITPrFM1eXTyoJ2DJM4N47T4i4lfAU5KOS01nAI8OZVlmI0Gze7I7JP0YuD49/iBw+zDWezGwPF1Z3Ap8dBjLMmtrjcb4eAtweER8StIHgHcBAu4Flg91pRGxDpg11NebjSSNDvmuAH4LEBG3RMQnI+ITFHuxK3IXZ9YJGoWsKyI29G6MiG6KoQjMrIFGIRvoC9Nf38pCzDpVo5A9IOmi3o2SLqT4ENnMGmh0dfFSYIWkD7E3VLMoxmM8O2dhZp2i0biLvwZOlnQaMDM13xYRq7NXZtYhmu27eCdwZ+ZazDpSLd+0afmM9A6/ncjfT2aWmUNmlplDZpaZQ2aWmUNmlplDZpaZQ2aWmUNmlplDZpZZbSGTNErSg5J+WFcNZlWoc0+2CNhY4/rNKlFLyCQdCZxF8R3UZh2trj3ZFcCngd/3N4OkhZK6JXXv2rWrusrMWqzykEl6L7AzIga8s9ojCFunqGNPdgrwPknbgO8Cp0v6dg11mFWi8pBFxGURcWREdAHzgNURcV7VdZhVxZ+TmWVW653REXEXcFedNZjl5j2ZWWYOmVlmDplZZg6ZWWYOmVlmDplZZg6ZWWYeQdg60lBGUt625KwMlXhPZpadQ2aWmUNmlplDZpaZQ2aWmUNmlplDZpaZQ2aWWR0D6Rwl6U5JGyU9ImlR1TWYVamOHh97gL+LiLWSxgNrJK2MiEdrqMUsuzoG0tkREWvT9G8pRhGeWnUdZlWp9ZxMUhdwInB/nXWY5VRbB2FJ44CbgUsj4vk+nl8ILASYNm1axdUNT3+dUwfbAbVVy7F61TUW/usoArY8Im7pax6PIGydoo6riwK+CWyMiC9WvX6zqtU1TPeHKYbnXpd+zqyhDrNKVH5OFhH3AKp6vWZ1cY8Ps8wcMrPMHDKzzBwys8wcMrPMHDKzzBwys8wcMrPMFBF119DQrFmzoru7u9/n261D7lBGr7X2NdD/v6Q1ETFroNd7T2aWmUNmlplDZpaZQ2aWmUNmlplDZpaZQ2aWmUNmllldA+nMlvS4pC2SFtdRg1lV6hhIZxTwNeA9wPHAfEnHV12HWVXq2JO9HdgSEVsj4mXgu8D7a6jDrBJ1hGwq8FTp8XY8TLd1sDpGEO5rpKr9eimXRxAGfifp8UGv6N9em5wM/Gawr+9jOa0wrFpazLX0bZ9aGvz/H91oYXWEbDtwVOnxkcDTvWeKiKXA0lasUFJ3o57SVXEtfevkWuo4XHwAmCHpGEljgHnArTXUYVaJOgY33SPpb4EfA6OAqyPikarrMKtKLd/qEhG3A7dXuMqWHHa2iGvpW8fWMiLujDYbydytyiyzERmyZrplSTpX0qPpy9+/U2pfIGlz+llQav9zSQ+lZX4lfcVTtloknSDp3tS2QdIHS/Mvk/RE6VtvTsi8TV4trevWUvsxku5P2+qGdKEq5zY5rVTHOkkvSZoz1G3STC2SvlRa5iZJz5aea817JSJG1A/FxZKfA9OBMcB64Phe88wAHgQmpsdvTL8nAVvT74lpumeenwHvpPgc70fAezLXciwwI02/GdgBTEiPlwFzq9gmafp3/Sz3RmBemr4K+FjuWkrzTAJ2A4cMZZs0W0uv+S+muBDX0vfKSNyTNdMt6yLgaxHxvwARsTO1/w2wMiJ2p+dWArMlHQEcGhH3RrEVrwPm5KwlIjZFxOY0/TSwExjqV4oOZ5v0Kf11Ph24KTVdS+Zt0stc4EcR8WIT6xxOLWXzgevTdMveKyMxZM10yzoWOFbSf0u6T9LsBq+dmqYHWmara3mNpLdT/KX9ean58+kw8kuSDs5cx1hJ3am95w1zGPBsROwZYJk5aukxj71v+B6D2SbN1gKApKOBY4DVDV476PdKbV/MPgzNdMsaTXFIcipFj5L/kjRzgNc21dWrlbVExLMA6S/jt4AFEfH79JrLgF9RBG8p8BngcxnrmBYRT0uaDqyW9BDwfBPLzFFLzzb5E4rPUnsMdps0W0uPecBNEfFqg9cO+r0yEvdkzXTL2g58PyJeiYgngMcp/lP7e+32ND3QMltdC5IOBW4D/iEi7ut5QUTsiML/AddQHPZkqyMdrhIRW4G7gBMp+u5NkDR6gGW2vJbkXGBFRLzS0zCEbdJsLT167zlb914ZzIlkO/xQ/BXcSrFr7zmZfWuveWYD16bpyRS7/cMoTmKfoDiRnZimJ6X5HgDewd6T2TMz1zIGWAVc2sdyj0i/BVwBLMlYx0Tg4FL7ZtLFAeB77Hvh4+M5t0np+fuA04azTZqtJc13HLCN9Llx7L3w0ZL3Su2hGWLQzgQ2UZzDXJ7aPge8r/Qf8UXgUeChnjdKeu4CYEv6+WipfRbwcFrmleUNnqMW4DzgFWBd6eeE9NzqNO/DwLeBcRnrODk9Xp9+X1ha5nSKK2lbKAJ3cAX/P13AL4GDei1z0NukmVrS43+mj9C26r3iHh9mmY3EczKzEcUhM8vMITPLzCEzy8whM8vMIWtzks6WFJL+qO5abGgcsvY3H7iHokdCFioGnLVMHLI2JmkccApwIaWQSfp0up9pvaQlqe0tkn6a2tZK+kNJp0r6Yel1V0o6P01vk/RPku4BzpF0kaQH0utvlnRImu9wSStS+3pJJ0v6F0mLSsv9vKRLKtkoI9BI7CB8IJkD3BERmyTtlvQ24PDU/hcR8aKkSWne5RS9FlZIGkvxB/Sovhf7mpci4l0Akg6LiP9I0/9KEeyvAl8B7o6Is9MebxxFX71bgC9LOojiD0AzfQkPSA5Ze5tP0U8Pinuh5lOE55pI91lFxG5J44GpEbEitb0E0OiGXeCG0vTMFK4JFEHq6QF/OvCRtNxXgeeA5yQ9I+lEitA/GBHPDOcf2skcsjYl6TCKN/hMSUFxl28AN7P/rRX9pWkP+54SjO31/Aul6WXAnIhYnw4pT21Q4jeA84E3AVc3mPeA5nOy9jUXuC4ijo6Irog4iqIn+G7ggtI506SIeB7YXhoP4+D0/JPA8enxG4AzBljfeGCHpNcBHyq1rwI+lpY7Kt2eA7CCojf9Sex735f14pC1r/kUb+SymynGA7kV6Ja0Dvj79NyHgUskbQD+B3hTRDxFMU7HBopztgcHWN8/AvdT3Gb/WKl9EXBaupFzDfBWgChu578TuDH23uhofXAvfBuSdMFjLXBOpLFKrG/ek9mgqfjSxi3AKgesMe/JzDLznswsM4fMLDOHzCwzh8wsM4fMLDOHzCyz/wcKt0vcyYQp1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "\n",
    "plt.hist([x['test_acc'] for x in base_data.values()], bins = 25)\n",
    "ax.set_xlabel('Accuracy')\n",
    "ax.set_ylabel('Count')\n",
    "fig.set_size_inches(3, 3)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### accuracy increase from KD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "nokd_data = benchmark['cifar100']['mobilenet']['nokd'][27]\n",
    "kd_data = benchmark['cifar100']['mobilenet']['kd'][27]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.13390700000000003"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean([nokd_data[x]['test_acc'] for x in nokd_data.keys()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22890200000000005"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean([kd_data[x]['test_acc'] for x in kd_data.keys()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### check correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_corr(label, data, top = False, verbose = True):\n",
    "    \n",
    "    full = []\n",
    "    acc = []\n",
    "    \n",
    "    for k in data.keys():\n",
    "        acc.append(data[k]['valid_acc'])\n",
    "        full.append(base_data[k]['test_acc'])\n",
    "    \n",
    "    if top:\n",
    "        joined = list(zip(acc, full))\n",
    "        joined.sort(key = lambda x : -x[1])\n",
    "      \n",
    "        acc = [x[0] for x in joined[0:20]]    \n",
    "        full = [x[1] for x in joined[0:20]]\n",
    "    \n",
    "    if verbose:\n",
    "        print('%10s  Pearson: %.2f Kendall: %.2f' % (label, stats.pearsonr(full, acc)[0], stats.kendalltau(full, acc)[0]))\n",
    "    \n",
    "    return stats.pearsonr(full, acc)[0], stats.kendalltau(full, acc)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_report(label1, label2, data, nokd_data):\n",
    "        calc_corr(label1, nokd_data, top = False)\n",
    "        calc_corr(label2, data, top = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "nokd_data = benchmark['cifar100']['mobilenet']['nokd'][27]\n",
    "kd_data = benchmark['cifar100']['mobilenet']['kd'][27]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  no KD 3%  Pearson: 0.11 Kendall: 0.09\n",
      "     KD 3%  Pearson: 0.34 Kendall: 0.23\n"
     ]
    }
   ],
   "source": [
    "print_report('no KD 3%', '   KD 3%', kd_data, nokd_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NAS with low-fidelity evaluations only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_X(ss):\n",
    "    \n",
    "    if ss == 'mobilenet':\n",
    "        arcs_cnt = 100\n",
    "    elif ss == 'shufflenet':\n",
    "        arcs_cnt = 300\n",
    "\n",
    "    #\n",
    "    #\n",
    "    #\n",
    "    if ss == 'mobilenet':\n",
    "        with open('random_arcs_mobilenetv2.pickle', 'rb') as infile:\n",
    "            obj = pickle.load(infile)\n",
    "            \n",
    "    elif ss == 'shufflenet':\n",
    "        with open('random_arcs_shufflenetv2.pickle', 'rb') as infile:\n",
    "            obj = pickle.load(infile)\n",
    "    \n",
    "    #\n",
    "    #\n",
    "    #\n",
    "    if ss == 'mobilenet':\n",
    "        X = np.zeros((100, 16))\n",
    "\n",
    "        for i in range(100):\n",
    "            X[i] = np.array(obj[i])[:, [1,2]].flatten()\n",
    "    elif ss == 'shufflenet':\n",
    "        X = np.zeros((arcs_cnt, 7))\n",
    "\n",
    "        for i in range(arcs_cnt):\n",
    "            X[i] = np.array(obj[i]['out_channels'] + obj[i]['num_blocks'])\n",
    "            \n",
    "    X_scaled = preprocessing.scale(X, axis = 0)\n",
    "    \n",
    "    return X_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_scaled = get_X('mobilenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.gaussian_process import GaussianProcessRegressor as GPR\n",
    "from sklearn.gaussian_process.kernels import RBF\n",
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def warn(*args, **kwargs):\n",
    "    pass\n",
    "import warnings\n",
    "warnings.warn = warn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_search_alg1(data = None, n_init = 30):\n",
    "\n",
    "    checked_points = list(np.random.choice(list(data.keys()), size = n_init, replace = False))\n",
    "\n",
    "    X_train = []\n",
    "    y_train = []\n",
    "    \n",
    "    for idx in checked_points:\n",
    "        X_train.append(X_scaled[idx])\n",
    "        y = data[idx]['valid_acc']\n",
    "\n",
    "        y_train.append(y)\n",
    "\n",
    "    Y = np.zeros((n_init, 1))\n",
    "    Y[:,0] = y_train\n",
    "\n",
    "    gpr = GPR(kernel = RBF(1.0))\n",
    "    gpr.fit(X_train, y_train)\n",
    "\n",
    "    preds = gpr.predict(X_scaled, return_std = True)\n",
    "    best_idx = np.argmax(preds[0])\n",
    "\n",
    "    return best_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_acc_list = [base_data[x]['test_acc'] for x in range(len(base_data))]\n",
    "test_acc_list.sort(reverse = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_percentile(val, test_acc_list):\n",
    "    \n",
    "    idx = 0 \n",
    "    \n",
    "    while idx < len(test_acc_list):\n",
    "        if val > test_acc_list[idx]:\n",
    "            break\n",
    "        idx += 1\n",
    "        \n",
    "    return idx / len(test_acc_list) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "percentiles = {}\n",
    "\n",
    "for i in range(len(base_data)):\n",
    "    val = base_data[i]['test_acc']\n",
    "    percentiles[i] = get_percentile(val, test_acc_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_comparison(data_nokd, data_kd):\n",
    "    \n",
    "    random.seed(2)\n",
    "    np.random.seed(2)\n",
    "    \n",
    "    n_trials = 1000\n",
    "    res_kd = []\n",
    "    res_nokd = []\n",
    "\n",
    "    for i in range(n_trials):\n",
    "        res_nokd.append(run_search_alg1(data_nokd))\n",
    "        res_kd.append(run_search_alg1(data_kd))\n",
    "    \n",
    "    A_nokd = [base_data[x]['test_acc'] for x in res_nokd] \n",
    "    A_kd = [base_data[x]['test_acc'] for x in res_kd] \n",
    "    \n",
    "    print(np.mean(A_nokd), np.std(A_nokd))\n",
    "    print(np.mean(A_kd), np.std(A_kd))\n",
    "    print(ttest_ind(A_kd, A_nokd))\n",
    "    print()\n",
    "    \n",
    "    B_nokd = [percentiles[x] for x in res_nokd] \n",
    "    B_kd = [percentiles[x] for x in res_kd] \n",
    "    \n",
    "    print(np.mean(B_nokd), np.std(B_nokd))\n",
    "    print(np.mean(B_kd), np.std(B_kd))\n",
    "    print(ttest_ind(B_kd, B_nokd))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "nokd_data = benchmark['cifar100']['mobilenet']['nokd'][27]\n",
    "kd_data = benchmark['cifar100']['mobilenet']['kd'][27]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6734325000000001 0.010392487370692345\n",
      "0.6771741000000002 0.008881816210100268\n",
      "Ttest_indResult(statistic=8.650606211408336, pvalue=1.0335909163268219e-17)\n",
      "\n",
      "30.915 17.107535620304873\n",
      "25.397 13.25637171325548\n",
      "Ttest_indResult(statistic=-8.058540329206473, pvalue=1.3149099770458328e-15)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "run_comparison(nokd_data, kd_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-fidelity NAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_GPR(lf_data = None, max_points = None, data = base_data,\\\n",
    "                    first_point = None, max_value = None):\n",
    "\n",
    "    n_init = 1\n",
    "\n",
    "    if not first_point:\n",
    "        checked_points = list(np.random.choice(list(range(len(data))), size = n_init, replace = False))\n",
    "    else:\n",
    "        checked_points = [first_point]\n",
    "        \n",
    "    while len(checked_points) < max_points:\n",
    "            \n",
    "        max_y = max([data[idx]['valid_acc'] for idx in checked_points])\n",
    "        \n",
    "        X_train = []\n",
    "        y_train = []\n",
    "        \n",
    "        for idx in checked_points:\n",
    "            X_train.append(X_scaled[idx])\n",
    "            y = data[idx]['valid_acc']\n",
    "    \n",
    "            y_train.append(y)\n",
    "\n",
    "        gpr = GPR(kernel = RBF(1.0), n_restarts_optimizer = 3, normalize_y = True)\n",
    "        gpr.fit(X_train, y_train)\n",
    "        \n",
    "        preds = gpr.predict(X_scaled, return_std = True)\n",
    "\n",
    "        max_score = 0\n",
    "        best_new_idx = 0\n",
    "\n",
    "        for idx in range(len(data)):\n",
    "            if idx not in checked_points:\n",
    "                score = preds[0][idx] + 0.5 * preds[1][idx]\n",
    "\n",
    "                if score > max_score:\n",
    "                    max_score = score\n",
    "                    best_new_idx = idx\n",
    "\n",
    "        if (max_value is None) or max_y < max_value:\n",
    "            checked_points.append(best_new_idx)\n",
    "            \n",
    "        if not (max_value is None) and max_y >= max_value:\n",
    "            break\n",
    "        \n",
    "    return checked_points, gpr.kernel_, preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RS:\n",
    "    def __init__(self, hf_data, max_hf_points):\n",
    "        self.hf_data = hf_data\n",
    "        self.lf_points = []\n",
    "        self.hf_points = []\n",
    "        self.max_hf_points = max_hf_points\n",
    "        \n",
    "        self.done = False\n",
    "        \n",
    "    def step(self):\n",
    "        max_value = max([self.hf_data[idx]['valid_acc'] for idx in self.hf_data])\n",
    "        \n",
    "        order = np.random.choice(range(len(self.hf_data)), size = len(self.hf_data), replace = False)\n",
    "        \n",
    "        for i in range(1, len(order)+1):\n",
    "            max_y = max([self.hf_data[x]['valid_acc'] for x in order[:i]])\n",
    "            \n",
    "            if max_y == max_value:\n",
    "                self.hf_points = order[:i]\n",
    "                break\n",
    "        \n",
    "        self.done = True\n",
    "        self.hf_iters = len(self.hf_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HF_GPR:\n",
    "    def __init__(self, hf_data, max_hf_points):\n",
    "        self.hf_data = hf_data\n",
    "        self.lf_points = []\n",
    "        self.hf_points = []\n",
    "        self.max_hf_points = max_hf_points\n",
    "        self.done = False\n",
    "        \n",
    "    def step(self):\n",
    "        max_value = max([self.hf_data[idx]['valid_acc'] for idx in self.hf_data])\n",
    "\n",
    "        points, kernel, preds = run_GPR(max_points = self.max_hf_points, data = self.hf_data, max_value = max_value)      \n",
    "        self.hf_points = points\n",
    "        self.done = True\n",
    "\n",
    "        self.hf_iters = len(self.hf_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MF_SearchAlg2:\n",
    "    def __init__(self, lf_data, hf_data, max_lf_points, max_hf_points):\n",
    "        self.lf_data = lf_data\n",
    "        self.hf_data = hf_data\n",
    "        \n",
    "        self.lf_points = []\n",
    "        self.hf_points = []\n",
    "        \n",
    "        self.max_lf_points = max_lf_points\n",
    "        self.max_hf_points = max_hf_points\n",
    "    \n",
    "        self.done = False\n",
    "        \n",
    "    def step(self):\n",
    "        if not self.lf_points:\n",
    "            points, kernel, preds = run_GPR(max_points = self.max_lf_points, data = self.lf_data,\n",
    "                                         first_point = None)\n",
    "        \n",
    "            self.lf_points = points \n",
    "            \n",
    "            idx = np.argmax([self.lf_data[x]['valid_acc'] for x in self.lf_points])\n",
    "            self.first_point = self.lf_points[idx]\n",
    "            \n",
    "        else:\n",
    "            max_value = max([self.hf_data[idx]['valid_acc'] for idx in self.hf_data])\n",
    "            \n",
    "            points, kernel, preds = run_GPR(max_points = self.max_hf_points, data = self.hf_data,\n",
    "                                             first_point = self.first_point, max_value = max_value)\n",
    "            \n",
    "            self.hf_points = points\n",
    "            self.done = True\n",
    "    \n",
    "            self.hf_iters = len(self.hf_points)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_experiment(creator, n_trials, verbose = False):\n",
    "    np.random.seed(2)\n",
    "    random.seed(2)\n",
    "\n",
    "    hf_iters_avg = 0\n",
    "    wall_time_avg = 0\n",
    "    \n",
    "    time_list_avg = np.zeros(1000)\n",
    "    acc_list_avg = np.zeros(1000)\n",
    "\n",
    "    for trial in range(n_trials):\n",
    "        obj = creator()\n",
    "        \n",
    "        while not obj.done:\n",
    "            obj.step()\n",
    "            \n",
    "        if verbose:\n",
    "            print()\n",
    "            print(obj.lf_points)\n",
    "            print(obj.first_point)\n",
    "            print(obj.hf_points)\n",
    "        \n",
    "        w_time = 0.0\n",
    "        time_list = []\n",
    "        acc_list = []\n",
    "        \n",
    "        for idx in range(len(obj.lf_points)):\n",
    "            point = obj.lf_points[idx]\n",
    "            \n",
    "            w_time += obj.lf_data[point]['wall_time']\n",
    "            time_list.append(w_time)\n",
    "            \n",
    "            best_idx = np.argmax([obj.lf_data[x]['valid_acc'] for x in obj.lf_points[0:idx+1]])\n",
    "            best_point = obj.lf_points[best_idx]\n",
    "            \n",
    "            acc_list.append(obj.hf_data[best_point]['valid_acc'])\n",
    "            \n",
    "        for idx in range(len(obj.hf_points)):\n",
    "            point = obj.hf_points[idx]\n",
    "            \n",
    "            w_time += obj.hf_data[point]['wall_time']\n",
    "            time_list.append(w_time)\n",
    "            \n",
    "            best_idx = np.argmax([obj.hf_data[x]['valid_acc'] for x in obj.hf_points[0:idx+1]])\n",
    "            best_point = obj.hf_points[best_idx]\n",
    "            \n",
    "            acc_list.append(obj.hf_data[best_point]['valid_acc'])\n",
    "            \n",
    "        time_list = time_list + [w_time for _ in range(1000 - len(time_list))]\n",
    "        acc_list = acc_list + [acc_list[-1] for _ in range(1000 - len(acc_list))]\n",
    "        \n",
    "        time_list_avg += np.array(time_list)\n",
    "        acc_list_avg += np.array(acc_list)\n",
    "\n",
    "        hf_iters_avg += obj.hf_iters\n",
    "        wall_time_avg += sum([obj.lf_data[x]['wall_time'] for x in obj.lf_points])\n",
    "        wall_time_avg += sum([obj.hf_data[x]['wall_time'] for x in obj.hf_points])\n",
    "\n",
    "    hf_iters_avg /= n_trials\n",
    "    wall_time_avg /= n_trials\n",
    "    \n",
    "    time_list_avg /= n_trials\n",
    "    acc_list_avg /= n_trials\n",
    "    \n",
    "    return (hf_iters_avg, wall_time_avg, time_list_avg, acc_list_avg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### random search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs = lambda : RS(base_data, len(base_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = run_experiment(rs, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86775.35825209857\n"
     ]
    }
   ],
   "source": [
    "print(res[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### High-fidelity only GPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_gpr = lambda : HF_GPR(base_data, len(base_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "res = run_experiment(hf_gpr, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13437.436940178872\n"
     ]
    }
   ],
   "source": [
    "print(res[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi-fidelity NAS (Algorithm 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "lf_data = benchmark['cifar100']['mobilenet']['nst'][9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mf_init = lambda : MF_SearchAlg2(lf_data, base_data, 10, len(base_data))\n",
    "res = run_experiment(mf_init, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15239.584509215354"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res[1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}