hyperparameter_opt.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter optimization Notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First we needed to create the data() and the create_model() funtions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the data function we needed to implement again the training, validation and the test data creation function. For this we also needed the VideoFrameGenerator function to be implemented in the same place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from hyperas.distributions import choice, uniform\n",
    "from hyperas import optim\n",
    "from hyperopt import Trials, STATUS_OK, tpe\n",
    "CLASS_NUM = 10\n",
    "NBFRAME = 20\n",
    "#model = create_Resnet_GRU(mode='backendGRU', inputDim=64, hiddenDim=20, nClasses=CLASS_NUM, frameLen=NBFRAME, every_frame=True)\n",
    "\n",
    "def data():\n",
    "    class VideoFrameGenerator(keras.utils.Sequence):\n",
    "    '''\n",
    "        Generates video frames from frame directories\n",
    "    '''\n",
    "        def __init__(self,\n",
    "                 from_dir,\n",
    "                 rescale=1/255.,\n",
    "                 batch_size=8,\n",
    "                 shape=(100, 100, 3),\n",
    "                 nbframe=20,\n",
    "                 shuffle=False,\n",
    "                 transform:keras.preprocessing.image.ImageDataGenerator=None,\n",
    "                 grayscale=False):\n",
    "\n",
    "            self.from_dir = from_dir\n",
    "            self.rescale = rescale\n",
    "            self.nbframe = nbframe\n",
    "            self.batch_size = batch_size\n",
    "            self.target_shape = shape\n",
    "            self.shuffle = shuffle\n",
    "            self.transform = transform\n",
    "            self.grayscale = grayscale        \n",
    "            self.classes = []\n",
    "            self.files = []\n",
    "            self.class_indices = {}\n",
    "            self._current = 0\n",
    "            self.__filecount = 0\n",
    "            self.__get_all_files()\n",
    "\n",
    "            print(\"\\nTotal data: %d classes for %d files for %s\" % (\n",
    "                len(self.classes),\n",
    "                len(self.files),\n",
    "                os.path.basename(self.from_dir)))\n",
    "    \n",
    "        \n",
    "        def __len__(self):\n",
    "            return self.__filecount//self.batch_size\n",
    "\n",
    "        def __iter__(self):\n",
    "              return self\n",
    "\n",
    "        def __next__(self):\n",
    "            return self.next()\n",
    "    \n",
    "        def next(self):\n",
    "            elem = self[self._current]\n",
    "            self._current += 1\n",
    "            if self._current == len(self):\n",
    "                self._current = 0\n",
    "                self.on_epoch_end()\n",
    "            return elem\n",
    "\n",
    "        def __getitem__(self, index):\n",
    "            labels = []\n",
    "            images = []\n",
    "            # get the next block of files (it contains a batchsize of the video frames)\n",
    "            frame_dirs = self.files[index*self.batch_size:(index+1)*self.batch_size]\n",
    "\n",
    "            for dir in frame_dirs:\n",
    "                # get classname from path\n",
    "                label_name = self._get_classname(dir)\n",
    "\n",
    "                # check if class name is valid\n",
    "                label = np.zeros(len(self.classes))\n",
    "                if label_name not in self.class_indices.keys():\n",
    "                    print(f'ERROR: {label_name} not in class labels.')\n",
    "                    continue\n",
    "          \n",
    "                # get the id of the class\n",
    "                col = self.class_indices[label_name]\n",
    "                label[col] = 1.\n",
    "\n",
    "                # get preprocessed frames for the actual video\n",
    "                frames = self.__readframe(dir)\n",
    "\n",
    "                if frames is None:\n",
    "                    continue\n",
    "\n",
    "                # use data augmentation\n",
    "                frames = self.__data_aug(frames)\n",
    "        \n",
    "                # add the sequence in batch\n",
    "                images.append(frames)\n",
    "                labels.append(label)\n",
    "\n",
    "            return np.array(images), np.array(labels)\n",
    "    \n",
    "        def on_epoch_end(self):\n",
    "            if self.shuffle:\n",
    "                random.shuffle(self.files)\n",
    "\n",
    "        def _get_classname(self, video: str) -> str:\n",
    "            classname = os.path.basename(os.path.dirname(video))\n",
    "            return classname\n",
    "    \n",
    "        def __get_all_files(self):\n",
    "            # get classes and sort them in ABC order\n",
    "            self.classes = glob.glob(os.path.join(self.from_dir, '*'))\n",
    "            self.classes = sorted([os.path.basename(c) for c in self.classes])\n",
    "\n",
    "            # create label indexes for classes\n",
    "            self.class_indices = dict(zip(self.classes, range(len(self.classes))))\n",
    "\n",
    "            # count all video file dirs\n",
    "            self.__filecount = len(glob.glob(os.path.join(self.from_dir, '*/*')))\n",
    "        \n",
    "            for classname in self.classes:\n",
    "                # list video file dirs for classes\n",
    "                videos = glob.glob(os.path.join(self.from_dir, classname, '*'))\n",
    "\n",
    "                self.files += videos\n",
    "\n",
    "            # shuffle files\n",
    "            random.shuffle(self.files)\n",
    "            self.on_epoch_end()\n",
    "        \n",
    "        def __readframe(self, frame_dir):\n",
    "            frames = []\n",
    "\n",
    "            # read frame images\n",
    "            files = glob.glob(os.path.join(frame_dir, '*'))\n",
    "            for f in files:\n",
    "                frame = cv.imread(f)\n",
    "                frame = cv.cvtColor(frame, cv.COLOR_BGR2RGB)\n",
    "                frame = cv.resize(frame, self.target_shape[:2])\n",
    "                if self.grayscale:\n",
    "                      frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) \n",
    "                frame = img_to_array(frame) * self.rescale\n",
    "                frames.append(frame)\n",
    "      \n",
    "            if len(frames)==0:\n",
    "                print(\"ERROR: not found frames\\n\")\n",
    "                return None\n",
    "\n",
    "            else:\n",
    "                # use padding to create a self.nbframes length of images\n",
    "                frames = pad_sequences([frames], padding=\"post\", maxlen=self.nbframe, dtype=float, truncating=\"post\")[0]\n",
    "\n",
    "                # add frames in memory\n",
    "                if len(frames) == self.nbframe:\n",
    "                    return frames\n",
    "            \n",
    "                else:\n",
    "                    print('\\n%s has not enough frames ==> %d' % (frame_dir, len(frames)))\n",
    "                    return None\n",
    "            \n",
    "        def __data_aug(self, frames):\n",
    "            \"\"\" Make random transformation based on ImageGenerator arguments\"\"\"\n",
    "            T = None\n",
    "            if self.transform is not None:\n",
    "                # get random transform from generator\n",
    "                T = self.transform.get_random_transform(self.target_shape[:2])\n",
    "        \n",
    "            result = frames\n",
    "            if T is not None:\n",
    "                # apply transformation\n",
    "                result = [self.transform.apply_transform(frame, T) for frame in frames]\n",
    "    \n",
    "            return np.array(result)\n",
    "  \n",
    "    \n",
    "    NBFRAME = 20\n",
    "    TARGET_SIZE = 100\n",
    "    BSIZE = 32\n",
    "    DATA_AUG = False\n",
    "    TO_GRAYSCALE = True\n",
    "\n",
    "    if DATA_AUG:\n",
    "        data_aug = keras.preprocessing.image.ImageDataGenerator(\n",
    "          horizontal_flip=True,\n",
    "          rotation_range=8,\n",
    "          height_shift_range=.2)\n",
    "    else:\n",
    "        data_aug = None\n",
    "\n",
    "    TRAIN_DIR = '/content/visual_speech_recognition/dataset10/train'\n",
    "    VAL_DIR = '/content/visual_speech_recognition/dataset10/val'\n",
    "    TEST_DIR = '/content/visual_speech_recognition/dataset10/test'\n",
    "\n",
    "\n",
    "    train_gen = VideoFrameGenerator(from_dir = TRAIN_DIR,\n",
    "                 rescale=1/255.,\n",
    "                 batch_size=BSIZE,\n",
    "                 shape=(TARGET_SIZE, TARGET_SIZE, 1 if TO_GRAYSCALE else 3),\n",
    "                 nbframe=NBFRAME,\n",
    "                 shuffle=True,\n",
    "                 grayscale = TO_GRAYSCALE,\n",
    "                 transform=data_aug)\n",
    "\n",
    "\n",
    "    test_gen = VideoFrameGenerator(from_dir = TEST_DIR,\n",
    "                 rescale=1/255.,\n",
    "                 batch_size=1,\n",
    "                 shape=(TARGET_SIZE, TARGET_SIZE, 1 if TO_GRAYSCALE else 3),\n",
    "                 nbframe=NBFRAME,\n",
    "                 shuffle=False,\n",
    "                 grayscale = TO_GRAYSCALE)\n",
    "  \n",
    "    val_gen = VideoFrameGenerator(from_dir = VAL_DIR,\n",
    "                 rescale=1/255.,\n",
    "                 batch_size=1,\n",
    "                 shape=(TARGET_SIZE, TARGET_SIZE, 1 if TO_GRAYSCALE else 3),\n",
    "                 nbframe=NBFRAME,\n",
    "                 shuffle=False,\n",
    "                 grayscale = TO_GRAYSCALE,\n",
    "                 transform=data_aug)\n",
    "\n",
    "    return train_gen,val_gen, test_gen\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the create_model() function we implemented again the training function, but now we used hyper parameter.\n",
    "We declared 5 different hyperparameter, those are as follows: activation, dropout, optimizer, hiddenDim, inputDim."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the activation parameter, the function can choose among: relu, elu and tanh.\n",
    "The dropout should be between 0 and 0,4.\n",
    "The optimizer can be selected among, rmsprop, adam, and sgd.\n",
    "For the hidden dimension and the input dimension we gave discret values, because those can't be float numbers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also save the best data for each evaluation into a csv file. The header of this file is mostly the above mentioned hyper parameters, and the validation accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(train_gen, val_gen):\n",
    "\n",
    "    CLASS_NUM = 10\n",
    "\n",
    "    import tensorflow as tf\n",
    "    from tensorflow.keras.layers import Input, Dense, GRU, GlobalAveragePooling2D, MaxPooling1D, Lambda, Bidirectional\n",
    "    from tensorflow.keras.layers import MaxPooling3D, ZeroPadding3D, Conv3D, Conv1D, Activation, BatchNormalization, Dropout\n",
    "    from keras.optimizers import Adam, SGD, RMSprop\n",
    "    from tensorflow.keras import Model, Sequential\n",
    "    from classification_models.keras import Classifiers\n",
    "    from hyperas.distributions import choice, uniform\n",
    "    from hyperas import optim\n",
    "\n",
    "    # hyper parameter ranges\n",
    "    activation = {{choice(['relu', 'elu','tanh'])}}\n",
    "    dropout = {{uniform(0, 0.4)}}\n",
    "    optimizer = {{choice(['rmsprop', 'adam', 'sgd'])}}\n",
    "    hiddenDim = {{choice([20,50,80])}}\n",
    "    inputDim = {{choice([32,64,128])}}\n",
    "\n",
    "    def GRU(x, hidden_size):\n",
    "        # in case of GRU add reset_after=False parameter\n",
    "        out = Bidirectional(keras.layers.GRU(hidden_size, return_sequences=True, kernel_initializer='Orthogonal', reset_after=False, name='gru1'), merge_mode='concat')(x)\n",
    "        out = Bidirectional(keras.layers.GRU(hidden_size, return_sequences=False, kernel_initializer='Orthogonal', reset_after=False, name='gru2'), merge_mode='concat')(out)\n",
    "        return out\n",
    "\n",
    "    def create_Resnet_GRU(mode,  nClasses, frameLen,activation, dropout, optimizer,input_shape=(NBFRAME,TARGET_SIZE,TARGET_SIZE,1),\n",
    "                        every_frame=True, hiddenDim=hiddenDim, inputDim = inputDim):\n",
    "        frontend3D = Sequential([\n",
    "                ZeroPadding3D(padding=(2, 3, 3)),\n",
    "                Conv3D(32, kernel_size=(5, 7, 7), strides=(1, 2, 2), padding='valid', use_bias=False),\n",
    "                BatchNormalization(),\n",
    "                Activation(activation),\n",
    "                ZeroPadding3D(padding=((0, 4, 8))),\n",
    "                MaxPooling3D(pool_size=(1, 2, 3), strides=(1, 1, 2))\n",
    "                ])\n",
    "  \n",
    "        # for temporal convolution\n",
    "        backend_conv1 = Sequential([\n",
    "                Conv1D(2*inputDim, 5, strides=2, use_bias=False),\n",
    "                BatchNormalization(),\n",
    "                Activation(activation),\n",
    "                MaxPooling1D(2, 2),\n",
    "                Conv1D(4*inputDim, 5, strides=2, use_bias=False),\n",
    "                BatchNormalization(),\n",
    "                Activation(activation),\n",
    "                ])\n",
    "  \n",
    "        # for temporal convolution\n",
    "        backend_conv2 = Sequential([\n",
    "                Dense(inputDim),\n",
    "                BatchNormalization(),\n",
    "                Activation(activation),\n",
    "                Dense(nClasses)\n",
    "                ])\n",
    "  \n",
    "        # -----------------------------------------------------------------\n",
    "        # creating the model\n",
    "        input_frames = Input(shape=input_shape, name='frames_input')\n",
    "        x = frontend3D(input_frames)\n",
    "\n",
    "        # reshape output for the input of Resnet 2D\n",
    "        x = Lambda(lambda x : tf.reshape(x, [-1, int(x.shape[2]), int(x.shape[3]), int(x.shape[4])]), name='lambda2')(x)\n",
    "\n",
    "        channels = int(x.shape[-1])\n",
    "  \n",
    "        # get resnet model\n",
    "        ResNet18, preprocess_input = Classifiers.get('resnet18')\n",
    "        resnet18 = ResNet18((None, None, channels), weights=None, include_top=False)\n",
    "\n",
    "        x = resnet18(x)\n",
    "\n",
    "        # Flatten with global average pooling  for the input of the dense layer\n",
    "        x = GlobalAveragePooling2D(name='global_avgpool_resnet')(x)\n",
    "        x = Dense(inputDim, name='dense_resnet')(x)\n",
    "        x = Dropout(dropout)(x)\n",
    "        x = BatchNormalization(name='bn_resnet')(x)\n",
    "\n",
    "        if mode == 'temporalConv':\n",
    "                x = Lambda(lambda x : tf.reshape(x, [-1, frameLen, inputDim]), name='lambda3')(x)   #x.view(-1, frameLen, inputDim)\n",
    "                x = Lambda(lambda x : tf.transpose(x, [0, 2, 1]), name='lambda4')(x)   #x.transpose(1, 2)\n",
    "                x = backend_conv1(x)\n",
    "                x = Lambda(lambda x : tf.reduce_mean(x, 2), name='lambda5')(x)\n",
    "                x = backend_conv2(x)\n",
    "\n",
    "        elif mode == 'backendGRU' or mode == 'finetuneGRU':\n",
    "                x = Lambda(lambda x : tf.reshape(x, [-1, frameLen, inputDim]), name='lambda6')(x)    #x.view(-1, frameLen, inputDim)\n",
    "                # add memory cells to the network (GRU or LSTM)\n",
    "                x = GRU(x, hiddenDim)\n",
    "\n",
    "                # add the dropout layer and the last Dense layer for classification\n",
    "                if every_frame:\n",
    "                    x = Dropout(dropout)(x)\n",
    "                    x = Dense(nClasses, activation='softmax')(x)  # predictions based on every time step\n",
    "                else:\n",
    "                    x = Dropout(dropout)(x)\n",
    "                    x = Dense(nClasses, activation='softmax')(x[:, -1, :])  # predictions based on last time-step\n",
    "\n",
    "                model = Model(inputs=input_frames, outputs=x)\n",
    "\n",
    "        return model\n",
    "\n",
    "    # construct the model\n",
    "    model = create_Resnet_GRU(mode='backendGRU', inputDim=inputDim,hiddenDim = hiddenDim,  nClasses=CLASS_NUM, frameLen=NBFRAME, every_frame=True,\n",
    "                            activation = activation, dropout = dropout, optimizer = optimizer)\n",
    "    # choose the optimizer according to the given string\n",
    "    if optimizer == \"rmsprop\":\n",
    "        optim = keras.optimizers.RMSprop(learning_rate = 0.001, momentum = 0.9)\n",
    "    elif optimizer == \"adam\":\n",
    "        optim = keras.optimizers.Adam(learning_rate = 0.001)\n",
    "    else:\n",
    "        optim = keras.optimizers.SGD(learning_rate = 0.001, momentum = 0.9)\n",
    "\n",
    "    model.compile(optimizer=optim,\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "    callbacks = [keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=3, verbose=0)]\n",
    "  \n",
    "    result = model.fit_generator(train_gen,\n",
    "                epochs=40,\n",
    "                verbose=2,                \n",
    "                validation_data=val_gen,\n",
    "                callbacks = callbacks,\n",
    "                shuffle=True)\n",
    "  \n",
    "    acc = np.amax(result.history['val_accuracy']) \n",
    "\n",
    "    #dropout, activation, optimizer, n_batch\n",
    "    with open('/content/drive/MyDrive/hyperas-log.csv', 'a') as csv_file:\n",
    "        csv_file.write(str(dropout) + ';')\n",
    "        csv_file.write(str(activation) + ';')\n",
    "        csv_file.write(str(optimizer) + ';')\n",
    "        csv_file.write(str(hiddenDim)+';')\n",
    "        csv_file.write(str(inputDim)+';')\n",
    "        csv_file.write(str(64) + ';')   \n",
    "        csv_file.write(str(acc) + '\\n')\n",
    "\n",
    "    return {'loss': -acc, 'status': STATUS_OK, 'model':model}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initializing the CSV file header"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We also added the batch size, but we mostly used 32, because it was the one which gave the best results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize log file\n",
    "with open('/content/drive/MyDrive/hyperas-log.csv', 'w') as csv_file:\n",
    "    csv_file.write('dropout' + ';')\n",
    "    csv_file.write('activation' + ';')\n",
    "    csv_file.write('optimizer' + ';')\n",
    "    csv_file.write('hiddenDim' + ';')\n",
    "    csv_file.write('inputDim' + ';')\n",
    "    csv_file.write('n_batch' + ';') \n",
    "    csv_file.write('acc' + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing the moduls for the optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import hyperas\n",
    "from hyperopt import Trials, STATUS_OK, tpe\n",
    "from hyperas import optim\n",
    "from hyperas.distributions import choice, uniform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_run, best_model = optim.minimize(\n",
    "     model=create_model,\n",
    "     data=data,\n",
    "     algo=tpe.suggest,\n",
    "     max_evals=100,\n",
    "     notebook_name='hyperparameter_opt',\n",
    "     trials= Trials())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization of the result "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/content/drive/MyDrive/hyperas-log.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-45c22a41037e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpandas\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mlog_file\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpandas\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'/content/drive/MyDrive/hyperas-log.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdelimiter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m';'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mbest10\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlog_file\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mby\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'acc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mbest10\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    684\u001b[0m     )\n\u001b[0;32m    685\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 686\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    687\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    688\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    450\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    451\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 452\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    453\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    454\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    934\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    935\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 936\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    937\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    938\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1166\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"c\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1167\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"c\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1168\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1169\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1170\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"python\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1996\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"usecols\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1997\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1998\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1999\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2000\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/drive/MyDrive/hyperas-log.csv'"
     ]
    }
   ],
   "source": [
    "import pandas\n",
    "log_file = pandas.read_csv('/content/drive/MyDrive/hyperas-log.csv', delimiter=';')\n",
    "\n",
    "best10 = log_file.sort_values(by=['acc'], ascending=False).head(n=10)\n",
    "best10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drawing heatmaps, one for the hidden dimensions, and one representing the input dimension changes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "max_val_acc = log_file.groupby(['hiddenDim', 'optimizer']).max()\n",
    "max_val_acc = max_val_acc.unstack()[['acc']]\n",
    "sns.heatmap(max_val_acc.acc, annot=True, fmt='.4g');\n",
    "\n",
    "b, t = plt.ylim()\n",
    "b += 0.5\n",
    "t -= 0.5\n",
    "plt.ylim(b, t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_val_acc = log_file.groupby(['inputDim', 'optimizer']).max()\n",
    "max_val_acc = max_val_acc.unstack()[['acc']]\n",
    "sns.heatmap(max_val_acc.acc, annot=True, fmt='.4g');\n",
    "\n",
    "b, t = plt.ylim()\n",
    "b += 0.5\n",
    "t -= 0.5\n",
    "plt.ylim(b, t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}