CrayHPO_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Distributed hyper-parameter optimization with the Cray HPO library\n",
    "\n",
    "Some documentation here:\n",
    "https://pubs.cray.com/content/S-2589/1.2.UP00/xctm-series-urika-xc-analytic-applications-guide/hyperparameter-optimization-hpo-support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/global/cscratch1/sd/bja')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Externals\n",
    "from crayai import hpo\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define hyperparameters\n",
    "params = hpo.Params([['--h1', 4, [4, 8, 16]],\n",
    "                     ['--h2', 8, [8, 16, 32]],\n",
    "                     ['--h3', 16, [16, 32, 64]],\n",
    "                     ['--dropout', 0.2, (0., 1.)],\n",
    "                     ['--optimizer', 'Adam', ['Adam', 'Nadam']]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training config\n",
    "n_epochs = 4\n",
    "\n",
    "# Job config\n",
    "n_nodes = 32\n",
    "alloc_args = '-q interactive -C haswell -t 2:00:00'\n",
    "\n",
    "# Optimizer config\n",
    "pop_size = 4\n",
    "num_demes = 4\n",
    "generations = 16\n",
    "log_file = 'results.log'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the evaluator\n",
    "cmd = 'python train.py --epochs %i' % (n_epochs)\n",
    "evaluator = hpo.Evaluator(cmd, nodes=n_nodes, alloc_args=alloc_args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the optimizer\n",
    "optimizer = hpo.genetic.Optimizer(evaluator,\n",
    "                                  pop_size=pop_size,\n",
    "                                  num_demes=num_demes,\n",
    "                                  generations=generations,\n",
    "                                  log_fn=log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run the optimizer over the hyperparameters\n",
    "optimizer.optimize(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#generation</th>\n",
       "      <th>epoch</th>\n",
       "      <th>best_fom</th>\n",
       "      <th>avg_fom</th>\n",
       "      <th>checkpoint_in</th>\n",
       "      <th>checkpoint_out</th>\n",
       "      <th>--h1</th>\n",
       "      <th>--h2</th>\n",
       "      <th>--h3</th>\n",
       "      <th>--dropout</th>\n",
       "      <th>--optimizer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.073547</td>\n",
       "      <td>0.115834</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>64</td>\n",
       "      <td>0.118847</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.058352</td>\n",
       "      <td>0.086159</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.118879</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.058352</td>\n",
       "      <td>0.086819</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.118879</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.056483</td>\n",
       "      <td>0.082912</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.227348</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0.056483</td>\n",
       "      <td>0.090415</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.227348</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.052548</td>\n",
       "      <td>0.078360</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.091499</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>0.052548</td>\n",
       "      <td>0.075095</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.091499</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>0.047567</td>\n",
       "      <td>0.075947</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.082243</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>0.047567</td>\n",
       "      <td>0.091844</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16</td>\n",
       "      <td>32</td>\n",
       "      <td>64</td>\n",
       "      <td>0.082243</td>\n",
       "      <td>Nadam</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #generation  epoch  best_fom   avg_fom  checkpoint_in  checkpoint_out  \\\n",
       "0            0      1  0.073547  0.115834            NaN             NaN   \n",
       "1            1      2  0.058352  0.086159            NaN             NaN   \n",
       "2            2      3  0.058352  0.086819            NaN             NaN   \n",
       "3            3      4  0.056483  0.082912            NaN             NaN   \n",
       "4            4      5  0.056483  0.090415            NaN             NaN   \n",
       "5            5      6  0.052548  0.078360            NaN             NaN   \n",
       "6            6      7  0.052548  0.075095            NaN             NaN   \n",
       "7            7      8  0.047567  0.075947            NaN             NaN   \n",
       "8            8      9  0.047567  0.091844            NaN             NaN   \n",
       "\n",
       "   --h1  --h2  --h3  --dropout --optimizer  \n",
       "0    16    16    64   0.118847       Nadam  \n",
       "1     4    32    64   0.118879       Nadam  \n",
       "2     4    32    64   0.118879       Nadam  \n",
       "3    16    32    64   0.227348       Nadam  \n",
       "4    16    32    64   0.227348       Nadam  \n",
       "5    16    32    64   0.091499       Nadam  \n",
       "6    16    32    64   0.091499       Nadam  \n",
       "7    16    32    64   0.082243       Nadam  \n",
       "8    16    32    64   0.082243       Nadam  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(log_file, delim_whitespace=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow-intel(cpu)/1.13.1-py36",
   "language": "python",
   "name": "tensorflow_intel_1.13.1_py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}