GEMS ( Neural Network).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GEMS (Neural Network) Demo\n",
    "\n",
    "A brief demonstration of GEMS for neural networks, applied to MNIST (5 nodes) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "# Import relevant libraries \n",
    "import os\n",
    "\n",
    "from utils.data_utils import *\n",
    "from utils.model_utils import *\n",
    "from utils.dnn3 import *\n",
    "\n",
    "%matplotlib notebook\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set experiment configurations\n",
    "exp = {\n",
    "    \"dataset\": 'mnist',\n",
    "    \"num_nodes\": 5, \n",
    "    \"model\": \"dnn2\",\n",
    "    \"local_epochs\": 40,\n",
    "    \"global_epochs\": 50,\n",
    "    \"dropout\": 0.5,\n",
    "    \"hidden\": 50,\n",
    "    \"secondary_epochs\": 10,\n",
    "    \"optimizer\": \"adam\", \n",
    "    \"loss\": 'categorical_crossentropy',\n",
    "    'final_epsilon': 0.70,\n",
    "    'max_radius': 4000.0, \n",
    "    'num_samples': 500, \n",
    "    'fisher_prop': 1.0,\n",
    "    'fisher_count': 1,\n",
    "    'ellipse': True,\n",
    "    'delta': 0.5,\n",
    "    'ellipse_type': 'fisher',\n",
    "    \"hidden_slack\": 1.0,\n",
    "    \"k\": 75,\n",
    "    'holdout_trials': 5,\n",
    "    'holdout_sizes': [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data \n",
    "data_dir = os.path.join(\"../datasets\", \"%s_%d\" % (exp['dataset'], exp['num_nodes']))\n",
    "Xtr, ytr, Xval, yval, Xts, yts = load_data(data_dir)\n",
    "Xts_flat, yts_flat = flatten(Xts, yts)\n",
    "Xtr_flat, ytr_flat = flatten(Xtr, ytr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training local model on node-0...\n",
      "Training local model on node-1...\n",
      "Training local model on node-2...\n",
      "Training local model on node-3...\n",
      "Training local model on node-4...\n"
     ]
    }
   ],
   "source": [
    "# Train local models \n",
    "all_results = {}\n",
    "models = []\n",
    "for t in range(exp['num_nodes']):\n",
    "    print(\"Training local model on node-%d...\" % t)\n",
    "    model, results = train_local_model(Xtr, ytr, Xts, yts, t, exp)\n",
    "    models.append(model)\n",
    "\n",
    "    all_results[t] = results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating good-enough spaces.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02c1efbf96d94b31b0a33f74a334a4c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=50), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of hidden neurons: 74\n",
      "Training new logistic models\n",
      "Pure GEMS accuracy: 0.453400\n",
      "Holdout Size: 100.\n",
      "\tRaw: 0.422720.\n",
      "\tGems (Tuned): 0.701960.\n",
      "\tAverage (Tuned): 0.469520.\n",
      "\t Node-0 Loc Model (Tuned): 0.227480\n",
      "\t Node-1 Loc Model (Tuned): 0.207440\n",
      "\t Node-2 Loc Model (Tuned): 0.182880\n",
      "\t Node-3 Loc Model (Tuned): 0.195880\n",
      "\t Node-4 Loc Model (Tuned): 0.195440\n",
      "Holdout Size: 200.\n",
      "\tRaw: 0.587400.\n",
      "\tGems (Tuned): 0.788080.\n",
      "\tAverage (Tuned): 0.636560.\n",
      "\t Node-0 Loc Model (Tuned): 0.309400\n",
      "\t Node-1 Loc Model (Tuned): 0.200440\n",
      "\t Node-2 Loc Model (Tuned): 0.196280\n",
      "\t Node-3 Loc Model (Tuned): 0.197880\n",
      "\t Node-4 Loc Model (Tuned): 0.165160\n",
      "Holdout Size: 300.\n",
      "\tRaw: 0.719000.\n",
      "\tGems (Tuned): 0.827160.\n",
      "\tAverage (Tuned): 0.694560.\n",
      "\t Node-0 Loc Model (Tuned): 0.326120\n",
      "\t Node-1 Loc Model (Tuned): 0.199400\n",
      "\t Node-2 Loc Model (Tuned): 0.211640\n",
      "\t Node-3 Loc Model (Tuned): 0.205560\n",
      "\t Node-4 Loc Model (Tuned): 0.154360\n",
      "Holdout Size: 400.\n",
      "\tRaw: 0.762240.\n",
      "\tGems (Tuned): 0.846760.\n",
      "\tAverage (Tuned): 0.737840.\n",
      "\t Node-0 Loc Model (Tuned): 0.344800\n",
      "\t Node-1 Loc Model (Tuned): 0.220200\n",
      "\t Node-2 Loc Model (Tuned): 0.227640\n",
      "\t Node-3 Loc Model (Tuned): 0.278240\n",
      "\t Node-4 Loc Model (Tuned): 0.140680\n",
      "Holdout Size: 500.\n",
      "\tRaw: 0.786240.\n",
      "\tGems (Tuned): 0.846760.\n",
      "\tAverage (Tuned): 0.753120.\n",
      "\t Node-0 Loc Model (Tuned): 0.444560\n",
      "\t Node-1 Loc Model (Tuned): 0.233160\n",
      "\t Node-2 Loc Model (Tuned): 0.246120\n",
      "\t Node-3 Loc Model (Tuned): 0.457400\n",
      "\t Node-4 Loc Model (Tuned): 0.132120\n",
      "Holdout Size: 600.\n",
      "\tRaw: 0.813520.\n",
      "\tGems (Tuned): 0.858320.\n",
      "\tAverage (Tuned): 0.763080.\n",
      "\t Node-0 Loc Model (Tuned): 0.473840\n",
      "\t Node-1 Loc Model (Tuned): 0.296000\n",
      "\t Node-2 Loc Model (Tuned): 0.321160\n",
      "\t Node-3 Loc Model (Tuned): 0.436560\n",
      "\t Node-4 Loc Model (Tuned): 0.188040\n",
      "Holdout Size: 700.\n",
      "\tRaw: 0.834200.\n",
      "\tGems (Tuned): 0.858600.\n",
      "\tAverage (Tuned): 0.763000.\n",
      "\t Node-0 Loc Model (Tuned): 0.559120\n",
      "\t Node-1 Loc Model (Tuned): 0.271200\n",
      "\t Node-2 Loc Model (Tuned): 0.453880\n",
      "\t Node-3 Loc Model (Tuned): 0.532160\n",
      "\t Node-4 Loc Model (Tuned): 0.140720\n",
      "Holdout Size: 800.\n",
      "\tRaw: 0.837000.\n",
      "\tGems (Tuned): 0.870720.\n",
      "\tAverage (Tuned): 0.766480.\n",
      "\t Node-0 Loc Model (Tuned): 0.535600\n",
      "\t Node-1 Loc Model (Tuned): 0.293040\n",
      "\t Node-2 Loc Model (Tuned): 0.443800\n",
      "\t Node-3 Loc Model (Tuned): 0.580400\n",
      "\t Node-4 Loc Model (Tuned): 0.192720\n",
      "Holdout Size: 900.\n",
      "\tRaw: 0.842040.\n",
      "\tGems (Tuned): 0.880320.\n",
      "\tAverage (Tuned): 0.773440.\n",
      "\t Node-0 Loc Model (Tuned): 0.572240\n",
      "\t Node-1 Loc Model (Tuned): 0.386680\n",
      "\t Node-2 Loc Model (Tuned): 0.491880\n",
      "\t Node-3 Loc Model (Tuned): 0.592560\n",
      "\t Node-4 Loc Model (Tuned): 0.217480\n",
      "Holdout Size: 1000.\n",
      "\tRaw: 0.848320.\n",
      "\tGems (Tuned): 0.872640.\n",
      "\tAverage (Tuned): 0.778400.\n",
      "\t Node-0 Loc Model (Tuned): 0.644080\n",
      "\t Node-1 Loc Model (Tuned): 0.418760\n",
      "\t Node-2 Loc Model (Tuned): 0.448240\n",
      "\t Node-3 Loc Model (Tuned): 0.620040\n",
      "\t Node-4 Loc Model (Tuned): 0.210680\n"
     ]
    }
   ],
   "source": [
    "# Compute the GEMS model \n",
    "gems_model, results = run_dnn_gems(exp, Xtr, ytr, Xval, yval, Xts, yts, models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}