MNIST_SaveActivations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import keras.backend as K\n",
    "import numpy as np\n",
    "\n",
    "import utils\n",
    "import loggingreporter \n",
    "\n",
    "cfg = {}\n",
    "cfg['SGD_BATCHSIZE']    = 128\n",
    "cfg['SGD_LEARNINGRATE'] = 0.001\n",
    "cfg['NUM_EPOCHS']       = 10000\n",
    "\n",
    "#cfg['ACTIVATION'] = 'relu'\n",
    "cfg['ACTIVATION'] = 'tanh'\n",
    "# How many hidden neurons to put into each of the layers\n",
    "cfg['LAYER_DIMS'] = [1024, 20, 20, 20]\n",
    "#cfg['LAYER_DIMS'] = [32, 28, 24, 20, 16, 12, 8, 8]\n",
    "#cfg['LAYER_DIMS'] = [128, 64, 32, 16, 16] # 0.967 w. 128\n",
    "#cfg['LAYER_DIMS'] = [20, 20, 20, 20, 20, 20] # 0.967 w. 128\n",
    "ARCH_NAME =  '-'.join(map(str,cfg['LAYER_DIMS']))\n",
    "trn, tst = utils.get_mnist()\n",
    "\n",
    "# Where to save activation and weights data\n",
    "cfg['SAVE_DIR'] = 'rawdata/' + cfg['ACTIVATION'] + '_' + ARCH_NAME "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_layer  = keras.layers.Input((trn.X.shape[1],))\n",
    "clayer = input_layer\n",
    "for n in cfg['LAYER_DIMS']:\n",
    "    clayer = keras.layers.Dense(n, activation=cfg['ACTIVATION'])(clayer)\n",
    "output_layer = keras.layers.Dense(trn.nb_classes, activation='softmax')(clayer)\n",
    "\n",
    "model = keras.models.Model(inputs=input_layer, outputs=output_layer)\n",
    "optimizer = keras.optimizers.SGD(lr=cfg['SGD_LEARNINGRATE'])\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def do_report(epoch):\n",
    "    # Only log activity for some epochs.  Mainly this is to make things run faster.\n",
    "    if epoch < 20:       # Log for all first 20 epochs\n",
    "        return True\n",
    "    elif epoch < 100:    # Then for every 5th epoch\n",
    "        return (epoch % 5 == 0)\n",
    "    elif epoch < 200:    # Then every 10th\n",
    "        return (epoch % 10 == 0)\n",
    "    else:                # Then every 100th\n",
    "        return (epoch % 100 == 0)\n",
    "    \n",
    "reporter = loggingreporter.LoggingReporter(cfg=cfg, \n",
    "                                          trn=trn, \n",
    "                                          tst=tst, \n",
    "                                          do_save_func=do_report)\n",
    "r = model.fit(x=trn.X, y=trn.Y, \n",
    "              verbose    = 2, \n",
    "              batch_size = cfg['SGD_BATCHSIZE'],\n",
    "              epochs     = cfg['NUM_EPOCHS'],\n",
    "              # validation_data=(tst.X, tst.Y),\n",
    "              callbacks  = [reporter,])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}