SlackMinimizerRunner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We illustrate the use of SlackMinimizer in two ways."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The script slack_minimizer.py can be run directly, showing the results on a random forest (see source comments around main() for details)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 0:\tTime = 0.000204086303711\n",
      "Data loaded. \tTime = 1.37143802643\n",
      "Random forest trained. \tTime = 1.5565161705\n",
      "Classifier 1 done.\n",
      "Classifier 2 done.\n",
      "Classifier 3 done.\n",
      "Classifier 4 done.\n",
      "Classifier 5 done.\n",
      "Classifier 6 done.\n",
      "Classifier 7 done.\n",
      "Classifier 8 done.\n",
      "Classifier 9 done.\n",
      "Classifier 10 done.\n",
      "Classifier 11 done.\n",
      "Classifier 12 done.\n",
      "Classifier 13 done.\n",
      "Classifier 14 done.\n",
      "Classifier 15 done.\n",
      "Classifier 16 done.\n",
      "Classifier 17 done.\n",
      "Classifier 18 done.\n",
      "Classifier 19 done.\n",
      "Classifier 20 done.\n",
      "Featurizing done. \tTime = 4.65109705925\n",
      "After iteration  0:\t Time = 0.103230953217\n",
      "Holdout: \t Error = 0.165342673718\t AUC: 0.778885440402\n",
      "Validation: \t Error = 0.203005217566\t AUC: 0.761232933181\n",
      "After iteration  5:\t Time = 0.548645019531\n",
      "Holdout: \t Error = 0.183751156667\t AUC: 0.857158992693\n",
      "Validation: \t Error = 0.218303067563\t AUC: 0.827456717138\n",
      "After iteration  10:\t Time = 0.921526908875\n",
      "Holdout: \t Error = 0.190210443311\t AUC: 0.874499096409\n",
      "Validation: \t Error = 0.226304882028\t AUC: 0.842281474332\n",
      "After iteration  15:\t Time = 1.23883891106\n",
      "Holdout: \t Error = 0.18840128767\t AUC: 0.877588001886\n",
      "Validation: \t Error = 0.226545434278\t AUC: 0.844666807424\n",
      "After iteration  20:\t Time = 1.56355190277\n",
      "Holdout: \t Error = 0.186927159571\t AUC: 0.876635302899\n",
      "Validation: \t Error = 0.225980051756\t AUC: 0.842467475016\n",
      "After iteration  25:\t Time = 1.92295503616\n",
      "Holdout: \t Error = 0.186089355626\t AUC: 0.873737919384\n",
      "Validation: \t Error = 0.2256462137\t AUC: 0.842266393195\n",
      "After iteration  30:\t Time = 2.27593302727\n",
      "Holdout: \t Error = 0.185410739563\t AUC: 0.873988371179\n",
      "Validation: \t Error = 0.22514586573\t AUC: 0.842525286039\n",
      "After iteration  35:\t Time = 2.66373586655\n",
      "Holdout: \t Error = 0.18505526463\t AUC: 0.874125874126\n",
      "Validation: \t Error = 0.224855254893\t AUC: 0.842550421266\n",
      "Final validation AUC:\t0.842666043313\n",
      "Written to files:\tlogs/muf_ssl_1000_tailprob0.01_auc.csv,\t logs/muf_ssl_1000_tailprob0.01_err.csv\n"
     ]
    }
   ],
   "source": [
    "%run slack_minimizer.py data/a1a_all.csv 1000 20000 -f 0.01 -k -1 -d 40 --tree_node_specialists"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We also give an example in which many heterogeneous non-tree classifiers are combined with SlackMinimizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import composite_feature\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import sklearn.linear_model, sklearn.ensemble\n",
    "import muffled_utils\n",
    "import time\n",
    "import sklearn.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data loaded. \tTime = 0.768465995789\n"
     ]
    }
   ],
   "source": [
    "labeled_file = 'data/a1a_all.csv'\n",
    "labeled_set_size = 1000\n",
    "unlabeled_set_size = 10000\n",
    "holdout_set_size = 500\n",
    "validation_set_size = 1000\n",
    "inittime = time.time()\n",
    "(x_train, y_train, x_unl, y_unl, x_out, y_out, x_validate, y_validate) = muffled_utils.read_random_data_from_csv(\n",
    "    labeled_file, labeled_set_size, unlabeled_set_size, holdout_set_size, validation_set_size)\n",
    "print('Data loaded. \\tTime = ' + str(time.time() - inittime))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Plain RF trained', 0.279616117477417)\n",
      "('DT trained', 0.28757500648498535)\n",
      "('AdaBoost trained', 0.42799901962280273)\n",
      "('LogitBoost trained', 0.5897660255432129)\n",
      "('Logistic regression trained', 0.6009221076965332)\n"
     ]
    }
   ],
   "source": [
    "# Now train a few different base classifiers\n",
    "inittime = time.time()\n",
    "skcl = []\n",
    "clrf = sklearn.ensemble.RandomForestClassifier(n_estimators=50, n_jobs=-1)\n",
    "skcl.append(('Plain RF', clrf))\n",
    "cldt = sklearn.tree.DecisionTreeClassifier()\n",
    "skcl.append(('DT', cldt))\n",
    "cletf = sklearn.ensemble.AdaBoostClassifier(n_estimators=50, algorithm='SAMME')\n",
    "skcl.append(('AdaBoost', cletf))\n",
    "clgb = sklearn.ensemble.GradientBoostingClassifier(n_estimators=50)#, loss='deviance')\n",
    "skcl.append(('LogitBoost', clgb))\n",
    "cllogistic = sklearn.linear_model.LogisticRegression()#(loss='log')\n",
    "skcl.append(('Logistic regression', cllogistic))\n",
    "#clgp = sklearn.gaussian_process.GaussianProcessClassifier()\n",
    "#skcl.append(('Gaussian process', clgp))\n",
    "# Now x_train is a (LABELED_SET_SIZE x d) matrix, and y_train a vector of size LABELED_SET_SIZE.\n",
    "for i in range(len(skcl)):\n",
    "    skcl[i][1].fit(x_train, y_train)\n",
    "    print(skcl[i][0] + ' trained', time.time() - inittime)\n",
    "classifier_list = list(zip(*skcl)[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.73201784024868233, 0.69135018245708879, 0.74608730909582355, 0.73031490741992167, 0.75761589403973517]\n"
     ]
    }
   ],
   "source": [
    "print [sklearn.metrics.roc_auc_score(y_validate, c.predict(x_validate)) for c in classifier_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classifier 1 done.\n",
      "Classifier 2 done.\n",
      "Classifier 3 done.\n",
      "Classifier 4 done.\n",
      "Classifier 5 done.\n",
      "Featurizing done. \tTime = 1.45948219299\n"
     ]
    }
   ],
   "source": [
    "k = 0\n",
    "failure_prob = 0.0005\n",
    "inittime = time.time()\n",
    "(b_vector, allfeats_out, allfeats_unl, allfeats_val) = composite_feature.predict_multiple(\n",
    "    classifier_list, x_out, x_unl, x_validate, y_out=y_out, k=k, \n",
    "    failure_prob=failure_prob, from_sklearn_rf=False, use_tree_partition=False)\n",
    "print ('Featurizing done. \\tTime = ' + str(time.time() - inittime))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After iteration  0:\t Time = 0.00942802429199\n",
      "Holdout: \t Error = 0.317332968464\t AUC: 0.749843674008\n",
      "Validation: \t Error = 0.325109400266\t AUC: 0.747871333964\n",
      "After iteration  5:\t Time = 0.0521280765533\n",
      "Holdout: \t Error = 0.19193955162\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.202221024037\t AUC: 0.793231517773\n",
      "After iteration  10:\t Time = 0.111564159393\n",
      "Holdout: \t Error = 0.191931702334\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.202212941786\t AUC: 0.793231517773\n",
      "After iteration  15:\t Time = 0.121046066284\n",
      "Holdout: \t Error = 0.191909829871\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.202191181511\t AUC: 0.793231517773\n",
      "After iteration  20:\t Time = 0.129214048386\n",
      "Holdout: \t Error = 0.191903260385\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.20218490245\t AUC: 0.793231517773\n",
      "After iteration  25:\t Time = 0.137324094772\n",
      "Holdout: \t Error = 0.191911789491\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.20219210679\t AUC: 0.793231517773\n",
      "After iteration  30:\t Time = 0.146389961243\n",
      "Holdout: \t Error = 0.191900347907\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.202180607701\t AUC: 0.793231517773\n",
      "After iteration  35:\t Time = 0.156490087509\n",
      "Holdout: \t Error = 0.19189377842\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.20217432864\t AUC: 0.793231517773\n",
      "After iteration  40:\t Time = 0.165151119232\n",
      "Holdout: \t Error = 0.191908383339\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.202187320726\t AUC: 0.793231517773\n",
      "After iteration  45:\t Time = 0.173420190811\n",
      "Holdout: \t Error = 0.19189121858\t AUC: 0.806121031057\n",
      "Validation: \t Error = 0.202170318835\t AUC: 0.793231517773\n"
     ]
    }
   ],
   "source": [
    "import slack_minimizer\n",
    "gradh = slack_minimizer.SlackMinimizer(\n",
    "    b_vector, allfeats_unl, allfeats_out, y_out, unlabeled_labels=y_unl,\n",
    "    validation_set=allfeats_val, validation_labels=y_validate)\n",
    "statauc = gradh.sgd(50, unl_stride_size=100, linesearch=True, logging_interval=5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}