Chapter 4 - Evaluation and Optimization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 4 - Evaluation and Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We generate two inputs:\n",
    "* features – a matrix of input features\n",
    "* target – an array of target variables corresponding to those features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "features = rand(100,5)\n",
    "target = rand(100) > 0.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The holdout method\n",
    "\n",
    "We divide into a randomized training and test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/henrik/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:10: DeprecationWarning: using a non-integer number instead of an integer will result in an error in the future\n",
      "/Users/henrik/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py:11: DeprecationWarning: using a non-integer number instead of an integer will result in an error in the future\n"
     ]
    }
   ],
   "source": [
    "N = features.shape[0]\n",
    "N_train = floor(0.7 * N)\n",
    "\n",
    "# Randomize index\n",
    "# Note: sometimes you want to retain the order in the dataset and skip this step\n",
    "# E.g. in the case of time-based datasets where you want to test on 'later' instances\n",
    "idx = random.permutation(N)\n",
    "\n",
    "# Split index\n",
    "idx_train = idx[:N_train]\n",
    "idx_test = idx[N_train:]\n",
    "\n",
    "# Break your data into training and testing subsets\n",
    "features_train = features[idx_train,:]\n",
    "target_train = target[idx_train]\n",
    "features_test = features[idx_test,:]\n",
    "target_test = target[idx_test]\n",
    "\n",
    "# Build, predict, evaluate (to be filled out)\n",
    "# model = train(features_train, target_train)\n",
    "# preds_test = predict(model, features_test)\n",
    "# accuracy = evaluate_acc(preds_test, target_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(70, 5)\n",
      "(30, 5)\n",
      "(70,)\n",
      "(30,)\n"
     ]
    }
   ],
   "source": [
    "print features_train.shape\n",
    "print features_test.shape\n",
    "print target_train.shape\n",
    "print target_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-fold cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[16 23 33 34 35 39 44 48 99]\n",
      "[ 3 10 29 37 45 52 54 62 68 83 98]\n",
      "[ 4 12 20 22 28 50 51 53 55 56 59 60 64 77 84 86 92 97]\n",
      "[ 6  7  9 11 18 27 41 49 57 61 70 73 80]\n",
      "[42 67 96]\n",
      "[30 63 66 78 90 91]\n",
      "[ 2  5 26 40 46 58 72 74 87 95]\n",
      "[ 0  8 14 24 32 36 38 43 85 88 89]\n",
      "[ 1 13 17 19 21 47 71 79 81 82 93 94]\n",
      "[15 25 31 65 69 75 76]\n"
     ]
    }
   ],
   "source": [
    "N = features.shape[0]\n",
    "K = 10 # number of folds\n",
    "\n",
    "preds_kfold = np.empty(N)\n",
    "folds = np.random.randint(0, K, size=N)\n",
    "\n",
    "for idx in np.arange(K):\n",
    "\n",
    "    # For each fold, break your data into training and testing subsets\n",
    "    features_train = features[folds != idx,:]\n",
    "    target_train = target[folds != idx]\n",
    "    features_test = features[folds == idx,:]\n",
    "    \n",
    "    # Print the indices in each fold, for inspection\n",
    "    print nonzero(folds == idx)[0]\n",
    "\n",
    "    # Build and predict for CV fold (to be filled out)\n",
    "    # model = train(features_train, target_train)\n",
    "    # preds_kfold[folds == idx] = predict(model, features_test)\n",
    "    \n",
    "# accuracy = evaluate_acc(preds_kfold, target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def roc_curve(true_labels, predicted_probs, n_points=100, pos_class=1):\n",
    "    thr = linspace(0,1,n_points)\n",
    "    tpr = zeros(n_points)\n",
    "    fpr = zeros(n_points)\n",
    "\n",
    "    pos = true_labels == pos_class\n",
    "    neg = logical_not(pos)\n",
    "    n_pos = count_nonzero(pos)\n",
    "    n_neg = count_nonzero(neg)\n",
    "      \n",
    "    for i,t in enumerate(thr):\n",
    "        tpr[i] = count_nonzero(logical_and(predicted_probs >= t, pos)) / n_pos\n",
    "        fpr[i] = count_nonzero(logical_and(predicted_probs >= t, neg)) / n_neg\n",
    "    \n",
    "    return fpr, tpr, thr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x10595f150>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFm9JREFUeJzt3X9QlOeBwPHnDbu5MRkTNVhGd7dDwm5hkbDYgmhyJmvT\nDIS0OKPphbTnD8JQxqnx0unNmaYzETqjkda5aRpmCOTQ5qfaSzIDueJmqnWb1l9EQEwUncWDdtm0\njKicmZhxAd/7gxDfbGB32R/s++P7mXlnWHny7uMzzOPXh1ciybIsAAD6ckuqJwAASDw2dwDQITZ3\nANAhNncA0CE2dwDQITZ3ANChiJv7k08+uTsjI2Po3nvv/XC6MVu2bPmNw+HwuVyunu7u7qWJnSIA\nYKYibu6VlZV7PB5P6XSfb29vL+vr67P7fD5Hc3PzjzZt2tSY2CkCAGYq4ua+cuXKP8+fP//KdJ9v\na2sr37BhwytCCFFcXHxiZGRk3tDQUEYiJwkAmJm4z9wDgYDFZrP5J19brdbBwcFBa7z3BQDEzpSI\nm8iyLClfS5L0lZ9pMNWvAQAiC91joxF3uVssloDf77dNvh4cHLRaLJbAVGNlWeaSZbFt27aUz0Et\nF2vBWrAWN6+uLlnk58vi0UdlEQhM/Fqs4t7cy8vL21599dX1Qghx/Pjx5fPmzRvJyMgYive+AGAU\nwaAQ27YJUVIixE9/KsS77wqxeHF894x4LPPEE0/s/dOf/vTg8PBwus1m89fV1W0bHR01CyFETU1N\nU1lZWXt7e3uZ3W7vu/322z/ds2dPZXxTAgDj6O4WYuNGIWw2IU6din9TnyTFk/0zeiNJkmfrvdTO\n6/UKt9ud6mmoAmtxE2txkxHWIhgUYvt2IRobhdi1S4h164SQpjhZlyRJyDGcubO5A8AsU9Z6c3P4\nWo91c+fHDwDALEnG2fp0EvIoJAAgvGSdrU+HcgeAJJrNWlei3AEgSWa71pUodwBIsFTVuhLlDgAJ\nlMpaV6LcASAB1FDrSpQ7AMRJLbWuRLkDQIzUVutKlDsAxECNta5EuQPADKi51pUodwCIktprXYly\nB4AItFLrSpQ7AIShpVpXotwBYAparHUlyh0AQmi11pUodwD4nNZrXYlyBwChj1pXotwBGJqeal2J\ncgdgWHqrdSXKHYDh6LXWlSh3AIai51pXotwBGIIRal2Jcgege0apdSXKHYBuGa3WlSh3ALpkxFpX\notwB6IqRa12JcgegG0avdSXKHYDmUetfRbkD0DRqfWqUOwBNotbDo9wBaA61HhnlDkAzqPXoUe4A\nNIFanxnKHYCqUeuxibi5ezye0pycnHMOh8NXX1+/NfTzw8PD6aWlpZ6CgoJTeXl5H/32t7/dmJSZ\nAjCc7m4hioqE6OycqPX164WQpFTPShskWZan/eT4+Hhadnb2+YMHD37HYrEEioqKPti7d+8TTqez\nd3JMbW1t7fXr1//p+eef/9nw8HB6dnb2+aGhoQyTyTT2pTeSJDncewHApGBQiO3bhWhsFGLXLiHW\nrTPupi5JkpBleca/+7Dl3tHRscxut/dlZmYOmM3m0YqKin2tra2rlWMWLVr096tXr94hhBBXr169\n46677roUurEDQLSo9cQI+w3VQCBgsdls/snXVqt18MSJE8XKMdXV1S9/+9vf/uPixYs//uSTT+b+\n7ne/+5fp7ldbW/vFx263W7jd7pgnDkBfqPUJXq9XeL3euO8TdnOXJCniOcqOHTueLSgoOOX1et0X\nLlzIevjhh//Q09Pjmjt37iehY5WbOwBM4kmYm0LDt66uLqb7hD2WsVgsAb/fb5t87ff7bVardVA5\n5ujRo/d9//vf/28hhMjKyrpw9913958/fz47ptkAMBSehEmesJt7YWHhSZ/P5xgYGMgMBoO37t+/\n//Hy8vI25ZicnJxzBw8e/I4QQgwNDWWcP38++5577vnfZE4agPZxtp5cYY9lTCbTWENDw+aSkpL3\nxsfH06qqqlqcTmdvU1NTjRBC1NTUND377LM7Kisr97hcrp4bN27c8stf/vI/FixYcHl2pg9Aazhb\nnx1hH4VM6BvxKCRgeMqz9eZmjmCikZRHIQEgEThbn338bBkAScWTMKlBuQNICmo9tSh3AAlHrace\n5Q4gYah19aDcASQEta4ulDuAuFDr6kS5A4gZta5elDuAGaPW1Y9yBzAj1Lo2UO4AokKtawvlDiAi\nal17KHcA06LWtYtyBzAlal3bKHcAX0Kt6wPlDuAL1Lp+UO4AqHUdotwBg6PW9YlyBwyKWtc3yh0w\nIGpd/yh3wECodeOg3AGDoNaNhXIHdI5aNybKHdAxat24KHdAh6h1UO6AzlDrEIJyB3SDWocS5Q7o\nALWOUJQ7oGHUOqZDuQMaRa0jHMod0BhqHdGg3AENodYRLcod0ABqHTNFuQMqR60jFpQ7oFLUOuIR\ncXP3eDylOTk55xwOh6++vn7rVGO8Xq976dKl3Xl5eR+53W5vwmcJGEx3txBFRUJ0dk7U+vr1QkhS\nqmcFTZFledprbGwsLSsrq6+/vz8zGAyaXS7XqbNnzzqVY65cuTIvNzf3jN/vt8qyLC5evJg+1b0m\n3gpAONevy/Jzz8nywoWy/MorsnzjRqpnhFT7fO8Mu1dPdYUt946OjmV2u70vMzNzwGw2j1ZUVOxr\nbW1drRzz5ptv/mDt2rVvW63WQSGESE9PH07an0SAjlHrSKSwm3sgELDYbDb/5Gur1ToYCAQsyjE+\nn89x+fLlBatWrTpcWFh48rXXXluXrMkCesTZOpIh7NMykiTJkW4wOjpq7urq+uahQ4ceunbt2m0r\nVqw4tnz58uMOh8MXOra2tvaLj91ut3C73TFMGdAPnoRBKK/XK7xeb9z3Cbu5WyyWgN/vt02+9vv9\ntsnjl0k2m82fnp4+PGfOnM/mzJnz2QMPPPB+T0+PK9LmDhhZMCjE9u1CNDYKsWuXEOvWcQSDCaHh\nW1dXF9N9wh7LFBYWnvT5fI6BgYHMYDB46/79+x8vLy9vU45ZvXp161/+8pd/Hh8fT7t27dptJ06c\nKM7NzT0b02wAA+BsHbMhbLmbTKaxhoaGzSUlJe+Nj4+nVVVVtTidzt6mpqYaIYSoqalpysnJOVda\nWurJz88/fcstt9yorq5+mc0d+CpqHbNJmnjSZhbeSJLk2XovQG2UZ+vNzZytI3qSJAlZlmecAfwL\nVSCJeBIGqcLPlgGShCdhkEqUO5Bg1DrUgHIHEohah1pQ7kACUOtQG8odiBO1DjWi3IEYUetQM8od\niAG1DrWj3IEZoNahFZQ7ECVqHVpCuQMRUOvQIsodCINah1ZR7sAUqHVoHeUOhKDWoQeUO/A5ah16\nQrkDglqH/lDuMDRqHXpFucOwqHXoGeUOw6HWYQSUOwyFWodRUO4wBGodRkO5Q/eodRgR5Q7dotZh\nZJQ7dIlah9FR7tAVah2YQLlDN6h14CbKHZpHrQNfRblD06h1YGqUOzSJWgfCo9yhOdQ6EBnlDs2g\n1oHoUe7QBGodmBnKHapGrQOxodyhWtQ6EDvKHapDrQPxo9yhKtQ6kBgRy93j8ZTm5OScczgcvvr6\n+q3Tjfvggw+KTCbT2DvvvLMmsVOEEVDrQGKFLffx8fG0zZs3Nxw8ePA7FoslUFRU9EF5eXmb0+ns\nDR23devW+tLSUo8sy1Jypwy9odaBxAtb7h0dHcvsdntfZmbmgNlsHq2oqNjX2tq6OnTciy+++NRj\njz321sKFCy8mb6rQG2odSJ6wm3sgELDYbDb/5Gur1ToYCAQsoWNaW1tXb9q0qVEIISRJkpMzVehJ\nd7cQRUVCdHZO1Pr69UJI/J0PSJiwxzLRbNRPP/30r3fu3PmMJEmyLMtSuGOZ2traLz52u93C7XbP\nYKrQg2BQiO3bhWhsFGLXLiHWrWNTB5S8Xq/wer1x3yfs5m6xWAJ+v982+drv99usVuugckxnZ+e3\nKioq9gkhxPDwcPqBAwceMZvNo+Xl5W2h91Nu7jAeztaByELDt66uLqb7SLI8fZyPjY2ZsrOzzx86\ndOihxYsXf7xs2bKOvXv3PhH6DdVJlZWVe773ve+9u2bNmne+8kYTZR/TJKFt1DoQO0mSRCwPqoQt\nd5PJNNbQ0LC5pKTkvfHx8bSqqqoWp9PZ29TUVCOEEDU1NU2xThjGQK0DqRG23BP6RpS7oVDrQGIk\npdyBWFDrQOrxs2WQMDy3DqgH5Y6EoNYBdaHcERdqHVAnyh0xo9YB9aLcMWPUOqB+lDtmhFoHtIFy\nR1SodUBbKHdERK0D2kO5Y1rUOqBdlDumRK0D2ka540uodUAfKHd8gVoH9INyB7UO6BDlbnDUOqBP\nlLtBUeuAvlHuBkStA/pHuRsItQ4YB+VuENQ6YCyUu85R64AxUe46Rq0DxkW56xC1DoBy1xlqHYAQ\nlLtuUOsAlCh3HaDWAYSi3DWMWgcwHcpdo6h1AOFQ7hpDrQOIBuWuIdQ6gGhR7hpArQOYKcpd5ah1\nALGg3FWKWgcQD8pdhah1APGi3FWEWgeQKJS7SlDrABKJck8xah1AMkS1uXs8ntKcnJxzDofDV19f\nvzX082+88cYPXS5XT35+/un777//yOnTp/MTP1X96e4WoqhIiM7OiVpfv14ISUr1rADogizLYa+x\nsbG0rKysvv7+/sxgMGh2uVynzp4961SOOXr06IqRkZE7ZVkWBw4cKC0uLj4eep+Jt4Isy/L167L8\n3HOyvHChLL/yiizfuJHqGQFQq8/3zoh7degV8cy9o6Njmd1u78vMzBwQQoiKiop9ra2tq51OZ+/k\nmBUrVhyb/Li4uPjE4OCgNfF/DOkDZ+sAZkPEzT0QCFhsNpt/8rXVah08ceJE8XTjW1paqsrKytqn\n+lxtbe0XH7vdbuF2u2c0WS0LBoXYvl2IxkYhdu0SYt06jmAAfJXX6xVerzfu+0Tc3CVJkqO92eHD\nh1ft3r37ySNHjtw/1eeVm7uRUOsAohUavnV1dTHdJ+I3VC0WS8Dv99smX/v9fpvVah0MHXf69On8\n6urql9va2srnz59/JabZ6AxPwgBIlYibe2Fh4Umfz+cYGBjIDAaDt+7fv//x8vLyNuWYv/3tb19f\ns2bNO6+//vq/2u32vuRNVzt4EgZAKkU8ljGZTGMNDQ2bS0pK3hsfH0+rqqpqcTqdvU1NTTVCCFFT\nU9P0i1/84rkrV67M37RpU6MQQpjN5tGOjo5lyZ68GnG2DkANpIknbWbhjSRJnq33ShXl2XpzM0cw\nAOInSZKQZXnGici/UE0AztYBqA0/WyZOPAkDQI0o9xhR6wDUjHKPAbUOQO0o9xmg1gFoBeUeJWod\ngJZQ7hFQ6wC0iHIPg1oHoFWU+xSodQBaR7mHoNYB6AHl/jlqHYCeUO6CWgegP4Yud2odgF4Zttyp\ndQB6Zrhyp9YBGIGhyp1aB2AUhih3ah2A0ei+3Kl1AEak23Kn1gEYmS7LnVoHYHS6KndqHQAm6Kbc\nqXUAuEnz5U6tA8BXabrcqXUAmJomy51aB4DwNFfu1DoARKaZcqfWASB6mih3ah0AZkbV5U6tA0Bs\nVFvu1DoAxE515U6tA0D8VFXu1DoAJIYqyp1aB4DESnm5U+sAkHgpK3dqHQCSJ+Lm7vF4SnNycs45\nHA5ffX391qnGbNmy5TcOh8Pncrl6uru7l0a6Z3e3EEVFQnR2TtT6+vVCSFIs09cmr9eb6imoBmtx\nE2txE2sRv7Cb+/j4eNrmzZsbPB5P6dmzZ3P37t37RG9vr1M5pr29vayvr8/u8/kczc3NP9q0aVPj\ndPej1ifwhXsTa3ETa3ETaxG/sJt7R0fHMrvd3peZmTlgNptHKyoq9rW2tq5WjmlrayvfsGHDK0II\nUVxcfGJkZGTe0NBQxlT3M3KtA8BsCru5BwIBi81m80++tlqtg4FAwBJpzODgoHWq+xm51gFgNoV9\nWkaSJDmam8iy/KUGn+6/27BBEhs2RD85Paurq0v1FFSDtbiJtbiJtYhP2M3dYrEE/H6/bfK13++3\nWa3WwXBjBgcHrRaLJRB6r9A/AAAAyRP2WKawsPCkz+dzDAwMZAaDwVv379//eHl5eZtyTHl5edur\nr766Xgghjh8/vnzevHkjGRkZQ8mcNAAgvLDlbjKZxhoaGjaXlJS8Nz4+nlZVVdXidDp7m5qaaoQQ\noqampqmsrKy9vb29zG63991+++2f7tmzp3J2pg4AmJYsywm9Dhw4UJqdnX3Obrf7du7cuXWqMU89\n9dRv7Ha7Lz8/v6erq2tpoueglivSWrz++us/zM/P77n33ntP33fffUd6enryUz3nVK3F5NXR0VGU\nlpY29vbbb69J9ZxTtQ6HDx92FxQUdC9ZsuSjBx980JvqOadqLS5evJheUlLicblcp5YsWfLRnj17\nNqZ6zsm6Kisrd3/ta18bysvL+3C6MTPdNxM6wbGxsbSsrKy+/v7+zGAwaHa5XKfOnj3rVI75/e9/\nX/bII4+0y7Isjh8/XlxcXHw81QubjCuatTh69OiKkZGRO2V54gvdyGsxOW7VqlV/fPTRR//nrbfe\nWpvqeadiHa5cuTIvNzf3jN/vt8ryxAaX6nmnai22bdtW+8wzzzw/uQ4LFiy4NDo6akr13JNxvf/+\n+yu7urqWTre5x7JvJvTHDyT6uXgti2YtVqxYcezOO+/8PyEm1mK6R0i1Lpq1EEKIF1988anHHnvs\nrYULF15MxTyTLZp1ePPNN3+wdu3atycfXEhPTx9OzWyTK5q1WLRo0d+vXr16hxBCXL169Y677rrr\nkslkGkvNjJNr5cqVf54/f/6V6T4fy76Z0M090c/Fa1k0a6HU0tJSVVZW1j47s5td0X5dtLa2rp78\nF87RPoarJdGsg8/nc1y+fHnBqlWrDhcWFp587bXX1s3+TJMvmrWorq5++cyZM0sWL178scvl6nnh\nhRf+bfZnqg6x7JsJ/amQiX4uXstm8ns6fPjwqt27dz955MiR+5M5p1SJZi2efvrpX+/cufMZSZJk\nWZal0K8RPYhmHUZHR81dXV3fPHTo0EPXrl27bcWKFceWL19+3OFw+GZjjrMlmrXYsWPHswUFBae8\nXq/7woULWQ8//PAfenp6XHPnzv1kNuaoNjPdNxO6uSfyuXiti2YthBDi9OnT+dXV1S97PJ7ScH8t\n07Jo1qKzs/NbFRUV+4QQYnh4OP3AgQOPmM3m0dBHb7UsmnWw2Wz+9PT04Tlz5nw2Z86czx544IH3\ne3p6XHrb3KNZi6NHj97385//fLsQQmRlZV24++67+8+fP59dWFh4crbnm2ox7ZuJ/KbA6Oio6Z57\n7rnQ39+fef369VsjfUP12LFjy/X6TcRo1uKvf/3r17OysvqOHTu2PNXzTfVaKK+NGzfu0ePTMtGs\nQ29vb85DDz10cGxsLO3TTz+9LS8v78MzZ87kpnruqViLn/zkJ/9ZW1u7TZZl8Y9//CPDYrEMXrp0\naUGq556sq7+/PzOab6hGu28mfILt7e2PfOMb3ziflZXVt2PHjp/JsixeeumlmpdeeqlmcsyPf/zj\nhqysrL78/Pyezs7Ob6Z6UZN1RVqLqqqq/1qwYMGlgoKC7oKCgu6ioqKOVM85VWuhvPS6uUe7Dr/6\n1a/+PTc390xeXt6HL7zwwpZUzzlVa3Hx4sX07373u+/m5+f35OXlffjGG2/8INVzTtZVUVGxd9Gi\nRR+bzeag1Wr1t7S0PBnvvinJsu6OuwHA8FTx/1AFACQWmzsA6BCbOwDoEJs7AOgQmzsA6BCbOwDo\n0P8DN1xo2SdDYJYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1058bdb50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Randomly generated predictions should give us a diagonal ROC curve\n",
    "preds = rand(len(target))\n",
    "fpr, tpr, thr = roc_curve(target, preds, pos_class=True)\n",
    "plot(fpr, tpr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The area under the ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def auc(true_labels, predicted_labels, pos_class=1):\n",
    "    fpr, tpr, thr = roc_curve(true_labels, predicted_labels,\n",
    " pos_class=pos_class)\n",
    "    area = -trapz(tpr, x=fpr)\n",
    "    return area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc(target, preds, pos_class=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi-class classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = pandas.read_csv(\"data/mnist_small.csv\")\n",
    "d_train = d[:int(0.8*len(d))]\n",
    "d_test = d[int(0.8*len(d)):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier()\n",
    "rf.fit(d_train.drop('label', axis=1), d_train['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "preds = rf.predict(d_test.drop('label', axis=1))\n",
    "cm = confusion_matrix(d_test['label'], preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO8AAADxCAYAAAAjrOBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF79JREFUeJzt3XtUU1e+B/B9QoKgDDA4UpSAIEohgBDk0VIk+H7b6tDW\nR4sXfHTG1vpataN3dURZ46MuB1HrvdWqlVurrnrl+ijS8RXEUnmU4APQWm9SAmId6wVFlBBy7h8z\nMGghydmccHLg+1krawGe7P0j+nXvnJyzN8OyLAEA8ZEIXQAA0EF4AUQK4QUQKYQXQKQQXgCRQngB\nRArhBaDAMAxr7cPDw+OBTWrA57wA3P0zmFYdy7IsYVnWuoM5kPLdIEBvwSW8toDwAlCyNry2gvAC\nUHJwcLDquJaWFpv0j/ACUMLICyBSCC+ASCG8ACKF8AKIlEQi7DVOCC8AJaFHXpv/15GbmzsxKCjo\nxrBhw25t3rz5Q1v3Z45er/cZNWrUhZCQkPLQ0NDr27dvf1/IegghpKWlxUGpVGqmTZt2Uuha6urq\n3JOSko4GBwdXKhSKisuXL78kZD0bN25cHRISUh4WFnZtzpw5XzY1NfURsp7nSSQSqx42889Lt2zy\nMBqNDgEBAT9qtVo/g8EgCw8PL6uoqAi2ZZ/mHrW1tV4ajSaCZVny6NEjl8DAwJtC1sOyLNm6deuK\nOXPmHJw2bdoJIetgWZYkJycf2Lt3byrLsqS5uVlaV1fnJlQtWq3Wz9/f/3+fPn3ah2VZ8sYbbxz5\n/PPP5wn9GrU+CCGsu7u7VY9/xIz/Gmw68hYVFcUMHTr0Rz8/P51MJmueNWvW4ePHj79qyz7N8fLy\nuhsREVFGCCEuLi4NwcHBlXfu3BkkVD3V1dXynJycyQsWLPiMtcG1r1zU19e75efnj0xNTd1HCCFS\nqdTo5uZWL1Q9rq6uD2UyWXNjY2Nfo9EobWxs7Ovt7V0jVD0dYRjGqsfzLM0At27dulIikZgePHjg\nYa5/m4a3pqbG28fHR9/6vVwur66pqfG2ZZ/W0ul0fhqNRhkbG1soVA3Lly/P2LJlywcSicQkVA2t\ntFqt/4ABA/6ekpKyPzIysnThwoV7Ghsb+wpVj4eHx4OVK1du9fX1rRo0aNAdd3f3urFjx54Vqp6O\n0E6bZTJZc0ZGxvLy8vKQy5cvv/TJJ5+8W1lZGUzIP4J95syZcYMHD/7JYv82+J3aMAxjl7csNTQ0\nuCQlJR3NzMxc6uLi0iBEDadOnZrq6el5T6lUaoQedQkhxGg0SktLSyMXL168q7S0NLJfv36PN23a\n9Ceh6rl9+3bAtm3blul0Or87d+4MamhocDl48OBcoerpCO3Ia24GuGLFir9+/PHHq6zp36bh9fb2\nrtHr9T6t3+v1eh+5XF5tyz4taW5ulv3+97//77feeuuL11577X+EqqOgoCDuxIkT0/39/bWzZ88+\ndP78+dHJyclZQtUjl8ur5XJ5dXR0dDEhhCQlJR0tLS2NFKqekpKSqLi4uIL+/fv/IpVKjTNnzjxW\nUFAQJ1Q9HaENb3vtZ4DHjx9/VS6XVw8fPvyqVQXY8k19c3OzdMiQIbe1Wq1fU1OTo9AnrEwmE/P2\n229nLVu2LEPoEx7tH2q1WjV16tSTQtcxcuTIizdv3gxkWZasXbs2bdWqVZuFqqWsrCw8JCTkemNj\no7PJZGKSk5MP7Ny5812hX6PWByGE9fT07PDh7u7O9uvXr+1BOjlh9ejRI5cRI0aUZGdnv/b48eO+\nMTExhfX19a4syxI/Pz/t/fv3+5utwda/ZE5OzqTAwMCbAQEBP27YsGG1kC94fn5+PMMwpvDw8LKI\niAhNRESE5vTp0xOF/oegVqtV9nC2uaysLDwqKqp4+PDhV2bMmHFMyLPNLMuSzZs3r1IoFOWhoaHX\nkpOTDxgMBpnQr1HrgxDCenl5WfXoKLwGg0E2fvz4bzIyMpaxLEuuXr0a5unp+bOfn5/Wz89PK5VK\nmwcPHqz7+eefPTurAStpAFBgGIYdNMi6Dyru3LlD2HbnNViWZebNm3egf//+v2RkZCzv6Dn+/v7a\n77//foS5JXSwhhUAJdr3vN9+++0rX3zxxVsXLlwYpVQqNUqlUnP69OlJz7VtcVTFyAtAgWEYVi6X\nW3VsdXX1MyMvX7p8bbO9fhwEQINLyIS+tpmXGxPeeOMNs39eXl5OQkJCzB5z5MgRPkohzc3NFo9J\nT08nH330kdljZDIZL/VYIy0tjaSlpXVbf5bYUz3dWQvXMPaI8AL0RggvgEhZuwCdrXRLeAcMGNAd\n3VgtISFB6BKekZiYKHQJz7CneuyplucJPfJa/KiIj/txPT09aZ5mMyqVSugSnmFv/0DtqR57quV5\nfFwe2RVmw9vS0uLw3nvv7czNzZ1YUVGhOHTo0OzWux8Aeju7Dq+93Y8LYE+EXknDbMv2fD8ugNCE\nHnnNnrCy9gKM8vLytq8HDBhgd+9xATqiVquJWq2mfr5dn2229n5cSxdgANijxMTEZ06IrVu3jtPz\n7fpsc1RUVMmtW7eG6XQ6P4PB4HjkyJE3p0+ffqK7igOwZ0K/5zU78kqlUuPOnTvfmzBhwjctLS0O\n8+fP3xscHFxps2oARETokdfiRRqTJk06PWnSpNPdUQyAmNh9eAGgY9juBECkMPICiJRdf1RkLT7u\nxT1z5gwPlRAybtw4XtqB7mHN/df2inbarNfrfZKTk7Pu3bvnyTAMu2jRot3vv//+9q+++ur1tLS0\ntBs3bgQVFxdHR0ZGlpprByMvACXaaXPrjgkRERFlDQ0NLiNGjPh+3LhxZ8LCwq5lZ2fPeOeddz61\nph2EF4ASbXi9vLzuenl53SXk2R0TxowZc45LOwgvACU+zjZ3Zc8shBeAUmcjb11dHamvt7zBYlf3\nzEJ4ASh1NvJ6eHgQD49/7c5ZVVX1q2P42DML4QWgRDttZlmWmT9//l6FQlGxbNmybZ0dY6kdhBeA\nEm14W3dMGD58+FWlUqkhhJANGzasaWpq6rNkyZId9+/f/92UKVO+7mgnhfYQXgBKtGeb4+PjL5lM\npg6Tz2UKjfACUMK1zQAihWubAUQK4QUQKUybAUSqR9xVBNAbYdoMIFKYNgOIFEZeAJHqEeHlYzUE\nvlbAGDJkCC/t3Lx5k5d2ZDIZL+3YG75WwDAajby04+zszEs7XPSI8AL0RggvgEjhhBWASGHkBRAp\nhBdApISeNlvsXa/X+4waNepCSEhIeWho6PXt27e/3x2FAdg7u95cm5DO15jFboHQ2wk9bbY48np5\ned2NiIgoI+TZNWZtXxqAfaPdnzc1NXXfCy+88HNYWNi11p8VFRXFxMTEFCmVSk10dHRxcXFxtMX+\nuRTblTVmAXoa2mlzSkrK/tzc3Intf7Zq1aqP09PTP9JoNMr169f/edWqVR9b6t/qE1bm1phNT09v\n+zohIYGoVCprmwUQjFqtJmq1mvr5tNPmkSNH5ut0Or/2Pxs4cGBtfX29GyGE1NXVuXt7e9dY7J9l\nWYudNTc3y6ZOnXpq0qRJp59fqpJhGNZgMHCrvgN8XUaIyyO7R0+8PJJhGKuWXP3nsWxKSkqHf1Zb\nW0vu3r3b9n1ZWdmv2tXpdH7Tpk07ee3atTBCCPnpp58Gx8fHX2IYhjWZTJLvvvvuZR8fH725GixO\nm61ZYxagN+psmjxo0CASGRnZ9rDG/Pnz927fvv39qqoq34yMjOWpqan7LD3HYnhb15i9cOHCKKVS\nqVEqlZrn5+sAvRGfHxUVFRXFzJgxI5sQQpKSko4WFRXFWHqOxfe85taYBejN+PyoaOjQoT/m5eWp\nVCpV3vnz50cHBgb+YOk5uMIKgBLtFVazZ88+lJeXp7p///7vfHx89OvXr//z7t27F7377rufNDU1\n9XF2dn6ye/fuRZbaQXgBKNGOvIcOHZrd0c8LCwtjubSD8AJQEvoKK17Ca08fh5SXl/PSzsmTJ3lp\nZ9q0aby0Y0+vMSH81WNvvxcXPSK8AL0RwgsgUggvgEghvAAiJfTN+AgvACWMvAAihfACiBTCCyBS\nCC+ASCG8ACKFs80AIoXwAogUps0AIoXwAogUwgsgUkKHF2tTAVCiXYCuox0T0tLS0uRyeTWXRR4R\nXgBKfO6YwDAMu2LFir9qNBqlRqNRTpw4MddS/z1u2iyV8vMrzZw5k5d2OluYm6tdu3bx0g4fi5Pb\nI74WgeeC9qOijnZMIIRYveB7qx4XXoDu0ll4tVot0el0nNvbsWPHkqysrOSoqKiSrVu3rnR3d68z\n2z/nHgCAENL5tHnIkCFk9OjRbQ9r/PGPf/wPrVbrX1ZWFjFw4MDalStXbrX0HIQXgBKfOyZ4enre\nYxiGZRiGXbBgwWfW7JiA8AJQ4jO8tbW1A1u/zs7OntH+THRn8J4XgBLt57zP75iwbt26tWq1OrGs\nrCyCYRjW399f++mnn75jqR2rwtvS0uIQFRVVIpfLq0+ePMnPQsQAIkd7trmjHROs2RXwV/1bc1Bm\nZuZShUJRwTCM5c18AXoJPqfNNCyGt7q6Wp6TkzN5wYIFn3H9HAqgJxM6vBanzcuXL8/YsmXLBw8f\nPnTt7Ji0tLS2rxMTE0liYiIvxQHYUl5eHrl48SL184W+ttlseE+dOjXV09PznlKp1KjV6sTOjmsf\nXgCxUKlURKVStX2fnp7O6fl2fTN+QUFB3IkTJ6bn5ORMfvr0qdPDhw9dk5OTs7KyspK7q0AAeyX0\nyGv2v44NGzas0ev1Plqt1v/w4cOzRo8efR7BBfgHu3/P2x7ONgP8i11Pm9tTqVR5KpUqz5bFAIiJ\n0NNmXGEFQAnhBRAphBdApBBenslkMl7aefLkCS/t8LUCRkVFBS/tjBgxgpd27A1ff+9cILwAIiWa\ns80A8CyEF0CkMG0GECmhR14sgwNAic9F1z/44IMtwcHBleHh4Vdmzpx5rL6+3s1S/wgvACU+F10f\nP37838rLy0OuXLkSHhgY+MPGjRtXW+of4QWgRBvekSNH5v/2t7/9v/Y/Gzdu3BmJRGIihJDY2NjC\n6upquaX+EV4ASra6q2jfvn2pkydPzrF0HE5YAVBycHDo8OeVlZWksrKSqs2//OUv/+7o6GiYM2fO\nl5aORXgBKHU2qioUCqJQKNq+z87Otqq9zz///N9ycnImnzt3bow1xyO8AJT4/Jw3Nzd34pYtWz7I\ny8tTOTk5PbXmOXjPC0BJIpFY9Xje7NmzD8XFxRXcvHnzRR8fH/2+fftSlyxZsqOhocFl3LhxZ5RK\npWbx4sUWL4rHyAtAiXbk5WvRdYQXgBIujwQQKYQXQKQ6+6iou/AS3ubm5i63wdfN1A8fPuSlHVfX\nTjeIEARfN9GnpKTw0s7+/ft5aUfMMPICiBTCCyBSCC+ASAl9Py/CC0AJIy+ASGHkBRApoUdei/91\n1NXVuSclJR0NDg6uVCgUFZcvX36pOwoDsHe01zbzxeLIu3Tp0szJkyfnHD16NMloNEofP37cz2bV\nAIiI0COv2fDW19e75efnjzxw4MA8QgiRSqVGNze3+u4pDcC+2XV4tVqt/4ABA/6ekpKy/8qVK+Ej\nRoz4PjMzc2nfvn0b2x+Xnp7e9nVCQgJRqVQ2KheAP2q1mqjVaurnCx1ehmU73y+7pKQk6uWXX/6u\noKAgLjo6unjZsmXbXF1dH65fv/7PbQ0wDGswGLpcCC6P7B64PLJzDMMQlmWtSiTDMOy5c+esanfM\nmDFWt8uF2XfTcrm8Wi6XV0dHRxcTQkhSUtLR0tLSSL6LABAjWy1AZy2z4fXy8rrr4+Oj/+GHHwIJ\nIeTs2bNjQ0JCym1WDYCI2HV4CSFkx44dS+bOnXswPDz8ytWrV4evWbNmg82qARCRroQ3MzNzaVhY\n2LXQ0NDrmZmZS2n6t/hRUXh4+JXi4uJomsYBejLaUfX69euhn3322YLi4uJomUzWPHHixNypU6ee\nCggIuM2lHSxAB0CJduS9ceNGUGxsbKGTk9NTBweHFpVKlXfs2LGZXPtHeAEo0YY3NDT0en5+/sgH\nDx54NDY29v3666+nWLO9yfN4ubaZr495+ODs7Cx0Cc/gY5URQvh7jXfv3s1LO3ydiDH3USUXfL3O\nXHT2Gmg0GqLRaDp9XlBQ0I0PP/xw8/jx4//Wr1+/x0qlUtO6TxGn/rv64jEMw/L1F8AHewtLT63H\n0dGRl3bsKbyOjo6cPuf99ttvrWr3lVdeMdvumjVrNvj6+lb94Q9/+E8rSyWE4K4iAGpdmX3cu3fP\n09PT815VVZVvdnb2jMLCwliubSC8AJS6Et6kpKSjv/zyS3+ZTNa8a9euxa6urpwvDUR4ASh1JbwX\nL15M6Gr/CC8AJaFvTEB4ASghvAAihfACiBTCCyBSCC+ASCG8ACKF8AKIFMILIFIIL4BIIbwAIiX0\nXkW4GR9ApDDyAlDCtJlnfN20/uTJE17akUr5eYn5qocvfCy0Twghu3bt4qWdhQsX8tIOFwgvgEgh\nvAAihfACiBTONgP0QnxsWo+RF4BSV6bNfGxab3Hk3bhx4+qQkJDysLCwa3PmzPmyqampD125AD0L\n7aLrrZvWp6am7iOEftN6syOvTqfz27Nnz8LKysrgPn36NL355ptHDh8+PGvevHkHuHYE0FsUFhaS\nwsLCTv/c2k3rLTE78rq6uj6UyWTNjY2NfY1Go7SxsbGvt7d3DZcOAHqqzkbal156iSxdurTt8Tyj\n0SgtLS2NXLx48a7S0tLIfv36Pd60adOfuPZvNrweHh4PVq5cudXX17dq0KBBd9zd3evGjh17lmsn\nAD0R7bSZr03rzU6bb9++HbBt27ZlOp3Oz83Nrf7111//6uDBg3Pnzp17sP1xaWlpbV8nJiaSxMRE\nrnUAdLu8vDxy8eJF6ufTnrBqv2l9YGDgD7Sb1psNb0lJSVRcXFxB//79fyGEkJkzZx4rKCiIMxde\nALFQqVREpVK1fZ+ens7p+V0529y6ab3BYHAMCAi4vX///hSubZgNb1BQ0I309PSPnjx54uzk5PT0\n7NmzY2NiYoqoKwboQboSXj42rTf7njc8PPxKcnJyVlRUVMnw4cOvEkLIokWL+NkjEgC6pMdt8ckX\ne7uryGg08tIOX/j6vfbs2cNLO3zcVcR1i8+qqiqr2vX19bW6XS5whRUAJdyYACBSCC+ASCG8dsrZ\n2ZmXduztvTNf+HoPvnjxYl7aycrK4qUdLhBeAJESOry4nxdApDDyAlASeuRFeAEoCR1eTJsBRAoj\nLwAloRegQ3gBKGHaDABUMPICUBJ65EV4ASgJHV5MmwFECuEFoES7AN3Tp0+dYmNjCyMiIsoUCkXF\n6tWrN9L0j2kzACXaabOTk9PTCxcujOrbt2+j0WiUxsfHX7p06VJ8fHz8JS7tdMvIq1aru6Mbq9lb\nPV1ZwdAW8vLyhC6hjb39XbVHO/ISQkjrAusGg8GxpaXFwcPD4wHX/hFeO5Cfny90Cc+wp/9M7O3v\nyhr5+flk48aNbY+OmEwmSURERNkLL7zw86hRoy4oFIoKrv1g2gxAqbNRNSEhgSQkJLR9v2nTpl8d\nI5FITGVlZRH19fVuEyZM+EatVicmJiaqufSPE1YAAnJzc6ufMmXK1yUlJVFcn8vL6pFdagDAjnBZ\nPfLRo0dWtfmb3/zmmXbv37//O6lUanR3d6978uSJ84QJE75Zu3btujFjxpzjUmuXp822WNISoCer\nra0dOG/evAMmk0liMpkkb7/99n9xDS4hPIy8AL0RwzBsQ0ODVce6uLhg3WYAe4LLIwGACkZeAEoY\neQGACkZeAEoYeQGACkZeAEoYeQGACkZeAEoYeQGACkZeAEoYeQGACkZeAEpCj7wILwAlocOLaTOA\nSGHkBaCEkRegF8rNzZ0YFBR0Y9iwYbc2b978IU0bWEkDgALDMKzJZLLqWIlE8sxKGi0tLQ4vvvji\nzbNnz4719vauiY6OLj506NDs4ODgSi41YOQF6GZFRUUxQ4cO/dHPz08nk8maZ82adfj48eOvcm0H\n4QWgRLtjQk1NjbePj4++9Xu5XF5dU1PjzbV/nLACoGTtCSsXF5dn1ojla7lkhBeAQldWg/T29q7R\n6/U+rd/r9XofuVxezbUdTJsBullUVFTJrVu3hul0Oj+DweB45MiRN6dPn36CazsYeQG6mVQqNe7c\nufO9CRMmfNPS0uIwf/78vVzPNBOCj4oARAvTZgCRQngBRArhBRAphBdApBBeAJFCeAFE6v8BePAz\na92hkfIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10cf8b290>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "matshow(cm, cmap='Greys')\n",
    "colorbar()\n",
    "savefig(\"figures/figure-4.19.eps\", format='eps')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The root-mean-square error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def rmse(true_values, predicted_values):\n",
    "    n = len(true_values)\n",
    "    residuals = 0\n",
    "    for i in range(n):\n",
    "        residuals += (true_values[i] - predicted_values[i])**2.\n",
    "    return np.sqrt(residuals/n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.26547176572681108"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse(rand(10), rand(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The R-squared error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def r2(true_values, predicted_values):\n",
    "    n = len(true_values)\n",
    "    mean = np.mean(true_values)\n",
    "    residuals = 0\n",
    "    total = 0\n",
    "    for i in range(n):\n",
    "        residuals += (true_values[i] - predicted_values[i])**2.\n",
    "        total += (true_values[i] - mean)**2.\n",
    "    return 1.0 - residuals/total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98002618717502987"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r2(arange(10)+rand(), arange(10)+rand(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grid search with kernel-SVM model\n",
    "\n",
    "Importing modules:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading data and performang poor-mans feature engineering:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "d = pandas.read_csv(\"data/titanic.csv\")\n",
    "\n",
    "# Target\n",
    "y = d[\"Survived\"]\n",
    "\n",
    "# Features\n",
    "X = d.drop([\"Survived\", \"PassengerId\", \"Cabin\",\"Ticket\",\"Name\", \"Fare\"], axis=1)\n",
    "X['Sex'] = map(lambda x: 1 if x==\"male\" else 0, X['Sex'])\n",
    "X['Embarked-Q'] = map(lambda x: 1 if x==\"Q\" else 0, X['Embarked'])\n",
    "X['Embarked-C'] = map(lambda x: 1 if x==\"C\" else 0, X['Embarked'])\n",
    "X['Embarked-S'] = map(lambda x: 1 if x==\"S\" else 0, X['Embarked'])\n",
    "X = X.drop([\"Embarked\", \"Sex\"], axis=1)\n",
    "X = X.fillna(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Performing grid-search to find the optimal hyper-parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum = 0.674\n",
      "Tuning Parameters: (gamma = 0.01, C = 4.01)\n"
     ]
    }
   ],
   "source": [
    "# grid of (gamma, C) values to try \n",
    "gam_vec, cost_vec = np.meshgrid(np.linspace(0.01, 10, 11),\n",
    "                     np.linspace(0.01, 10, 11))\n",
    "\n",
    "AUC_all = [] # initialize empty array to store AUC results\n",
    "\n",
    "# set up cross-validation folds\n",
    "N = len(y)\n",
    "K = 10 # number of cross-validation folds\n",
    "folds = np.random.randint(0, K, size=N)\n",
    "\n",
    "# search over every value of the grid\n",
    "for param_ind in np.arange(len(gam_vec.ravel())):\n",
    "\n",
    "    # initialize cross-validation predictions\n",
    "    y_cv_pred = np.empty(N)\n",
    "\n",
    "    # loop through the cross-validation folds\n",
    "    for ii in np.arange(K):\n",
    "        # break your data into training and testing subsets\n",
    "        X_train = X.ix[folds != ii,:]\n",
    "        y_train = y.ix[folds != ii]\n",
    "        X_test = X.ix[folds == ii,:]\n",
    "\n",
    "        # build a model on the training set\n",
    "        model = SVC(gamma=gam_vec.ravel()[param_ind], C=cost_vec.ravel()[param_ind])\n",
    "        model.fit(X_train, y_train)\n",
    "\n",
    "        # generate and store model predictions on the testing set\n",
    "        y_cv_pred[folds == ii] = model.predict(X_test)\n",
    "\n",
    "    # evaluate the AUC of the predictions\n",
    "    AUC_all.append(roc_auc_score(y, y_cv_pred))\n",
    "\n",
    "indmax = np.argmax(AUC_all)\n",
    "print \"Maximum = %.3f\" % (np.max(AUC_all))\n",
    "print \"Tuning Parameters: (gamma = %.2f, C = %.2f)\" % (gam_vec.ravel()[indmax], cost_vec.ravel()[indmax])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plotting the contours of the parameter performance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEMCAYAAAAPqefdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtUE1e/N/A9kKCESxARrElqqKAkXBOCXKoCigK2Agoo\n3rAUEbW0am2r9vQ59fT0ValaRakWrbWiIlSsQnsELFYURUAIggIKUtKGWFFErlFJwrx/uOITecgF\nArng77PWrMXM7Jm9h9qv2z2XjeE4jgAAAOgHA203AAAAgOogtAEAQI9AaAMAgB6B0AYAAD0CoQ0A\nAHoEQhsAAPTIsIX2+++//6ONjU2zs7PzLem21tZWy9mzZ/8+efLkujlz5lxoa2uzGK76AQBAU3Jz\nc4McHBzu2Nvb1ycmJm7qr0xBQYEfi8WqcHJyuu3n51cg3b59+/Ytjo6O1c7OzreWLFmS9vz581EK\nK8NxfFiWK1euTOdyuSwnJ6db0m2ffvrpN4mJiZ/hOI527NixadOmTTuGq35YYIEFFk0sYrHYcNKk\nSfcaGxvpPT09RFdX15s1NTUM2TJPnjyxYDKZ1Xw+n4rjOHr06JEVjuOosbGRbmtr++ezZ89G4TiO\nFi5cmPHTTz+tUFTfsPW0p0+fXjhmzJgnstuys7NDVqxYcQwhhFasWHHs3LlzYcNVPwAAaEJpaelU\nOzu7e3Q6nUckEkVRUVHpWVlZobJl0tLSloSHh5+hUqlNCCFkZWXVghBC5ubmHUQiUSQUCklisZgg\nFApJFApFoKg+jY5pNzc329jY2DQjhJCNjU1zc3OzjSbrBwCAoSYQCCg0Go0vXadSqU0CgYAiW6a+\nvt6+tbXV0t/f/xKHwyk7fvz4coQQsrS0bN24cePuN9988+8JEybct7CwaAsICMhXVB9heC5DOQzD\ncAzD+n2HXt52AADoD47jmDrHDyRzTE1NOzs7O80HcqxIJCJyuVz2xYsXZwmFQpK3t/d1Ly+vYgMD\ng969e/eu5/F4dDKZ3B4ZGXn65MmTS5cuXXpS3rk0Gto2NjbNDx48GD9+/PgH//zzzxvW1tYP5ZUd\nPXo0QgghCwvl9yrHjBmjUv1WVlZqlZHXFnNz8363I4QQmUxGCCF0+fJl5Ovrq9L55F2PvO2q/I5M\nTU2VlhmMQ4cOoVWrVg3LubVpJF7XSLwmhBDicDhDch5p5ijT1dVlJrtOoVAEfD6fJl3n8/k06TCI\nFI1G41tZWbUYGxs/NTY2fjpjxowrlZWVrr29vQY+Pj5FY8eOfYwQQgsWLPilqKjIR1Foa3R4JCQk\nJPvYsWMrEELo2LFjK8LCws4NxXmfPHmivBAAAAwDDodTVl9fb8/j8eg9PT1GGRkZi0JCQrJly4SG\nhmZdvXp1mkQiMRQKhaSSkhJPJpNZM2XKlLvFxcVeT58+NcZxHMvPzw9gMpk1iuobtp724sWLT12+\nfNm3paXFikaj8b/66qv/3rx5846FCxf+fOTIkVg6nc77+eefFw5X/QAAoAkEAkGcnJycEBgYmCeR\nSAxjY2OPMBiM2pSUlHiEEIqPj09xcHC4ExQUlOvi4lJlYGDQGxcXd1gaztHR0akcDqfMwMCgl81m\nc1etWnVIUX0Yjuve8DGGYfhAhkcQUm2IRJvDIzweD9HpdJXOp0/DI+Xl5cjd3X1Yzq1NI/G6RuI1\nIfRieGQoxrRVHR559uyZ2vWpA96I1JC+gT1SjMQQQGhkXtdIvKbXEYQ2AADoEQhtAADQIxDaAACg\nRyC0AQBAj0BoAwCAHoHQBgAAPaK1b4+oqq2tTeVntQEAYLBUzZkHDx4Mc0sUg542AADoEQhtAADQ\nI69VaLe0tGi7CQAAoJbXKrQBAEDfQWgDAIAegdAGAAA9AqENAAB6BEIbAAD0CIQ2AACoKTc3N8jB\nweGOvb19fWJi4qb+yhQUFPixWKwKJyen235+fgXS7W1tbRYRERGZDAajlslk1hQXF3spqkvn34gE\nAABdJpFIDBMSEpLz8/MDKBSKwMPD40ZISEg2g8GolZZpa2uz+OCDD77Ly8sLpFKpTS0tLS+nyFq3\nbl3S3Llzz2dmZkaIxWJCd3e3iaL6oKcNAABqKC0tnWpnZ3ePTqfziESiKCoqKj0rKytUtkxaWtqS\n8PDwM9JZ2q2srFoQQqi9vZ1cWFg4/f333/8RoRfzTZLJ5HZF9UFoAwCAAj09Pairq+vl0pdAIKDQ\naDS+dJ1KpTYJBAKKbJn6+nr71tZWS39//0scDqfs+PHjyxFCqLGx0XbcuHGPYmJijrLZbG5cXNxh\noVBIUtQeGB4BAACk2uTgCCFUW1v7yjqGYUpnRxeJREQul8u+ePHiLKFQSPL29r7u5eVVLBaLCVwu\nl52cnJzg4eFxY/369Xt37Nix+auvvvpveeeCnjYAAKiBQqEI+Hw+TbrO5/Np0mEQKRqNxp8zZ84F\nY2Pjp2PHjn08Y8aMK1VVVS40Go1PpVKbPDw8biCEUERERCaXy2Urqg9CGwAA1MDhcMrq6+vteTwe\nvaenxygjI2NRSEhItmyZ0NDQrKtXr06TSCSGQqGQVFJS4slgMGptbGyaaTQav66ubjJCCOXn5wc4\nOjpWK6oPhkcAAEANBAJBnJycnBAYGJgnkUgMY2NjjzAYjNqUlJR4hBCKj49PcXBwuBMUFJTr4uJS\nZWBg0BsXF3eYyWTWIITQ/v37P1y6dOnJnp4eo0mTJjUcPXo0RlF9GI4rHY7ROAzD8NGjR79cV+Xj\n5KqOR1lZWQ16v7x2mJubyz2GTCYP+HzyrkXedlV+P6ampkrLAKCPOBwOwnEcU+ccGIbhDAZDpbK1\ntbVq16cOGB4BAAA9AqENAAB6BEIbAAD0CIQ2AADoEQhtAADQIxDaAACgRyC0AQBAj8DLNQAAgJS/\nw6EroKcNAAB6BEIbAAD0CIQ2AADoEa2E9vbt27c4OjpWOzs731qyZEna8+fPR2mjHQAAoG80Hto8\nHo9++PDhOC6Xy75165azRCIxTE9Pj9J0OwAAQB9p/OkRc3PzDiKRKBIKhSRDQ0OJUCgkUSgUgabb\nAQAA+kjjoW1padm6cePG3W+++ebfxsbGTwMDA/MCAgLy+5YTi8Uvf+7p6UFGRkYabScAQDeVl5ej\n8vJybTdDazQ+PNLQ0DBp796963k8Hv3+/fsTurq6TE+ePLm0bzkCgfBygcAGAEi5u7ujVatWvVxe\nNxoP7bKyMo6Pj0/R2LFjHxMIBPGCBQt+KSoq8tF0OwAAYKjk5uYGOTg43LG3t69PTEzc1F+ZgoIC\nPxaLVeHk5HTbz8+vQHafRCIxZLFYFfPmzftVWV0aD20HB4c7xcXFXk+fPjXGcRzLz88PkE67AwAA\n+kYikRgmJCQk5+bmBtXU1DBPnTq1uLa29pVpcNra2iw++OCD73799dd5t2/fdsrMzIyQ3Z+UlLSO\nyWTWqDKzu8ZD29XVtTI6OjqVw+GUubi4VCGE0KpVqw5puh0AADAUSktLp9rZ2d2j0+k8IpEoioqK\nSs/KygqVLZOWlrYkPDz8jHSWdisrqxbpvqamJur58+fnrly58gdVpjHTyrdHPvvss28+++yzb7RR\nNwAADERbWxtqb2+Xu18gEFBoNBpfuk6lUptKSko8ZcvU19fbi0Qior+//6XOzk6zdevWJS1fvvw4\nQght2LBhz86dOz/t6OiQP9msDPhgFAAAIPkfjOq7/e+//35lXZUhDZFIRORyueyLFy/OEgqFJG9v\n7+teXl7Fd+/enWJtbf2QxWJVFBQU+KnSTghtAABQA4VCEfD5fJp0nc/n06TDIFI0Go1vZWXVYmxs\n/NTY2PjpjBkzrlRWVrpyuVx2dnZ2yPnz5+c+e/ZsdEdHh3l0dHRqampqtLz64NsjAACgBg6HU1Zf\nX2/P4/HoPT09RhkZGYtCQkKyZcuEhoZmXb16dZpEIjEUCoWkkpISTyaTWbNt27bP+Xw+rbGx0TY9\nPT1q5syZfygKbISgpw0AAGohEAji5OTkhMDAwDyJRGIYGxt7hMFg1KakpMQjhFB8fHyKg4PDnaCg\noFwXF5cqAwOD3ri4uMP9PTWnylALhuNKy2gchmH46NGjX65bWFgoPWbMmDEqnVvZh84V7ZfXDnNz\n+fcPyGTygM8n71rkbVfl92Nqaqq0DAD6iMPhIFWeulAEwzB8/vz5KpU9e/as2vWpA4ZHAABAj0Bo\nAwCAHoHQBgAAPQKhDQAAegRCGwAA9Ijc0D5+/Pjy/p4XPH78+PK0tLQlw9ssAAAA/ZEb2vv37/9w\n/vz5Z/tunz9//tldu3Z9MrzNAgAA0B+5L9eIRCKimZlZZ9/tpqamXSKRiDi8zQIAAM1S5X0HXSC3\np/3s2bPRXV1d//FGRmdnpxmENgAAaIfc0I6NjT0SGRl5msfj0aXbGhsbbRctWpQRGxt7RCOtAwAA\n8Aq5wyOffPLJLlNT0y5fX9/LnZ2dZgi9GBrZsmXL9jVr1hzUXBMBAABIKfxg1OrVq79fvXr199KP\nc5ubm3doplkAAAD6o9JX/iCsAQBAN8DLNQAAoEdeq9BW9llWAADQdQMO7Rs3bnjcv39/wnA0BgAA\ngGIDDu39+/d/+M477/zfokWLMoajQQAAoG9yc3ODHBwc7tjb29cnJiZu6q9MQUGBH4vFqnBycrrt\n5+dXgNCL+ST9/f0vOTo6Vjs5Od3et2/fR8rqUngjEsdxrKmpiSo7Pbz0eySqTvcOAAAjmUQiMUxI\nSEjOz88PoFAoAg8PjxshISHZDAajVlqmra3N4oMPPvguLy8vkEqlNrW0tFghhBCRSBTt2bNng5ub\n282uri5Td3f38tmzZ/8ue2xfSnvawcHBOf1thydKAAAAodLS0ql2dnb36HQ6j0gkiqKiotKzsrJC\nZcukpaUtCQ8PPyOdpd3KyqoFIYTGjx//wM3N7SZCL96DYTAYtcqGnxWGNoZhuLu7e3lpaelU9S4L\nAAB0m7m5eb9LR0cHqqmpebn0JRAIKLKjEVQqtUkgEFBky9TX19u3trZa+vv7X+JwOGXHjx9f3vc8\nPB6PXlFRwfL09CxR1E6lz2kXFxd7nThxYtnEiRP/MjEx6UboRZhXVVW5KDt2KOjLR1wAACMTlUpF\nVCr15XpJyauZqsoM6iKRiMjlctkXL16cJRQKSd7e3te9vLyK7e3t6xFCqKuryzQiIiIzKSlpnamp\naZeicykN7by8vEBlZQAA4HVFoVAEfD6fJl3n8/k06TCIFI1G41tZWbUYGxs/NTY2fjpjxowrlZWV\nrvb29vUikYgYHh5+ZtmyZSfCwsLOKatP6Zg2nU7n8fl82qVLl/zpdDrPxMSkW5vTxwMAgC7hcDhl\n9fX19jwej97T02OUkZGxKCQkJFu2TGhoaNbVq1enSSQSQ6FQSCopKfFkMpk1OI5jsbGxR5hMZs36\n9ev3qlKf0p721q1bt5aXl7vfvXt3SkxMzNGenh6jZcuWnbh27drbg71IAAAYKQgEgjg5OTkhMDAw\nTyKRGMbGxh5hMBi1KSkp8QghFB8fn+Lg4HAnKCgo18XFpcrAwKA3Li7uMJPJrLl69eq0EydOLHNx\ncalisVgVCCG0ffv2LUFBQbny6sNwXPFwjKura2VFRQXL3d29vKKigoUQQi4uLlXDOaaNYRg+evRo\nhJDqY9pjxoxRWkaVNyIVlZHXFnNz+U8/ksnkAZ9P3rXI267K78jU9D8+jQ7AiMDhcJC6//rHMAxf\nt26dSmWTkpLUrk8dSodHRo0a9dzAwKBXut7d3W0yvE0CAAAgj9LQjoyMPB0fH5/S1tZmcejQoVWz\nZs26uHLlyh800TgAAACvUjqm/emnn+68cOHCHDMzs866urrJ//u///uv2bNn/66JxgEAAHiV0tDe\ntGlTYmJi4qY5c+Zc6LtteJsGAACgL6XDIxcuXJjTd9v58+fnDk9zAAAAKCK3p33w4ME1Bw4cWNvQ\n0DDJ2dn5lnR7Z2en2dtvv31NM80DAAAgS25oL1myJC04ODhn8+bNOxITEzdJH3ExMzPrHDt27GN1\nKm1ra7NYuXLlD9XV1Y4YhuE//vjj+15eXsXqnBMAAF4HckObTCa3k8nk9vT09KjCwsLp9+7ds4uJ\niTn66NGjcY2Njba2traNg6103bp1SXPnzj2fmZkZIRaLCfAYIQBA2xS9U6FLlI5pb926des333zz\n2fbt27cghFBPT4/R0qVLTw62wvb2dnJhYeH0999//0eEXrxNRCaT2wd7Pk2BD1cBAHSB0qdHzp49\nO1/6RiRCLz6O0tXVNejX6xobG23HjRv3KCYm5mhlZaWru7t7eVJS0joSiSSULScWixFCCHV1dSEj\nIyNkZGQ02CoBACNIeXk5Ki8v13YztEbjb0SKxWICl8tlr1279gCXy2WbmJh079ixY3PfcgQCAREI\nBGRqagqBDQB4yd3dHa1aterl8rrR+BuRVCq1iUqlNnl4eNxACKGIiIhMLpfLHuz5AADgdaLxNyLH\njx//gEaj8evq6iZPnjy5Lj8/P8DR0bF6sOcDAIDXidLQRgihOXPmXPD09CwRi8UEDMPw1tZWS0tL\ny9bBVrp///4Ply5derKnp8do0qRJDUePHo0Z7LmGkipfAQQAAG1SGtopKSnxX3755f/Ijm1jGIb/\n+eefbw22UldX18obN254DPZ4AAB4XSkd0965c+ent2/fdvrrr78mNjY22jY2NtqqE9gAADDS5Obm\nBjk4ONyxt7evl/ddpoKCAj8Wi1Xh5OR028/Pr2Agx8pS2tN+6623/jQ2Nn46oCvQMFUmQAAAgOEg\nkUgMExISkvPz8wMoFIrAw8PjRkhISDaDwaiVlmlra7P44IMPvsvLywukUqlNLS0tVqoe25fS0N6x\nY8dmb2/v697e3teNjIx6EHoxPLJv376PhuKCR7r29na9edMKADBwpaWlU+3s7O7R6XQeQghFRUWl\nZ2VlhcoGb1pa2pLw8PAz0gl/raysWlQ9ti+lob1q1apDAQEB+c7OzrcMDAx6cRzHVJkyHgAAXgcC\ngYBCo9H40nUqldpUUlLiKVumvr7eXiQSEf39/S91dnaarVu3Lmn58uXHVTm2L6WhLZFIDL/99tuP\nB3MxAACgL+R9quLevXuooaFB7nGqdGJFIhGRy+WyL168OEsoFJK8vb2ve3l5FQ+mA6w0tIODg3NS\nUlLiQ0JCskeNGvVcul2dR/4AAEBf2NnZITs7u5freXl5r+ynUCgCPp9Pk67z+XyadBhEikaj8a2s\nrFqMjY2fGhsbP50xY8aVyspKVyqV2qTs2L6UhnZaWtoSDMPwvq+aNzY22io79nXR0dGhcEZ2AMDI\nxeFwyurr6+15PB59woQJ9zMyMhadOnVqsWyZ0NDQrISEhGSJRGL4/PnzUSUlJZ4ff/zxt5MnT65T\ndmxfSkObx+PR1bwmAAAYsQgEgjg5OTkhMDAwTyKRGMbGxh5hMBi1KSkp8QghFB8fn+Lg4HAnKCgo\n18XFpcrAwKA3Li7uMJPJrEEIof6OVVQfhuPKh1Ru377tVFNTw3z27Nlo6bbo6OhUNa9VfqMwDB89\n+kVVqnwSVdVH/pS98ahov7J2KOppy3t6RN455V2PvO2q/I5MTQf9YUYAdBqHw0HSSVoGC8Mw/Ntv\nv1Wp7Mcff6x2fepQ2tPeunXr1suXL/tWV1c7vvPOO/+Xk5MTPG3atKvDGdoAAAD6p/SNyMzMzIj8\n/PyAN95445+jR4/GVFZWura1tcGMAAAAoAVKQ9vY2PipoaGhhEAgiNvb28nW1tYPZe92AgAA0Byl\nwyMeHh43njx5MiYuLu4wh8MpMzEx6fbx8SnSROOGEnzBDwAwEigMbRzHsc2bN+8YM2bMk9WrV38f\nGBiY19HRYe7q6lqpqQYCAAD4N6U97blz556/ffu2E0IIqTMDOwAAAPUpHNPGMAx3d3cvLy0tnaqp\nBgEAAJBPaU+7uLjY68SJE8smTpz4l4mJSTdCL8K8qqrKZfibBwAAQJbS0M7LywvUREMAAECb9OW7\n/EpDW/qd14cPH1rLvhEJAABA85Q+p52dnR1ib29fb2tr2+jr63uZTqfzgoODczTROAAAAK9SGtpf\nfPHF19evX/eePHlyXWNjo+3FixdneXp6lmiicQAAAF6lNLSJRKLIysqqpbe310AikRj6+/tfKisr\n42iicQAAAF6ldEx7zJgxTzo7O82mT59euHTp0pPW1tYPTU1NuzTROAAAAK9S2tM+d+5cGIlEEu7d\nu3d9UFBQrp2d3b1ff/11niYaBwAA4FVKe9qmpqZd//zzzxulpaVTLS0tW4OCgnLHjh37WBONAwAA\n8CqlPe0ffvhhpaenZ8kvv/yy4MyZM+Genp4lR44cidVE4wAAQB/k5uYGOTg43LG3t69PTEzc1Hd/\nQUGBH5lMbmexWBUsFqvi66+//kK6b/v27VscHR2rnZ2dby1ZsiTt+fPnoxTVpbSn/c0333xWUVHB\nkvauHz9+PNbb2/t6bGzskcFcHAAAjCQSicQwISEhOT8/P4BCoQg8PDxuhISEZPedNszX1/dydnZ2\niOw2Ho9HP3z4cFxtbS1j1KhRzxctWpSRnp4etWLFimPy6lPa07aysmqRvfFoamraZWVl1TKYixuM\ntrY2TVUFAAADVlpaOtXOzu4enU7nEYlEUVRUVHpWVlZo33L9TVFmbm7eQSQSRUKhkCQWiwlCoZBE\noVAEiupT2tOeNGlSg5eXV3FoaGgWQghlZWWFuri4VO3evXsjhmH4xx9/rNrEagAAMAIJBAIKjUbj\nS9epVGpTSUmJp2wZDMPwoqIiH1dX10oKhSLYtWvXJ0wms8bS0rJ148aNu998882/jY2NnwYGBuYF\nBATkK6pPpdCeNGlSA4ZhOEIvpoLHMAzv6urSmZlinzx5ojffDQAA6CZ5GXL79m10+/ZtucdJs1ER\nNpvN5fP5NBKJJMzJyQkOCws7V1dXN7mhoWHS3r171/N4PDqZTG6PjIw8ffLkyaVLly49Ke9cKk3s\nq6wMAACMVE5OTsjJyenlenp6+iv7KRSKQHYKRj6fT6NSqU2yZczMzDqlPwcHB+esXbv2wOPHj8eW\nlZVxfHx8iqT3DBcsWPBLUVGRj6LQVjqmDQAAQD4Oh1NWX19vz+Px6D09PUYZGRmLQkJCsmXLNDc3\n20jHtEtLS6fiOI6NHTv28ZQpU+4WFxd7PX361BjHcSw/Pz+AyWTWKKpPaU8bAACAfAQCQZycnJwQ\nGBiYJ5FIDGNjY48wGIzalJSUeIQQio+PT8nMzIw4ePDgGgKBICaRSML09PQohBByc3O7GR0dncrh\ncMoMDAx62Ww2d9WqVYcU1YfhuOLhmMePH4/V9Ms0GIbho0f/+yuwFhYWSo9RNqatysS+isooa4O5\nubncfWQyeUDnlHct8rar8vsxNdWZWxAADCkOh9PvkxkDgWEYfu7cOZXKhoWFqV2fOpQOj3h5eRVH\nRkaePn/+/FxtNlRdLS0ae0oRAACGjdLQvnv37pS4uLjDqamp0XZ2dve2bNmyva6ubrK6FUskEkMW\ni1Uxb968X9U9FwAAvC6UhraBgUHvnDlzLqSnp0cdPnw47tixYys8PDxu+Pr6Xi4qKvIZbMVJSUnr\nmExmjSqPywAAAHhBaWi3tLRYJSUlrXN3dy/ftWvXJ8nJyQktLS1Wu3fv3rhkyZK0wVTa1NREPX/+\n/NyVK1f+oM9DLgAAoGlKnx7x8fEpWrZs2YmsrKxQ2WcPORxO2erVq78fTKUbNmzYs3Pnzk87Ojrk\n370DAADwH5SG9tdff/3FwoULf5bd9vPPPy9cuHDhz5s3b94x0Ap/++23d62trR+yWKyKgoICP3nl\nxGLxy597enqQkZHRQKsCAIxA5eXlqLy8XNvN0Bqlj/yx2Wwul8tly25jsVgVFRUVrMFU+Pnnn287\nfvz4cgKBIH727Nnojo4O8/Dw8DOpqanRLxs1DI/8IaT8sT945A8A/fO6PfInt6edk5MTfP78+blN\nTU3Ujz76aJ+0kZ2dnWZEIlE02Aq3bdv2+bZt2z5HCKHLly/77tq16xPZwAYAAG1QpfOjC+SG9oQJ\nE+67u7uXZ2Vlhbq7u5dLQ9vc3Lxjz549G4aqAfD0CAAAqE7p8IhIJCKq07MeDBgegeERAFQ1VMMj\nBQUFKpX18/PTzeERZ2fnW/L2YRiGV1VVuQxPkwAAAMgjN7RhxnUAANA9ckObTqfzNNgOAAAAKlD6\nRuT169e9PTw8bpiYmHQTiUSRgYFBr7m5eYcmGgcAAOBVSkM7ISEhOS0tbcnkyZPrnj17NvrIkSOx\na9euPaCJxgEAAHiVSjPX2Nvb10skEkNDQ0NJTEzM0dzc3KDhbhgAAID/pDS0TUxMup8/fz7K1dW1\n8rPPPvvm22+//Rg+8gQAAP+Wm5sb5ODgcMfe3r4+MTFxU9/9BQUFfmQyuZ3FYlWwWKyKr7/++gvp\nvra2NouIiIhMBoNRy2Qya4qLi70U1aX02yOpqanRvb29BsnJyQl79uzZ0NTURD1z5kz44C4NAABG\nFolEYpiQkJCcn58fQKFQBB4eHjdCQkKyGQxGrWw5X1/fy9nZ2SF9j1+3bl3S3Llzz2dmZkaIxWJC\nd3e3iaL6lIa29CkSY2PjpzAzOwAAvKq0tHSqnZ3dPWlWRkVFpWdlZYX2De3+Rija29vJhYWF048d\nO7YCoRfzTZLJ5HZF9SkN7atXr077n//5ny95PB5dLBYTEHrxcs2ff/751gCuCwAA9NLNmzfRzZs3\n5e4XCAQUGo3Gl65TqdSmkpIST9kyGIbhRUVFPq6urpUUCkWwa9euT5hMZk1jY6PtuHHjHsXExByt\nrKx0dXd3L09KSlpHIpGE8upTGtqxsbFH9u7du57NZnMNDQ0lKl4nAADoFXmfepg2bRqaNm3ay/Wf\nfvrplf2qfD+JzWZz+Xw+jUQiCXNycoLDwsLO1dXVTRaLxQQul8tOTk5O8PDwuLF+/fq9O3bs2PzV\nV1/9t7xzKb0RaWFh0RYcHJxjY2PTbGVl1SJdlB0HAACvAwqFIuDz+TTpOp/Pp8lOGIMQQmZmZp3S\n3nNwcHDCToFuAAAZ2UlEQVSOSCQitra2WlKp1CYqldrk4eFxAyGEIiIiMvt+CrsvpT1tf3//S59+\n+unOBQsW/DJq1Kjn0u1sNps70IsDAICRhsPhlNXX19vzeDz6hAkT7mdkZCw6derUYtkyzc3NNtbW\n1g8xDMNLS0un4jiOWVpatiKEEI1G49fV1U2ePHlyXX5+foCjo2O1ovqUhnZxcbEXhmF4WVkZR3b7\npUuX/Adzgdqi7At/AAAwGAQCQZycnJwQGBiYJ5FIDGNjY48wGIzalJSUeIQQio+PT8nMzIw4ePDg\nGgKBICaRSML09PQo6fH79+//cOnSpSd7enqMJk2a1HD06NEYRfUp/TSrNgzHp1lVCW34NCsA+meo\nPs1aVlamsfrUoXRM+8GDB+NjY2OPBAUF5SKEUE1NDfPIkSOxw980AAAAfSkN7ffee++nOXPmXLh/\n//4EhF680j6UM9cAAABQndLQbmlpsVq0aFGG9HE/IpEoIhAIYmXHAQAAGHpKQ9vU1LTr8ePHY6Xr\nxcXFXsre2BlKQzXVGAAAjARKnx7ZvXv3xnnz5v36559/vuXj41P06NGjcZmZmRGaaBwAAIBXKQ1t\nd3f38itXrsy4e/fuFBzHsSlTptzV1ES/0MsGAIBXKQ3tp0+fGh84cGDt1atXp2EYhk+fPr1wzZo1\nB0ePHv1MEw0EAADwb0pDOzo6OtXc3Lzjo48+2ofjOJaWlrZk+fLlx0+fPh05nA1TpZcNAABDxczM\nTNtNUInS0K6urnasqalhStdnzpz5B5PJrBneZqkGhkYAAK8bpU+PsNls7vXr172l68XFxV7u7u7l\nw9ss3dPW1qbtJgAAgPKedllZGeftt9++RqPR+BiG4X///febU6ZMuevs7HwLwzC8qqrKRRMN7Qt6\n2QCA15HS0IZJfAEAQHeoPN0YAAAA7VM6pq2LYGgEAPC60svQHi4tLTAhDwBAt70WoQ0TIAAARgq9\nC20YGgEA6Jrc3NwgBweHO/b29vWJiYmb+u4vKCjwI5PJ7SwWq4LFYlV8/fXXX8jul0gkhiwWq2Le\nvHm/KqtL6Y1IAAAA8kkkEsOEhITk/Pz8AAqFIvDw8LgREhKSzWAwamXL+fr6Xs7Ozg7p7xxJSUnr\nmExmTWdnp9LXMvWqpw29bACAriktLZ1qZ2d3j06n84hEoigqKio9KysrtG85eVOUNTU1Uc+fPz93\n5cqVP6gyjRn0tAEAQIGSkhJUWloqd79AIKDQaDS+dJ1KpTaVlJR4ypbBMAwvKirycXV1raRQKIJd\nu3Z9Iv0cyIYNG/bs3Lnz046ODvkTzcqA0AYAAISQiYlJv9tnzpyJZs6c+XI9OTn5lf0YhimdHZ3N\nZnP5fD6NRCIJc3JygsPCws7V1dVN/u233961trZ+yGKxKgoKCvxUaafGh0f4fD7N39//kqOjY7WT\nk9Ptffv2faTKcTA0AgDQRRQKRcDn82nSdT6fT6NSqU2yZczMzDpJJJIQIYSCg4NzRCIR8fHjx2OL\niop8srOzQ2xtbRsXL1586o8//pgZHR2dqqg+jYc2kUgU7dmzZ0N1dbVjcXGx13ffffdBbW0tQ9Pt\nAACAocDhcMrq6+vteTwevaenxygjI2NRSEhItmyZ5uZmG+l4dWlp6VQcx7GxY8c+3rZt2+d8Pp/W\n2Nhom56eHjVz5sw/UlNToxXVp/HhkfHjxz8YP378A4RezD/JYDBq79+/P6HvnVYAANAHBAJBnJyc\nnBAYGJgnkUgMY2NjjzAYjNqUlJR4hBCKj49PyczMjDh48OAaAoEgJpFIwvT09Kj+zqXKUAuG40rL\nDBsej0f39fW9XF1d7Whqatr1slEYhpuamr4sZ2lpKXe8SRUDeblGUVlFEzOYm8u/h0Amkwd0PnlD\nQfK2qzJhhOzvEwB9Vl5ejsrL//116EOHDsl9MkNVGIbhTU1NygsihKhUqtr1qUNrNyK7urpMIyIi\nMpOSktbJBraUNGTUHcuGtyEhsMHI4u7ujtzd3V+uHzp0SIut0TytPKctEomI4eHhZ5YtW3YiLCzs\n3HDVM9DA1mQvW56h7mVDYAMwsmg8tHEcx2JjY48wmcya9evX71VUVp1etq73sDUxByYENgAjj8ZD\n+9q1a2+fOHFi2aVLl/yl7+H3N9GCph/x04Ve9lCCwAZgZNL4mPa0adOu9vb2DutfFvrayx6OG5AA\ngJFFr749ogpdCWzoZQMAhsOICu3BBvZwDI0M9HzwxicAQBUj5tsjutLDHmoDHRqBXjYAg0MikbTd\nBJWMqJ62JunqDUgAwMg2IkJbnV72YIdGBmOobkACAF5feh/awxXYg6WJXjYMjQDw+tLr0B7OwIYb\nkAAAXaS3oa2LNx6HspcNYQ4A6I/ehrY61An8oexlDwYMjQDwetPL0B7ucezBhOxgetlwAxIAMFB6\nF9ravPE4mF72UILX1gEAehXamghsbT/mNxgwNAKAduXm5gY5ODjcsbe3r09MTNzUd39BQYEfmUxu\nl34k7+uvv/4CocHNmas3b0Tq4o1HKbgBCcDrSyKRGCYkJCTn5+cHUCgUgYeHx42QkJDsvlMo+vr6\nXs7Ozg6R3SadM9fNze1mV1eXqbu7e/ns2bN/VzT9ol71tAdrKAJfV29AAgC0q7S0dKqdnd09Op3O\nIxKJoqioqPSsrKzQvuX6m6Js/PjxD9zc3G4i9OqcuYrq04uetqbGsbV9A3IwYGgEgKExatSofrdf\nuXIFFRYWyj1OIBBQaDQaX7pOpVKbSkpKPGXLYBiGFxUV+bi6ulZSKBTBrl27PmEymTWyZXg8Hr2i\nooLl6elZoqidOh/auvDGo6Z62TA0AoDumTFjBpoxY8bL9W3btr2yX5UZ1NlsNpfP59NIJJIwJycn\nOCws7FxdXd1k6X5lc+bK0unhEU0Gti73smFoBADdRaFQBHw+nyZd5/P5NCqV+srU7mZmZp0kEkmI\nEELBwcE5IpGI2NraaonQwOfM1dmetqqhq25vejCvqysK68E8LaJo32BepjEzM5O7D2hfZ2entpsA\nhhCHwymrr6+35/F49AkTJtzPyMhYdOrUqcWyZZqbm22sra0fYhiGl5aWTsVxHLO0tGwdyJy5Unof\n2vIMpneqqZBWth+CemSD/1YjC4FAECcnJycEBgbmSSQSw9jY2CMMBqM2JSUlHiGE4uPjUzIzMyMO\nHjy4hkAgiEkkkjA9PT0KoX/Pmevi4lLFYrEqEEJo+/btW4KCgnLl1YfhuNLhGI3DMAyfP3/+f2wf\nimECVcanBxPSCKk29DGY+R7lhTX8zw8AQlOmTOn3yYyBwDAM7+7uVqmsiYmJ2vWpQ2d72gMJ6KF4\nU1FeUA9XSCs7P/SqAQD90dnQHqpXxgd6s1CdkFb1JiMENQBgsHQ2tOUZ6kkGNBHSyuoZquEPExOT\nAZUHAOgfnQ3twYSzumPew92TljUUvWoIaQBePzob2sP1bLKqwTuUAS0FQQ0AUJfOhrYiw/HmoCrn\nHKqAloKgBgAMlM6GtrrBrM7xQx3OfSkLa3VCmkQiDfpYAIaDUCjUdhNGlBEb2qpSNaAH+2Gm4exN\nQ0ADfaAvf07lfTBK1+hsaCszXGPew9Fz7g+ENABgMHQ2tIc6lNX9hOlQPCc9kKCGgAYA9EdnQ1sb\n34lWJ5jVvVk4lCGtL//MGwmeP3+u7SaA14zOhraqNPWm4FA+wTFUAQ3hrH3w3wBoms6GtiZf2x7q\nR+qg1wwAGC46G9q69GzycI8vQzADAFSls6GtDl27iQehDAAYKjob2roSvEMVuFeuXHllnjl16NJf\nApcvX0a+vr7absaQG4nXNRKvSVfk5uYGrV+/fq9EIjFcuXLlD5s2bUqU3V9QUOAXGhqa9dZbb/2J\nEELh4eFnvvjii69VObYvnQ1tXQqmoVBYWNhvaOv7dY7UIBiJ1zUSr0kXSCQSw4SEhOT8/PwACoUi\n8PDwuBESEpLNYDBqZcv5+vpezs7ODhnMsbJ0NrS1ZbhClEAg6H1AAwD+U2lp6VQ7O7t7dDqdhxBC\nUVFR6VlZWaF9g7e/2W5UPVaWzoY2BBwAQB8IBAIKjUbjS9epVGpTSUmJp2wZDMPwoqIiH1dX10oK\nhSLYtWvXJ0wms0aVY/vS2dAmEHS2aYP21VdfabsJwwKuS3+MxGsaKqpmjqmpaafsOoZhSifaZbPZ\nXD6fTyORSMKcnJzgsLCwc3V1dZMH1c7BHDTctDlpJgDg9aNO5lAoFAGfz6dJ1/l8Po1KpTbJljEz\nM3sZ9MHBwTlr16490NraakmlUpuUHduXwWAbCgAAACEOh1NWX19vz+Px6D09PUYZGRmLQkJCsmXL\nNDc320j/YigtLZ2K4zhmaWnZqsqxfelkTxsAAPQFgUAQJycnJwQGBuZJJBLD2NjYIwwGozYlJSUe\nIYTi4+NTMjMzIw4ePLiGQCCISSSSMD09PUrRsQorxHEclmFc/v77b5qfn98lJpNZ7ejoeDspKekj\nbbdpqBaxWGzo5uZW8e677/6q7bYM1fLkyROL8PDwTAcHh1oGg1Fz/fp1L223aSiWbdu2bWEymdVO\nTk63Fi9enPbs2bNR2m7TQJeYmJgfra2tm52cnG5Jtz1+/NgyICDgd3t7+7rZs2dfePLkiYW22znc\nCwyPDDMikSjas2fPhurqasfi4mKv77777oPa2lqGtts1FJKSktYxmcwaVW7E6It169YlzZ0793xt\nbS2jqqrKRWmvRw/weDz64cOH47hcLvvWrVvOEonEUNrT0ycxMTFHc3Nzg2S37dixY/Ps2bN/r6ur\nmzxr1qyLO3bs2Kyt9mkKhPYwGz9+/AM3N7ebCCFkamraxWAwau/fvz9B2+1SV1NTE/X8+fNzV65c\n+QM+Qm4ct7e3kwsLC6e///77PyL04p+uZDK5XdvtUpe5uXkHkUgUCYVCklgsJgiFQhKFQhFou10D\nNX369MIxY8Y8kd2WnZ0dsmLFimMIIbRixYpj586dC9NO6zQHQluDeDwevaKiguXp6Vmi7baoa8OG\nDXt27tz5qYGBQa+22zJUGhsbbceNG/coJibmKJvN5sbFxR0WCoW68T0FNVhaWrZu3Lhx95tvvvn3\nhAkT7ltYWLQFBATka7tdQ6G5udnGxsamGSGEbGxsmpubm2203abhBqGtIV1dXaYRERGZSUlJ60xN\nTbu03R51/Pbbb+9aW1s/ZLFYFSOll40QQmKxmMDlctlr1649wOVy2SYmJt0j4Z/bDQ0Nk/bu3bue\nx+PR79+/P6Grq8v05MmTS7XdrqGGYRg+kobq5IHQ1gCRSEQMDw8/s2zZshNhYWHntN0edRUVFflk\nZ2eH2NraNi5evPjUH3/8MTM6OjpV2+1SF5VKbaJSqU0eHh43EEIoIiIik8vlsrXdLnWVlZVxfHx8\nisaOHfuYQCCIFyxY8EtRUZGPtts1FGxsbJofPHgwHiGE/vnnnzesra0fartNww1Ce5jhOI7FxsYe\nYTKZNevXr9+r7fYMhW3btn3O5/NpjY2Ntunp6VEzZ878IzU1NVrb7VLX+PHjH9BoNL70TbX8/PwA\nR0fHam23S10ODg53iouLvZ4+fWqM4ziWn58fwGQya7TdrqEQEhKSfezYsRUIIXTs2LEVI6FTpJS2\nH18Z6UthYeE0DMN6XV1db7q5uVW4ublV5OTkBGm7XUO1FBQU+M6bNy9b2+0YquXmzZuuHA7nhouL\nS+X8+fN/aWtrI2u7TUOxJCYmfiZ95C86OvpYT08PUdttGugSFRV16o033rhPJBJ7qFQq/8cff4x5\n/Pix5axZs/Jfp0f+MBwf8UNAAAAwYsDwCAAA6BEIbQAA0CMQ2gAAoEcgtAEAQI9AaOsJHo9Hd3Z2\nvqXJOofzJaCenh6jgICAfBaLVXH69OnIwsLC6Y6OjtVsNpt7//79CZGRkacVHR8XF3d4sN9wuXz5\nsu/169e9B9dyALQLPs36GpBIJIaGhoaSgR43nG+XcblcNoZheEVFBQshhFavXv39559/vm3p0qUn\nEULo9OnTkYqOP3z4cNxg67506ZK/mZlZp7e39/XBngMArdH2M4ewqLY0NjbSpZ+kbGhoeIvFYnHL\nysrc7927NykoKCjH3d29bPr06Vfu3LkzBcdxtGLFip/i4+O/9/T0LP744493v/fee0c//PDDfT4+\nPtfeeuuthszMzHDpub/55ptPPTw8Sl1cXCq//PLLrdLtpqamnf215dixY9EuLi6Vrq6uN5cvX54q\nbZ+/v/8fLi4ulbNmzcr/+++/aTiOo4cPH44LDw/P9PDwKPXw8Ci9du2az8OHD8dNmjTpHplMbnNz\nc6tISUlZZWlp+djW1vbPZcuWHefxeBMdHR1v4/iLz79u3Lhxl5OT0y0XF5fK5OTkD3AcR76+vgVl\nZWXuOI6jvLy8Od7e3kVsNrs8MjLy566uLhMcx9HEiRN5X3755VY2m13u7OxcdefOnSmNjY308ePH\n/0OhUJrc3NwqCgsLp8n7nXd3d5MiIyN/ZjKZ1fPnz//F09OzWFrnmjVrDnA4nBuOjo63ZX9nEydO\n5G3ZsmWbm5tbhbu7e1l5eTl79uzZFyZNmnTv+++/j8dxHF26dMlvxowZl0NDQ8+99dZbDZs2bdqR\nmpq63MPDo9TZ2bmqoaHhLRzHUXZ29jxPT89iFovFDQgI+L25udla238OYdH+ovUGwKLaIg3tO3fu\nTGGxWNyqqipnHMfRzJkzL9bX19vhOI6Ki4s9Z86ceRHHX4T2vHnzsnt7ezEcx9F77713dOHChRk4\njqOamhqGnZ1dPY6/CLxVq1al4DiOJBKJwTvvvPPblStXpuN4/6F9+/Ztx8mTJ999/PixJY6/+P40\njuPo3Xff/TU1NXU5juPoxx9/jAkLCzuL4zhavHhx2tWrV9/GcRz99ddfbzIYjBocf/FSjux3uN97\n772jZ86cWSB7rTiOowMHDqyJjIz8WSKRGOA4jlpbW8fgOI78/PwulZeXsx89emQ1Y8aMy0Kh0BjH\ncbRjx45NX3311b9wHEd0Or1RGvIHDhxYs3LlysM4jqOtW7d+uXv37o+V/c537tz5yerVqw9Kr5tA\nIIjKy8vZsu0Qi8WGfn5+l27duuUkrVMazhs2bPjW2dm5qqury+TRo0dWNjY2D3D8RWhbWFg8efDg\ngc3z58+NJkyYIJAGf1JS0kfr16/fI/u7xXEcHT58eOXGjRt3afvPISzaX2B4RI88fPjQOiws7NzZ\ns2fnOzg43Onq6jK9fv26t+z4b09PjxFCL4Y2IiMjT8sOcUhf8WUwGLXSr6FduHBhzoULF+awWKwK\nhBDq7u42uXfvnt306dML+2vDH3/8MXPhwoU/W1patiKEkIWFRRtCCBUXF3tJP4u5bNmyE5999tk3\nCL14FVx27Lmzs9Osu7vbBO/nQ1P9bbt48eKsNWvWHJR+TVD205w4jmPFxcVeNTU1TB8fnyLp9Ut/\nRgihBQsW/ILQi4lVf/nllwWK6urr2rVrb0s/PeDo6Fjt4uJSJd2XkZGx6PDhw3FisZjwzz//vFFT\nU8N0cnK6jdCLV6sRQsjZ2flWd3e3iYmJSbeJiUn3qFGjnnd0dJgjhJCHh8cN6dfp7Ozs7gUGBuYh\nhJCTk9PtS5cu+SP0Yr7AhQsX/vzgwYPxPT09Rra2to3K2gxGPghtPWJhYdE2ceLEvwoLC6c7ODjc\n6e3tNbCwsGiTjgv3RSKRhLLrRkZGPdKfZUNry5Yt21etWnVIlTZgGIbLCzx5QVxSUuIpW/dAKQvY\n2bNn/56Wlrakv32jRo16jhBChoaGErFYPOA/7/3V3djYaLt79+6NZWVlHDKZ3B4TE3P02bNno/vW\naWBg0Ct73QYGBr3SNkjLSLfLHiMt8+GHH+7/5JNPdr377ru/Xb582Xfr1q1bB9p+MPLA0yN6xMjI\nqOeXX35ZkJqaGn3q1KnF5ubmHba2to2ZmZkRCL0ImKqqKpeBnDMwMDDvxx9/fL+7u9sEIYQEAgHl\n0aNH4+SVnzlz5h+nT5+ObG1ttUQIoSdPnoxBCCEfH58i6WwoJ0+eXDpjxowrCCE0Z86cC/v27ftI\nevzNmzfdBtK+2bNn/56SkhIvkUgMZetD6MVfIF5eXsXXrl17u6GhYRJCL/6lUF9fb6/onGZmZp2d\nnZ1m0vWzZ8/O//zzz7f1Lff2229f+/nnnxcihFBNTQ3z1q1bzggh1NHRYW5iYtJtbm7e0dzcbJOT\nkxPcXz2q9OYV6ejoMJ8wYcJ9hBD66aef3lPnXGDkgNDWIxiG4SQSSfjbb7+9u2fPng2//fbbuydP\nnlx65MiRWDc3t5tOTk63s7OzQ2TL9z2+78+zZ8/+fcmSJWne3t7XXVxcqiIjI093dXWZ9nc8Qggx\nmcya//qv//p/vr6+l93c3G5u3LhxN0II7d+//8OjR4/GuLq6Vp48eXJpUlLSOoQQ2rdv30dlZWUc\nV1fXSkdHx+pDhw6tkp5blfatXLnyhzfffPNvFxeXKjc3t5unTp1aLHuMlZVVy08//fTe4sWLT7m6\nulb6+PgU3b17d0p/vzvpOefNm/fr2bNn57PZbO7Vq1enNTQ0TOpvhpq1a9ceePTo0ThHR8fqf/3r\nX//r6OhYTSaT211dXStZLFaFg4PDnaVLl56cNm3aVXn/vfq7JkXffZbdt3Xr1q2RkZGnORxO2bhx\n4x69Dt+KBsrBB6PAa2/58uXH9+7du37s2LGPZbf39vYaiEQi4qhRo543NDRMks5FSCAQxNpqKwAQ\n2gDI0dnZaTZz5sw/RCIREcdx7JtvvvlMesMQAG2B0AYAAD0CY9oAAKBHILQBAECPQGgDAIAegdAG\nAAA9AqENAAB6BEIbAAD0yP8H2wuTQDJwp/QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10a5a7610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "AUC_grid = np.array(AUC_all).reshape(gam_vec.shape)\n",
    "\n",
    "contourf(gam_vec, cost_vec, AUC_grid, 20, cmap='Greys')\n",
    "xlabel(\"kernel coefficient, gamma\")\n",
    "ylabel(\"penalty parameter, C\")\n",
    "colorbar()\n",
    "savefig(\"figures/figure-4.25.eps\", format='eps')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}