Stochastic_validations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stochastic validations\n",
    "\n",
    "We'd like to score the teams based on 100 realizations of their models — most of which are stochastic and take a random seed parameter. Please see the notebooks with `_VALIDATION` suffixes to see how the realizations we generated. Each one of those notebooks generartes a file called `<team>_100_realizations.npy`, which is what we are consuming here.\n",
    "\n",
    "This notebook is super hacky I'm afraid."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "from utils import accuracy, accuracy_adjacent, display_cm, facies_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using globals. I am a miserable person."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Well Name</th>\n",
       "      <th>Depth</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">STUART</th>\n",
       "      <th>2808.0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2808.5</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2809.0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2809.5</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2810.0</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Unnamed: 0\n",
       "Well Name Depth             \n",
       "STUART    2808.0           0\n",
       "          2808.5           1\n",
       "          2809.0           2\n",
       "          2809.5           3\n",
       "          2810.0           4"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PRED = pd.read_csv('prediction_depths.csv')\n",
    "PRED.set_index([\"Well Name\", \"Depth\"], inplace=True)\n",
    "PRED.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>LithCode</th>\n",
       "      <th>LithLabel</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Well Name</th>\n",
       "      <th>Depth</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">STUART</th>\n",
       "      <th>2807.5</th>\n",
       "      <td>3</td>\n",
       "      <td>NM Shly Silt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2808.0</th>\n",
       "      <td>3</td>\n",
       "      <td>NM Shly Silt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2808.5</th>\n",
       "      <td>3</td>\n",
       "      <td>NM Shly Silt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2809.0</th>\n",
       "      <td>3</td>\n",
       "      <td>NM Shly Silt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2809.5</th>\n",
       "      <td>3</td>\n",
       "      <td>NM Shly Silt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  LithCode     LithLabel\n",
       "Well Name Depth                         \n",
       "STUART    2807.5         3  NM Shly Silt\n",
       "          2808.0         3  NM Shly Silt\n",
       "          2808.5         3  NM Shly Silt\n",
       "          2809.0         3  NM Shly Silt\n",
       "          2809.5         3  NM Shly Silt"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRUE = pd.read_csv('blind_stuart_crawford_core_facies.csv')\n",
    "TRUE.rename(columns={'Depth.ft': 'Depth'}, inplace=True)\n",
    "TRUE.rename(columns={'WellName': 'Well Name'}, inplace=True)\n",
    "TRUE.set_index([\"Well Name\", \"Depth\"], inplace=True)\n",
    "TRUE.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_accuracies(y_preds):\n",
    "    \"\"\"\n",
    "    Get the F1 scores from all the y_preds.\n",
    "    y_blind is a 1D array. y_preds is a 2D array.\n",
    "    \"\"\"\n",
    "    accs = []\n",
    "    for y_pred in y_preds:\n",
    "        PRED['Facies'] = y_pred\n",
    "        all_data = PRED.join(TRUE, how='inner')\n",
    "        y_blind = all_data['LithCode'].values\n",
    "        y_pred = all_data['Facies'].values\n",
    "        y_pred = y_pred[y_blind!=11]\n",
    "        y_blind = y_blind[y_blind!=11]\n",
    "        cv_conf = confusion_matrix(y_blind, y_pred)\n",
    "        accs.append(accuracy(cv_conf))\n",
    "    return np.array(accs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LA-Team             0.6388\n",
      "PA-Team             0.6250\n",
      "ispl                0.6231\n",
      "esaTeam             0.6225\n",
      "SHandPR             0.6200\n",
      "HouMath             0.6188\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEACAYAAABWLgY0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG8RJREFUeJzt3WtwXGed5/Hvv093u20rkUwU7BCPo5kEEshUJsxQIezs\nbhSWmaRqd4rsLMNAgMG7S4psMTszRa3LQG0ZOS92NhWWYi47460NJAwsRVGZwoZQQJZNZGKYXEh8\nSRxfcrHSdmLJacstqyV3n748+6LbslpqSS31aav1+PepUnL6XJ7+P+7WT0enz6PHnHOIiIifYstd\ngIiItI9CXkTEYwp5ERGPKeRFRDymkBcR8ZhCXkTEYwuGvJltNLPHzewlM3vBzP5zbf2XzeyEmT1f\n+7qz/eWKiMhi2EL3yZvZBmCDc26fmXUBzwEfBv4YGHfOfbX9ZYqIyFLEF9rBOTcMDNeWc2Z2CLi6\nttnaWJuIiLRoUdfkzawPuBl4urbqc2a2z8weNLPuiGsTEZEWNR3ytUs1jwB/7pzLAX8HXOucu5nq\nmb4u24iIdJgFr8kDmFkceBT4sXPurxpsvwb4oXPupgbb9MdxRESWwDnX8iXxBa/J13wDeGl6wJvZ\nhtr1eoA/BF6c62Cf/wjawMAAAwMDy11G20Tdv62bt3Jv372RtLVjaAf3P3z/ko/Xa7ey+d4/s2g+\n8lww5M3sd4FPAC+Y2V7AAV8C7jazm4EKMAR8NpKKREQkMs3cXfMLIGiw6SfRlyMiIlHSiNcW9ff3\nL3cJbeVz/3zuG6h/UqWQb5HvbzSf++dz30D9kyqFvIiIxxTyIiIeU8iLiHhMIS8i4jGFvIiIxxTy\nIiIeU8iLiHhMIS8i4jGFvIiIxxTyIiIeU8iLiHis2b8nL9Jx9u/dz9bNW1tup3dTL1vu2xJBRSKd\nRyEvK1ZpohTJBCQ7hnZEUI1IZ9LlGhERjynkRUQ8ppAXEfGYQl5ExGMKeRERjynkRUQ8ppAXEfGY\nQl5ExGMKeRERjynkRUQ8ppAXEfGYQl5ExGMKeRERjynkRUQ8ppAXEfGYQl5ExGMKeRERjynkRUQ8\nppAXEfGYQl5ExGMKeRERjy0Y8ma20cweN7OXzOwFM/uz2vp1ZvaYmR0xs5+aWXf7yxURkcVo5ky+\nBHzeOfce4APA58zsBuALwM+cc9cDjwNfbF+ZIiKyFAuGvHNu2Dm3r7acAw4BG4EPA9+s7fZN4K52\nFSkiIkuzqGvyZtYH3Aw8Bax3zo1A9QcBcGXUxYmISGvize5oZl3AI8CfO+dyZuaaPXZgYGBqub+/\nn/7+/kWUKO22bdvXSKezLbXx+t4nCCbGAFi7Nsl73/uehvvt3/MKu/ddtWB7qe5u3n97f0s1dZpt\nDzxAOpOJtM1Nvb3ct2VLpG3K8hgcHGRwcDDydpsKeTOLUw34bznndtVWj5jZeufciJltAE7Ndfz0\nkJfOk05n6esbaKmN7L4h/mJjX3U5O8hdfX0N99u6b5jbenoWbG93trUfOp0oncnQd++9kbY5tGNH\npO3J8pl5Arx9+/ZI2m32cs03gJecc381bd0PgM215U8Du2YeJCIiy2vBM3kz+13gE8ALZrYXcMCX\ngPuB75nZfwDSwB+1s1AREVm8BUPeOfcLIJhj84eiLUdERKKkEa8iIh5TyIuIeEwhLyLiMYW8iIjH\nmh4MJSvTA9seIJOePQDn0N59hBMTAIyMjPJS4mfztuOSxtUbrp1z++jJvdDTt6jaHjz9fbLB6Ybb\nRsKQZ596ct7jT9tBvvLUf5l63BN/B5953+cXVYOI7xTynsukM9zbN3sAzu59u7htY3VQ0pHcEKlU\n37ztPJz/LpvnCfFt6T2Lri0bnOYzd/U23PZ6Ps81N2yc9/j1B1bzBzdd2OfB755YdA0ivtPlGhER\njynkRUQ8ppAXEfGYQl5ExGMKeRERjynkRUQ8ppAXEfGY7pO/yLb95TbSI+mW2ti0fhP3ffG+Bfd7\n4IFtPPfKj/jK8Cuzto2Ex3k2mwTg7OocQdA1b1uvrD7GI6fP8ZErbl9a0SKyLBTyF1l6JE3fXX0t\ntTG0c6ip/TKZNH/8kW7e1zN7UNHrhye4JpUC4HQmTjw+/2xN+0rD7NsztuhaRWR56XKNiIjHFPIi\nIh5TyIuIeEwhLyLiMYW8iIjHFPIiIh5TyIuIeEz3yUtHyo2P8/rhI/Pucy43UbfPSHqU3cO7Gu6b\n6u7m/bf3R1liR9i7fz+bt26NtM1Nvb3ct2VLpG3K8lHIS2eqVKYGa83lQBCr22d9MsltPY0Hde3O\nZiMtr1NMlEr03Tt75q9WDO3YEWl7srx0uUZExGMKeRERjynkRUQ8ppAXEfGYQl5ExGMKeRERjynk\nRUQ8pvvkO8ivHnmCUm7hiTly6Rxbt25ecL+DB/dy3VXzz/jUimMnnqBQqtabOzfC4aGdAIThMH//\n0MsUi+VZxxw6s5pn4r8C4HTPaU5nGr8Fw7DI6cyFe9uLxSKJRGLGPqW6fc6O5TgyPNSwvXSYZ9fO\n3XXrurtT9N/+/gV6eenRACu/KOQ7SCk3xsc2zz9DE0D2MHz4jr4F97vnnj1A+0K+UBoj1VetNzib\nmFomXw3ey1PvnHVM8nCWVane6oPK3DNSmRXqtuXzp1i9un7fmNUfHwQlUqm+hu0lydLTc1vdumx2\nd8N9L3UaYOUXXa4REfGYQl5ExGMLhryZfd3MRszswLR1XzazE2b2fO3rzvaWKSIiS9HMmfxDwB0N\n1n/VOffbta+fRFyXiIhEYMGQd87tAc402GTRlyMiIlFq5Zr858xsn5k9aGbdkVUkIiKRWWrI/x1w\nrXPuZmAY+Gp0JYmISFSWdJ+8c+6taQ//N/DD+fYfGBiYWu7v76e/v38pTysi4q3BwUEGBwcjb7fZ\nkDemXYM3sw3OueHawz8EXpzv4OkhLyIis808Ad6+fXsk7S4Y8mb2HaAfuMLM0sCXgdvN7GagAgwB\nn42kGhERidSCIe+cu7vB6ofaUIuIiERMI15FRDymkBcR8ZhCXkTEYwp5ERGPKeRFRDymSUOkaa9O\nDPMwO6cej8XTBNnqcInJrhyPZg8DUC7nAAjy+VltHF8d8r2g2saJ4ghwbZurFrm0KeSlaS4Rsvmu\nCzMxZbLDxHtSADwzEnDL+upyqVQN93g8NauNoxm4qTab0/YfDrW5YhHR5RoREY8p5EVEPKaQFxHx\nmEJeRMRjCnkREY8p5EVEPHbJ3EL51DNPsfvp3S210dvTy+a7NxMEQURViYi01yUT8vsP7edw5TA9\nG3oW3nkOh54+xN0fuZvVq1dHWFlzhvceg4kC4+MTFM+W+NbRHQse8+bro7z60hjJYPac6+Viga7U\nWgAKhZD4JfNOaM62Bx4gnclE2ubegwfpi7TFlWPv/v1s3ro10jY39fZy35Ytkbbpo0vqW3tN9xq6\n1y99zvFsLBthNYs0UeDWrhSZfB5WraX38hsWPOSxxAE2JLu5Jt47a9ux4gjx2qCkfP5U5OWudOlM\nhr577420zT333BNpeyvJRKkU+b/n0I6FT3RE1+RFRLymkBcR8ZhCXkTEYwp5ERGPKeRFRDymkBcR\n8ZhCXkTEY5fUffIX27ZtXyOdrr+3fs+L+9jHUMP9w/Qwhw83vhffxnJk8nkKYcgqZk/GIfPL5U4y\nfHhX3bowHGb3ziz7c79gYPPmum37XnmFoauumrfNVHc3t/b3R1ypSLQU8m2UTmfp6xuoW7dvaIie\nnr6G+2eTWVKpxiNyS0GeeDylgUtLFJSL3Drj3zZPlut7ejhEFwN9fXXbhoaH6euZf3T0YHYZB8eJ\nNEmXa0REPKaQFxHxmEJeRMRjCnkREY8p5EVEPKaQFxHxmEJeRMRjuk9+BQrDcTLZwwvvV8yRDwMm\nS7NnOKpUiu0oTUQ6jEJ+BapQId6z8KhXSwTEkgGxBnP7udC1ozQR6TC6XCMi4jGFvIiIxxYMeTP7\nupmNmNmBaevWmdljZnbEzH5qZkufHVtERNqmmTP5h4A7Zqz7AvAz59z1wOPAF6MuTEREWrdgyDvn\n9gBnZqz+MPDN2vI3gbsirktERCKw1Gvyb3fOjQA454aBK6MrSUREotKRt1D+/OdP8JOffCvSNkN3\nGVwfaZMiIh1vqSE/YmbrnXMjZrYBmHcmi4GBganl/v5++heYTWd09C1uuSXgN39z/pl5mnXo0Ag/\n/tlZjDWRtOcD5ypM5quDpMquMLU8l3xljLCYqxuEVQjHibcwS5Vzbs7nLVfCum2NaixXQsYnThIE\niWqN4RiZs40Hic2sHSCfP8PYcxmOlF9m58583bZ0mCV/eGje+tNhnp27dtet6+5OcXv/++c9TqSR\nwcFBBgcHI2+32ZC32td5PwA2A/cDnwZ2NThmyvSQb1YsZgRBNHd4xmK28E6XoFiq+vJbLEYsOf9b\nIVYKsERQNwgrf2qsped3uKkaZrKS1W1rWGMphqNMLLW6WmMymHOQmBUabMsFrOrqxTKr6Onpr9uU\nzD5FKtU3b/1JsvT03Fa3LpvdPcfeIvObeQK8ffv2SNpt5hbK7wC/BN5lZmkz+/fAfwd+z8yOAB+q\nPRYRkQ6z4Jm8c+7uOTZ9KOJaREQkYhrxKiLiMYW8iIjHFPIiIh5TyIuIeEwhLyLiMYW8iIjHFPIi\nIh5TyIuIeEwhLyLiMYW8iIjHFPIiIh5TyIuIeEwhLyLisY6cGUo6z1gYMhGWOZq5MLlGsZAkUXtc\nDJNT2yqV6rlDLJaf1c54oRT5u+6tYpGSGS8V3uTv4//YcJ98b4nnrP6JK2tLxCzOa5en+dxrf1O3\nLXRl3vzeIcrFNaxbe2PDNl05JLvn5anH5St6CG5/Z4u9EYmWQl6aUwm4zLq4KX7D1KowliMZ7wLg\nDctxdW25UqmGeyw2ewKPJ/MHIy+tZEait5f4ZUf51EevbbjP2TDk8mSybl25HBIESfaffIvfuaq/\nbtvoqRO87e0b+c7OU3yw55MN2zxRytPVe2FOyZ2ZB1vriEgb6HKNiIjHFPIiIh5TyIuIeEwhLyLi\nMYW8iIjHFPIiIh5TyIuIeEz3yS/SfQ/cx8nRk/Pu8/o/7SUYn2BkZJQDiW/XbcvnT5PPXtPwuHKY\npnR4uPG28XFIzb7vXERkPgr5RTp+6jjXfeS6effJHt7HX7x7I4cPl0ilNtZtG3lxlPU9PQ2PezQ7\nzPvmCPJnxsaWVrCIXNJ0uUZExGMKeRERjynkRUQ8ppAXEfGYQl5ExGMKeRERjynkRUQ8pvvkl+hX\njzxBKdf43vXRt9I8nBtmbCJHkK+fHenc2hyrs4cbHvdmfhzQgCep98Tg04yNzZ5lC2Dk1Bl27tq9\n6Da7u1Pc3v/+VkuTFUAhv0Sl3Bgf29x4UNPwnmFu7UqRyeSJx+tDO3cqoOvtjYP8L7+nAU8y29hY\nnp6e2xpuSyS+Pee2+WSzi//BICuTLteIiHhMIS8i4jGFvIiIx1q6Jm9mQ8AYUAGKzrlboihKRESi\n0eoHrxWg3zl3JopiREQkWq1errEI2hARkTZpNaAd8FMze9bM7omiIBERiU6rl2v+mXNu2MyuBP6v\nmR1yzu2ZudPAwMDUcn9/P/39/S0+7eIdOvg87o0Cqa6lDzbKHc9RKVfI5F4kM3QSmH2f/Jt732Ri\npMy+0Tz5fIxYrH4QS7GQJJFpPLBlIixztLYtHlT4jXVrllyrtF8YjpPLHJl6fG7sOMX0ZQzuyjbc\nf/TUKQZ37ZqzvVR3N7cuw/eGdIbBwUEGBwcjb7elkHfODdf+/5aZfR+4BZg35JdLpTzBRz+5jrU9\na5fcRvYoVEoV3vaeHv7mv6YbP89EhXcnrmdjMs5kKU8sVv9DJYzlSMa7Gh57mR3gpvgNABwoNR4V\nK53DKhU2ThvsdmWQZGMyxQ1zzPx1IpGgf45tAIPZxj8c5NIw8wR4+/btkbS75Ms1ZrbGzLpqy2uB\n3wdejKQqERGJRCtn8uuB75uZq7Xzf5xzj0VTloiIRGHJIe+cOwbcHGEtIiISMd3+KCLiMYW8iIjH\nFPIiIh5TyIuIeEyThgDDe4/BRGHB/cKxEOcc4egpCmdyDO+ZfS/7uZEyxeI6SLb2TzsWhhzNXHhc\nDJMczeQpFhNQYs4BVdNNhGXGC6WOepVDV+FkqVRdxqaWZzoXBHXbyi5GMGPfgqtwKggoViok2ldy\n084Mpyn/KEd2T+M7iW0oz1MPPjrn8dkw5KknXyb+jh7e9/nPtKtMucR00Lf/MpoocGsTI2HzBXAV\nx+quFI8GQcNj9o3moeBar6kSTA2MAnjDclwd7yKsTEDFzTmgarrL7AAVZ63XEiEHJHp7AYglE1PL\nM8XCkEQyeeFxOSQIkvX7jCRJrLuCQocMIgpC4991f4ze3sYDng4kn+am3rmn3BvK5+nbeD3fPfFg\nu0qUS5Au14iIeEwhLyLiMYW8iIjHFPIiIh5TyIuIeEwhLyLiMd1CKXMqlMuUXfV20LKrUHYVJqfd\nq16sOEq1x2XnprZVKhUAYrHZ98BHcHOpiCyCQl7mdMY5KmvXYkApjFMKYGzthUlXyqtWEcSrb6Hi\nqlWM1Zadq44fMAvq2nOAm7wopS9asVRfmHNliqVJSuU8k/lMw2PKlZDJfAazgNWr1gFQKIRkMo3v\n2w/DEj94+SHC4HTD7ZOuwi/eSnFs9ADH/uDCoKlyqUyYbDwuYs1wluyDX2lc3xU9XPFv/R1UtXf/\nfjZv3Rppm68dPcpvvOtdkba5qbeX+7ZsibTNxVDIy7wSqRRmRuxsjFgQkFi9empbLAwJagOWbNrg\nJeeqZ/Bm9W8v5zr1PN5h8RlXLs2weAwLYsRSjb9NrGTEUnEq+Qu/sTgH8XjjwVAWi1NKjPPJuzY2\n3H62UiK55goOHn+NG3/tjqn1udxbdHVd2fCYh/7X09ze2zjId2b8HlQ1USrRd++9kba55557+GDE\nbQ7t2BFpe4ula/IiIh5TyIuIeEwhLyLiMYW8iIjHFPIiIh5TyIuIeGzF3UI5NjbGqVNvLeqYdHqU\niYlJstkY+fLsyUEKhQJu7SrMOutvr4uItGrFhXw6fZIjRyokk5c1fcyJE0kmcjHOnAmIF4LZO5wr\nMTqaxS0wwUb5bAlXcUxYgTAsNhz0ks/H6CpXmJzMUyqXSep3Je+VywUm8xnK5ZCyK8w9eKocUirn\nZw28qjLKrkI4eZpKOSScvDBgysqFusfTVSolcpkjDbedGx+m8R37cilZcSEPsHr12+jq2tD0/mNj\nCeKJFMlkF8lVa2dtL1iMctmRTK6bv6EgjzNHPL4aixUaDnqJxfKYxYjFUlCaaLpGWbkcjlgqjgUx\nbFWM2BxTP1pQHVw1a+AV4EoVcI7LY3HiZlweu9BGaDGSscZtBhgb43PMalYpLr4z4h2dZ4qIeEwh\nLyLiMYW8iIjHFPIiIh5TyIuIeEwhLyLisRV3C+VEbpzx06cIcyNNHzOWyVLKn6WUKcP4+KztxUKJ\nkWKMeJift51yoQg4gpxRKJU5mZu9f7lj/2a6iFyKVlzIj585zfqxDD2p5od5uIlJ1pXy3JBzBIX6\nEa+TpRKvhMYaewdryvMPsKrUpreLFeLEK5P0FOonf8iWxym50abrks53cuJ1nqTx5BsTxSKvlRKU\ni0WCYoJS1xC/HP8HgiDReP81Z5jMnwV621ixSL0VF/IAa4Mkb5tjOrRGRhOQisXpTiSmZjI67/xs\nRWtjq+gJ5m+zEivhnCMIEiQszroZ+59zeSg3XZasALFEgY/f1TiUz4YhlyeTlGszZB0YSXJj7xUE\nQbLh/mOnKjw0uLg/ySHSKl2TFxHxmEJeRMRjLYW8md1pZofN7KiZRTttuoiItGzJIW9mMeBvgTuA\nG4GPm9kNURW2Uhw4MfsvUfrk0FDjv6jog0NDjf+yoy8mc/6+dgDnzpxZ7hJWhFbO5G8BXnbOve6c\nKwLfBT4cTVkrh0J+5Tr8ut93Qk1O+P1DTCHfnFZC/mrg+LTHJ2rrRESkQ7RyC2WjGTYiGQkUi8U5\neDDH8ePHZ2179p8mmEi/RTJo/iwsH1YYLZ7j5fECzJj9qVxxVMoVRsqnGS3l5m/IVf9jpTjnXJ5j\npTc4UznLsdIbABRc2HRNK4E5RyUMKZ49S6VUolIsUzx7dmp7pVSiEq++hVypRPH8sqveR2rWYIIW\nEbmozC1xhKaZ3QoMOOfurD3+AuCcc/fP2E9DQEVElsAtNF1dE1oJ+QA4Avwr4CTwDPBx59yhVosS\nEZFoLPlyjXOubGZ/CjxG9dr+1xXwIiKdZcln8iIi0vnaPhjKzD5qZgfN7AUz+/a09Z+uHXfEzP6k\nlTraocW+/djMzpjZDy5exYuz1P6Z2W+Z2S9r6/aZ2UcvbuXNaaF/m8zsV2b2fG39Zy9u5c1p5f1Z\n23aZmZ0ws7++OBU3r8XvvXLttdtrZjsvXtXNa7F/v2ZmPzWzl8zsRTPbtOATOueW9EX1B8QrwDVA\nAtgH3DBjn+uA54DLa497a/9fB7wKdAM955eXWkvUX630rbZ8O/CvgR8sd1/a8NpdB1xbW74KePP8\nPp3y1WL/4kCitrwGOAZsWO4+Rfn+rD3+GvBt4K+Xuz9R9g04u9x9aHP/ngA+WFteA6QWes52D4a6\nB/ifzrmzAM658yNr7gAec86NOeeyVK/r39lCLVFrpW84554AFrgfc1ktuX/OuVecc6/Wlk8Cp4Ar\nL1rlzWmlf6XaMQCraXyr8HJr6f1pZr8DvJ3q912naalvdObrNd2S+2dm7wYC59zjtfWTzrn5J8Gg\n/YOh3gVcb2Z7ar/i3zHHsW80OHY5tdK3lSCS/pnZLVTPel9tX6lL0lL/zGyjme0HXgfud84Nt73i\nxVly/8zMgK8AW+jMQGz1vbnKzJ6pre/EEfit9O9dwJiZ/aOZPWdm99dez3m1ezBUnOqvHv8S2AQ8\naWY3Nnnsclpy387/9O1wLffPzK4C/gH4VDsLXaKW+uecOwH8lpltAHaZ2SPOuU76Q/CtfO99CviR\nc+6NWj50WtC3+t7c5JwbNrNfBx43swPOuWPtLXlRWnnt4sA/B26m+oPie8Bm4KH5nrCVM/kTtQLO\n20j1+uzMfXY55yrOuSGq99W/s8ljl1MrfVsJWuqfmV0OPAp8yTn3bPvLXbRIXr/aGfxB4F+0r9Ql\naaV/HwD+1Mxeo3pG/ykz+2/tL7lpLb1253/rqgX7IPDeNte7WK3m5t7apZ4KsBP47QWfsYUPEAIu\nfICQpPoBwrtn7HMH8PD5Dw+o/vq7jvoPXs8v91yMDz7a3bdp2/uBHy53X9rw2iWA/wf82XL3o039\nu5rah1m1x0eAG5e7T1G/P2vrP03nffDaymvXAySnrT/CjA81l/urxf7FgL3AFbVt3wD+04LP2WLB\nd9b+IV8GvlBbtx34N9P2+R9Uz4b2A380bf3m2nFHgT9Z7n/8iPv2c2AEmADSwO8td3+i6h/wCaAA\nPF97wz0P3LTc/Ymwfx+qPd5b+wb8j8vdl6jfn9O2d1zIt/jafQA4UHvt9gObl7svUb92VP/CwP7a\n1zeA+ELPp8FQIiIe0/R/IiIeU8iLiHhMIS8i4jGFvIiIxxTyIiIeU8iLiHhMIS8i4jGFvIiIx/4/\nSBKb5rU5IOsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1112a2320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from glob import glob\n",
    "from os import path\n",
    "import operator\n",
    "\n",
    "scores, medians = {}, {}\n",
    "for f in glob('./*/*_100_realizations.npy'):\n",
    "    team = path.basename(f).split('_')[0]\n",
    "    y_preds = np.load(f)\n",
    "    scores[team] = get_accuracies(y_preds)\n",
    "    medians[team] = np.median(scores[team])\n",
    "    plt.hist(pd.Series(scores[team]), alpha=0.5)\n",
    "    \n",
    "for t, m in sorted(medians.items(), key=operator.itemgetter(1), reverse=True):\n",
    "    print(\"{:20s}{:.4f}\".format(t, m))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look more closely at LA Team"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  3.,   4.,   8.,  13.,  20.,  17.,  16.,  13.,   3.,   3.]),\n",
       " array([ 0.62   ,  0.62375,  0.6275 ,  0.63125,  0.635  ,  0.63875,\n",
       "         0.6425 ,  0.64625,  0.65   ,  0.65375,  0.6575 ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAEACAYAAABWLgY0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADTdJREFUeJzt3WuMXHUZx/Hfr2wBKxYbhF0CtDUql5AAYkLiNeMllhgN\nxAAJMQTUEGIk+AYD8qaLMRqIEiXqGwVEkRDUSKEmtgpMYjXKreVSShWkxSo7IKm0hnDrPr6YU1zW\n1hnmnDln5tnvJ5l09szleebf2d+c/Z/LOCIEAMhpUdMNAACGh5AHgMQIeQBIjJAHgMQIeQBIjJAH\ngMR6hrzto23fZftR2w/bvqRYvsz2ettbba+zfejw2wUAvBHutZ+87SlJUxGxyfYhku6XdIakz0p6\nLiKutn2ZpGURcfnQOwYA9K3nmnxEzETEpuL6vyVtkXS0ukF/Y3G3GyWdOawmAQCDeUNz8rZXSjpF\n0h8lTUZER+p+EEg6vOrmAADl9B3yxVTNzyV9qVij53wIADDiJvq5k+0JdQP+JxGxpljcsT0ZEZ1i\n3v6Z/TyWDwMAGEBEuOxz9Lsmf72kRyPiO3OW3S7pguL6+ZLWzH/QXhExUpfVq1c33sO4XEZxrIp3\nVY2X3u/hURynUbwwTv1fqtJzTd72+yV9RtLDtjcW7/orJF0l6Vbbn5P0lKSzK+sKAFCJniEfEb+X\ndMB+bv5Yte0AAKq0II94bbVaTbcwNhir/jBO/WGc6tfzYKjSBewYdg0sLLZV785drnSOFOiHbUWN\nG14BAGOIkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5\nAEiMkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5AEiMkAeAxAh5AEiM\nkAeAxAh5AEhsoukGgNF3kGzXVm1ycoVmZrbVVg+5OSKGW8COYdfAwtIN3DrfU/XX43cGthURpdcu\nmK4BgMQIeQBIjJAHgMQIeQBIjJAHgMQIeQBIjJAHgMQIeQBIjJAHgMQIeWDkdE+jUMdlampl0y8W\nQ8ZpDTB2FsJpDeqrxykURhWnNQAA9ETIA0BihDwAJEbIA0BihDwAJNYz5G1fZ7tj+6E5y1bb3mH7\ngeJy+nDbBAAMop81+RskrdrH8msi4tTi8uuK+wIAVKBnyEfEBkk793FTfV96CQAYSJk5+S/a3mT7\nh7YPrawjAEBlJgZ83PclfTUiwvbXJF0j6fP7u/P09PRr11utllqt1oBlASCndrutdrtd+fP2dVoD\n2ysk3RERJ72R24rbOa0BKsVpDaqtxe/naKr7tAbWnDl421Nzbvu0pEfKNgIAqF7P6RrbN0tqSTrM\n9lOSVkv6sO1TJM1K2ibpoiH2CAAYEGehxNhhuqbaWvx+jibOQgkA6ImQB4DECHkASIyQB4DECHkA\nSIyQB4DECHkASIyQB4DECHkASIyQB4DECHkASIyQB4DECHkASIyQB4DECHkASIyQB4DECHkASIyQ\nB4DEen7HK9DL1NRKdTrbm24DAzmo+DrFekxOrtDMzLba6oHveEUF+M7Vca5X/2sjD/rDd7wCAHoi\n5AEgMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEg\nMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEgMUIeABIj5AEgMUIe\nABLrGfK2r7Pdsf3QnGXLbK+3vdX2OtuHDrdNAMAg+lmTv0HSqnnLLpf024g4TtJdkr5SdWMAgPJ6\nhnxEbJC0c97iMyTdWFy/UdKZFfcFAKjAoHPyR0RER5IiYkbS4dW1BACoykQdRaanp1+73mq11Gq1\n6igLAGOj3W6r3W5X/ryOiN53sldIuiMiTip+3iKpFREd21OS7o6IE/bz2OinBsaXbUl1/h9Tbzxr\ndeuRB/2xrYhw2efpd7rGxWWv2yVdUFw/X9Kaso0AAKrXc03e9s2SWpIOk9SRtFrSbZJ+JukYSU9J\nOjsi/rWfx7Mmnxxr8uNcjzX5UVXVmnxf0zWlChDy6RHy41yPkB9VdU/XAADGECEPAIkR8gCQGCEP\nAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR\n8gCQGCEPAIlNNN0Aqjc1tVKdzvam2wAwAviO14T4zlXqjWatbj3yoD98xysAoCdCHgASI+QBIDFC\nHgASI+QBIDFCHgASI+QBIDFCHgASI+QBIDFCHgASI+QBIDFCHgASI+QBIDFCHgASI+QBIDFCHgAS\nI+QBIDFCHgASI+QBIDFCHgASI+QBIDFCHgASI+QBIDFCHgASI+QBIDFCHgASI+QBILGJMg+2vU3S\n85JmJb0SEadV0RQAoBqlQl7dcG9FxM4qmgEAVKvsdI0reA4AwJCUDeiQtM72vbYvrKIhAEB1yk7X\nvC8iZmwfLuk3trdExIb5d5qenn7teqvVUqvVKlkWAHJpt9tqt9uVP68joponsldL2h0R18xbHlXV\nQH9sq/tHVm0VqTe29ep/beRBf2wrIlz2eQaerrG9xPYhxfU3S/q4pEfKNgQAqE6Z6ZpJSb+0HcXz\n/DQi1lfTFgCgCpVN1+y3ANM1tWO6hnqjWatbjzzoT+PTNQCA0UfIA0BihDwAJEbIA0BihDwAJEbI\nA0BiZU9rgD5MTa1Up7O96TYALEDsJ18D9lun3ujWYz/5UcV+8gCAngh5AEiMkAeAxAh5AEiMkAeA\nxAh5AEiMkAeAxAh5AEhswR3xGhHavHmzZmdnm24FAIZuwYX82rVrddZZ5+ngg5fXUu+ll56tpQ4w\nHg4qjgCvx6JFSzQ7+0Jt9SYnV2hmZltt9fqx4EJ+9+7dOvDAT2jXrptrqvgtSZfWVAsYdS+pztMo\nzM7We9qGTqe+D7B+MScPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQ\nGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEPAIkR8gCQGCEP\nAIkR8gCQGCEPAIkR8gCQGCEPAImVCnnbp9t+zPafbV9WVVMAgGoMHPK2F0n6rqRVkk6UdK7t46tq\nbJhefbXTdAtjpN10A2Oi3XQDwD6VWZM/TdJfImJ7RLwi6RZJZ1TT1nDt2fNM0y2MkXbTDYyJdtMN\nAPtUJuSPkvS3OT/vKJYBAEbERInHeh/LosTz1WLx4sWand2mpUs/VUu9l19+Qi++WEspAPgfZUJ+\nh6Tlc34+WtI/9nVHe1+fB83atWttzRXrHoMq611Zc71+jGK9fsapynrjWCt/vVHLO0cMtvJt+wBJ\nWyV9VNLTku6RdG5EbKmuPQBAGQOvyUfEHtsXS1qv7tz+dQQ8AIyWgdfkAQCjL90Rr/0coGX7HNub\nbT9s+6Zi2cm2/1As22T7nHo7r1eJcVpu+z7bDxTLL6q38/oNOlZzbnuL7R22r62n42aUGSfbe4r3\n1Ebbt9XXdf1KjtMxttfZftT2I7aX7+vxrxMRaS7qfmg9LmmFpMWSNkk6ft593inpfklLi5/fNmf5\nO4rrR6q7EXlp069pBMdpQtLi4voSSU9Kmmr6NY3iWM25/duSbpJ0bdOvZ1THSdKupl/DmIzT3ZI+\nUlxfIungXjWzrcn3c4DWhZK+FxG7JCki/ln8+3hEPFFcf1rSM5IOr63zepUZp1eLx0jSm1T/rhJ1\nG3isJMn2eyQdoe62q8xKjZPyv4/2GnicbJ8g6YCIuKtY/kJE9NxBO1vI93OA1rGSjrO9oZieWTX/\nSWyfpu7a6hPDa7VRpcbJ9tG2H5S0XdJVETEz9I6bM/BYubsv3TclfVn5Q6zs795Btu8plo/FkfMD\nKjNOx0p63vYvbN9v+yr3sb9mmf3kR1E/B2hNqPvn0IfU3c//d7ZP3PupaftIST+WdN4wG21YqXGK\niB2STrY9JWmN7Z9HxLPDbbkxA4+Vuu+hX0XE34vfxcxBX/Z3b3lEzNh+u6S7bD8UEU8Ot+VGlHk/\nTUj6gKRT1P2guFXSBZJu+H8Fs63J93OA1g5JayJiNiK2qbuv/7skyfZSSWslXRER9w6/3caUGqe9\nijX4zZI+OLxWG1dmrN4r6WLbf1V3jf48218ffsuNKPWe2vvXYBHsbUnvHnK/TSkzTjskbSymemYl\n3Sbp1J4Vm94QUfFGjQP0340aB6q7UeOEefdZJelHezdoqDvlsEzdjSB3Srqk6dcx4uN0lIqNPcXP\nWyWd2PRrGsWxmnef85V7w2uZ99RbJR04Z/lWzdsYmeVScpwWSdoo6bDituslfaFXzVRr8hGxR9Le\nA7Q2S7olIrbYvtL2J4v7rJP0nO3N6ob6pRGxU9I56v4pdEGxG9cDtk9q5pUMV8lxOkHSn2xvVHdL\n/9URsbmRF1KDkmO1YFTwnrqveE/dKekbEfFYIy9kyMqMU3TX3i9VdzrrweIpf9CrJgdDAUBiqdbk\nAQCvR8gDQGKEPAAkRsgDQGKEPAAkRsgDQGKEPAAkRsgDQGL/AerWJJvngKLwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x112cfcda0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "s = pd.Series(scores['LA-Team'])\n",
    "plt.hist(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    100.000000\n",
       "mean       0.638575\n",
       "std        0.007777\n",
       "min        0.620000\n",
       "25%        0.633750\n",
       "50%        0.638750\n",
       "75%        0.645000\n",
       "max        0.657500\n",
       "dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:python3]",
   "language": "python",
   "name": "conda-env-python3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}