45_OVRplots2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfollow up for 44a/b\\n\\nShaina Lu\\nGillis & Zador Labs\\nMay 2021\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "follow up for 44a/b\n",
    "use saved trained model\n",
    "\n",
    "Shaina Lu\n",
    "Gillis & Zador Labs\n",
    "May 2021\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "from scipy import stats\n",
    "from sklearn import metrics\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split #stratify train/test split\n",
    "import random\n",
    "from joblib import dump, load\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function for matplotlib formatting\n",
    "def set_style():\n",
    "    plt.style.use(['seaborn-white','seaborn-notebook'])\n",
    "    plt.rcParams['figure.figsize'] = [6,4]\n",
    "    plt.rcParams['axes.spines.top'] = False       #remove top line\n",
    "    plt.rcParams['axes.spines.right'] = False     #remove right line\n",
    "    plt.rcParams['axes.linewidth'] = 2.0          #set weight of axes\n",
    "    plt.rcParams['axes.titlesize'] = 20           #set font size of title\n",
    "    plt.rcParams['axes.labelsize'] = 18           #set font size of x,y labels\n",
    "    plt.rcParams['axes.labelpad'] = 14            #space between labels and axes\n",
    "    plt.rcParams['xtick.labelsize'] = 14          #set x label size\n",
    "    plt.rcParams['ytick.labelsize'] = 14          #set y label size\n",
    "    plt.rcParams['legend.fontsize'] = 16          #set legend font size\n",
    "    \n",
    "set_style()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#file paths\n",
    "ALLEN_FILT_PATH = \"/home/slu/spatial/data/ABAISH_filt_v6_avgdup.h5\"\n",
    "ONTOLOGY_PATH = \"/data/slu/allen_adult_mouse_ISH/ontologyABA.csv\"\n",
    "ST_CANTIN_FILT_PATH = \"/home/slu/spatial/data/cantin_ST_filt_v2.h5\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pre-processing functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def read_ABAdata():\n",
    "    \"\"\"read in all ABA datasets needed using pandas\"\"\"\n",
    "    metabrain = pd.read_hdf(ALLEN_FILT_PATH, key='metabrain', mode='r')\n",
    "    voxbrain = pd.read_hdf(ALLEN_FILT_PATH, key='avgvoxbrain', mode='r')\n",
    "    propontvox = pd.read_hdf(ALLEN_FILT_PATH, key='propontology', mode='r')\n",
    "    #geneIDName = pd.read_hdf(ALLEN_FILT_PATH, key='geneIDName', mode='r')\t\n",
    "\n",
    "    return metabrain, voxbrain, propontvox\n",
    "\n",
    "#ST\n",
    "def read_STdata():\n",
    "    \"\"\"read in all ST datasets needed using pandas\"\"\"\n",
    "    STspotsmeta = pd.read_hdf(ST_CANTIN_FILT_PATH, key='STspotsmeta', mode='r')\n",
    "    STspots = pd.read_hdf(ST_CANTIN_FILT_PATH, key='STspots', mode='r')\n",
    "    STpropont = pd.read_hdf(ST_CANTIN_FILT_PATH, key='STpropont', mode='r')\n",
    "    \n",
    "    return STspotsmeta, STspots, STpropont\n",
    "\n",
    "def read_ontology():\n",
    "    ontology = pd.read_csv(ONTOLOGY_PATH)\n",
    "    ontology = ontology.drop([ontology.columns[5], ontology.columns[6]], axis=1)\n",
    "    ontology = ontology.fillna(-1)  #make root's parent -1\n",
    "\n",
    "    return ontology\n",
    "\n",
    "def filterproponto(sampleonto):\n",
    "    \"\"\"pre-processing for propogated ontology\"\"\"\n",
    "    #remove brain areas that don't have any samples\n",
    "    sampleonto_sums = sampleonto.apply(lambda col: col.sum(), axis=0)\n",
    "    sampleonto = sampleonto.loc[:,sampleonto_sums > 5] #greater than 5 becuase less is not enough for train/test split to have non-zero areas\n",
    "    \n",
    "    return sampleonto\n",
    "\n",
    "def getleaves(propontvox, ontology):\n",
    "    \"\"\"helper function to get only leaf brain areas\"\"\"\n",
    "    #leaves are brain areas in the ontology that never show up in the parent column\n",
    "    allareas = list(propontvox)\n",
    "    parents = list(ontology.parent)\n",
    "    for i in range(len(parents)): #convert parents from float to int, ids are ints\n",
    "        parents[i] = int(parents[i])\n",
    "    \n",
    "    #remove parents from all areas\n",
    "    leaves = []\n",
    "    for area in allareas:\n",
    "        if int(area) not in parents:\n",
    "            leaves.append(area)\n",
    "    \n",
    "    print(\"number of leaf areas: %d\" %len(leaves))\n",
    "    return leaves\n",
    "\n",
    "def findoverlapareas(STonto, propontvox, ontology):\n",
    "    \"\"\"find leaf brain areas overlapping between the two datasets\"\"\"\n",
    "    leafST = getleaves(STonto, ontology)\n",
    "    leafABA = getleaves(propontvox, ontology)\n",
    "\n",
    "    leafboth = [] \n",
    "    for i in range(len(leafABA)):\n",
    "        if leafABA[i] in leafST:\n",
    "            leafboth.append(leafABA[i])\n",
    "    \n",
    "    STonto = STonto.loc[:,leafboth]\n",
    "    propontvox = propontvox.loc[:,leafboth]\n",
    "    \n",
    "    return STonto, propontvox    \n",
    "\n",
    "def zscore(voxbrain):\n",
    "    \"\"\"zscore voxbrain or subsets of voxbrain (rows: voxels, cols: genes)\"\"\"\n",
    "    #z-score on whole data set before splitting into test and train\n",
    "    scaler = StandardScaler(copy=True, with_mean=True, with_std=True)\n",
    "    scaler.fit(voxbrain)\n",
    "    z_voxbrain = scaler.transform(voxbrain)\n",
    "    \n",
    "    #store z-scored voxbrain as pandas dataframe\n",
    "    z_voxbrain = pd.DataFrame(z_voxbrain)\n",
    "    z_voxbrain.columns = voxbrain.columns\n",
    "    z_voxbrain.index = voxbrain.index\n",
    "    \n",
    "    return z_voxbrain\n",
    "\n",
    "def analytical_auroc(featurevector, binarylabels):\n",
    "    \"\"\"analytical calculation of auroc\n",
    "       inputs: feature (mean rank of expression level), binary label (ctxnotctx)\n",
    "       returns: auroc\n",
    "    \"\"\"\n",
    "    #sort ctxnotctx binary labels by mean rank, aescending\n",
    "    s = sorted(zip(featurevector, binarylabels))\n",
    "    feature_sort, binarylabels_sort = map(list, zip(*s))\n",
    "\n",
    "    #get the sum of the ranks in feature vector corresponding to 1's in binary vector\n",
    "    sumranks = 0\n",
    "    for i in range(len(binarylabels_sort)):\n",
    "        if binarylabels_sort[i] == 1:\n",
    "            sumranks = sumranks + feature_sort[i]\n",
    "    \n",
    "    poslabels = binarylabels.sum()\n",
    "    neglabels = (len(binarylabels) - poslabels)\n",
    "    \n",
    "    auroc = ((sumranks/(neglabels*poslabels)) - ((poslabels+1)/(2*neglabels)))\n",
    "    \n",
    "    return auroc\n",
    "\n",
    "def getoverlapgenes(STspots, ABAvox):\n",
    "    ABAgenes = list(ABAvox)\n",
    "    STgenes = list(STspots)\n",
    "    \n",
    "    #get overlapping genes\n",
    "    overlap = []\n",
    "    for i in range(len(ABAgenes)):\n",
    "        if ABAgenes[i] in STgenes:\n",
    "            overlap.append(ABAgenes[i])\n",
    "    \n",
    "    print(\"number of overlapping genes: %d\" %len(overlap))\n",
    "    \n",
    "    #index datasets to keep only genes that are overlapping\n",
    "    STspots = STspots.loc[:,overlap]\n",
    "    ABAvox = ABAvox.loc[:,overlap]\n",
    "    \n",
    "    return STspots, ABAvox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def makeyvector(mod_propont):\n",
    "    \"\"\"collapse propont into 1D y label vector\"\"\"\n",
    "    newpropont = pd.DataFrame(columns=mod_propont.columns)\n",
    "    for i in range(mod_propont.shape[1]):\n",
    "        currcol = mod_propont.iloc[:,i]\n",
    "        currcol.loc[currcol>0] = int(mod_propont.columns[i])\n",
    "        newpropont[mod_propont.columns[i]] = currcol\n",
    "        \n",
    "    collapsedy = newpropont.sum(axis=1)\n",
    "\n",
    "    return newpropont, collapsedy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of leaf areas: 461\n"
     ]
    }
   ],
   "source": [
    "#read in data\n",
    "ontology = read_ontology()\n",
    "STspotsmeta, STspots, STpropont = read_STdata()\n",
    "#pre-processing\n",
    "STpropont = filterproponto(STpropont)\n",
    "STspots = STspots.astype('float64') #convert int to float for z-scoring\n",
    "#get leaf areas\n",
    "leaves = getleaves(STpropont, ontology)\n",
    "STpropont = STpropont.loc[STspotsmeta.id.isin(leaves),leaves] #subset prop onto for leaf areas\n",
    "STspots = STspots.loc[STspotsmeta.id.isin(leaves),:] #subset data for samples from leaves\n",
    "\n",
    "#collapse propont to 1D y vector\n",
    "newpropont, collapsedy = makeyvector(STpropont)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__predict using saved model__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model \n",
    "model = load(\"OVR2STmod.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OneVsRestClassifier(estimator=LinearRegression(copy_X=True, fit_intercept=True,\n",
       "                                               n_jobs=None, normalize=False),\n",
       "                    n_jobs=None)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OneVsRestClassifier(estimator=LinearRegression(copy_X=True, fit_intercept=True,\n",
      "                                               n_jobs=None, normalize=False),\n",
      "                    n_jobs=None)\n"
     ]
    }
   ],
   "source": [
    "mod_data = STspots\n",
    "#split train test for X data and y labels\n",
    "#split data function is seeded so all will split the same way\n",
    "Xtrain, Xtest, ytrain, ytest = train_test_split(mod_data, collapsedy, test_size=0.5,\\\n",
    "                                                random_state=42, shuffle=True,\\\n",
    "                                                stratify=collapsedy)\n",
    "#z-score train and test folds\n",
    "zXtrain = zscore(Xtrain)\n",
    "zXtest = zscore(Xtest)\n",
    "\n",
    "#predict\n",
    "print(model)\n",
    "trainpredictions = model.predict(zXtrain)\n",
    "testpredictions = model.predict(zXtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  945.,   962.,   962., ..., 10703.,   276.,   844.])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainpredictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 430.  601. 1047.]\n"
     ]
    }
   ],
   "source": [
    "#print(np.unique(trainpredictions))\n",
    "print(np.unique(testpredictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predvectortomatrix(predvector, mod_propont):\n",
    "    \"\"\"break predictions vector out into matrix where ncols = nareas\"\"\"\n",
    "    predmatrix = pd.DataFrame(0, index=(range(len(predvector))), columns = mod_propont.columns)\n",
    "    for i in range(len(predmatrix.columns)):\n",
    "        area = predmatrix.columns[i]\n",
    "        for j in range(len(predvector)):\n",
    "            if int(predvector[j]) == int(area):\n",
    "                predmatrix.iloc[j,i] = 1\n",
    "                \n",
    "    return predmatrix\n",
    "\n",
    "def matrixtoauroc(predmatrix, ytrue):\n",
    "    \"\"\"calculate AUROC for each brain area\"\"\"\n",
    "    aurocs = []\n",
    "    \n",
    "    for col in predmatrix.columns:\n",
    "        #aurocs.append(analytical_auroc(sp.stats.mstats.rankdata(predmatrix.loc[:,col]),ytrue.loc[:,col]))\n",
    "        aurocs.append(roc_auc_score(ytest2.loc[:,col], STtestpredmatrix.loc[:,col]))\n",
    "        \n",
    "    return aurocs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "STtrainpredmatrix = predvectortomatrix(trainpredictions, STpropont)\n",
    "STtestpredmatrix = predvectortomatrix(testpredictions, STpropont)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14142"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "STtestpredmatrix.sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "mod_propont = STpropont\n",
    "#to get propont split on the same indicies\n",
    "null1, null2, ytrain2, ytest2 = train_test_split(mod_data, mod_propont, test_size=0.5,\\\n",
    "                                                random_state=42, shuffle=True,\\\n",
    "                                                stratify=collapsedy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "STtestaurocs = matrixtoauroc(STtestpredmatrix, ytest2)\n",
    "STtrainaurocs = matrixtoauroc(STtrainpredmatrix, ytrain2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXWklEQVR4nO3df0xV9/3H8dfVTsnVanqBuG4aERtutAperpo6QAe10Yhdq+s2U6stZjpNuxm10HYWqKidGtOq+AOFqWjUaly1TtetNtrNBatlXppFHENta62rAkGskALq5/vH4v165w+g3HtZ93k+Ev/wnM+95/NO43OX4/XMYYwxAgBYo1NHbwAAEF6EHwAsQ/gBwDKEHwAs818ffrfbLbfb3dHbAID/Gf/14QcABBfhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsMx9Hb0BIBhiXj7QIdf9dEl6h1wXaA8+8QOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZdoc/oKCArndbuXl5fmPGWOUn5+v5ORkxcfHa8qUKaqsrAx4XV1dnTIzM+X1euX1epWZmakrV660fwIAQJu0KfxlZWXatWuX3G53wPHCwkJt3LhR2dnZ2r17t1wulzIyMnT16lX/mnnz5qm8vFyFhYUqKipSeXm5srKygjMFAKDVWh3+r776Si+++KIWL16snj17+o8bY7RlyxbNmDFDY8aMUVxcnJYuXar6+nrt379fknTmzBkdOXJEeXl5SkxMlMfj0YIFC3T48GGdPXs2+FMBAO6q1eHPzs7WmDFjNGLEiIDj58+fV1VVlZKSkvzHIiIiNGzYMPl8PkmSz+eT0+lUYmKif43X65XT6fSvAQCER6vCv2vXLp07d06zZ8++7VxVVZUkKSoqKuB4ZGSkqqurJUnV1dVyuVxyOBz+8w6HQy6Xy78GABAe97W04OzZs3rjjTe0bds2denS5a7rbo16a88bY1p8HQAguFoMf1lZmWpra/X444/7j12/fl0fffSR3nrrLf99/KqqKj344IP+NTU1Nf6fAqKiolRTUxMQemOMamtrFRkZGdSBAAD31mL4R48erUGDBgUce+WVVxQTE6Nf/OIX6tevn6Kjo1VSUqL4+HhJUmNjo0pLS/3f2vF4PGpoaJDP5/Pf5/f5fGpoaJDH4wn2TACAe2gx/D169FCPHj0CjjmdTvXs2VNxcXGSpKlTp6qgoECxsbGKiYnRunXr5HQ6NX78eElS//79lZKSotzcXC1cuFDGGOXm5io1NVWxsbEhGAsAcDcthr81pk+frsbGRuXl5amurk4JCQnauHGjunfv7l+zfPlyLVq0SNOmTZMkpaWlKScnJxiXBwC0gcMYYzp6E/dy8x+LVVRUdPBO8N8s5uUDHXLdT5ekd8h1gfbgWT0AYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWaTH827Zt0+OPP67ExEQlJibqZz/7mT744AP/eWOM8vPzlZycrPj4eE2ZMkWVlZUB71FXV6fMzEx5vV55vV5lZmbqypUrQR8GANCyFsPfq1cvvfjii9qzZ49+97vf6ZFHHtHzzz+vf/zjH5KkwsJCbdy4UdnZ2dq9e7dcLpcyMjJ09epV/3vMmzdP5eXlKiwsVFFRkcrLy5WVlRW6qQAAd9Vi+EePHq1Ro0apb9++6tevn+bMmaNu3bqprKxMxhht2bJFM2bM0JgxYxQXF6elS5eqvr5e+/fvlySdOXNGR44cUV5enhITE+XxeLRgwQIdPnxYZ8+eDfmAAIBAbbrHf/36dR04cEANDQ3yeDw6f/68qqqqlJSU5F8TERGhYcOGyefzSZJ8Pp+cTqcSExP9a7xer5xOp38NACB87mvNooqKCk2aNEmNjY1yOp1avXq13G63Tpw4IUmKiooKWB8ZGalLly5Jkqqrq+VyueRwOPznHQ6HXC6XqqurgzUHAKCVWhX+fv36ae/evbpy5Yree+89vfTSS9q6dav//K1Rv5M7nTfGtPg6AEDwtSr8Xbp0Ud++fSVJgwcP1t///ndt3rxZs2bNkiRVVVXpwQcf9K+vqanx/xQQFRWlmpqagNAbY1RbW6vIyMigDgMAaNk3+h7/jRs31NTUpN69eys6OlolJSX+c42NjSotLZXH45EkeTweNTQ0BNzP9/l8/r8nAACEV4uf+JcvX64f/vCH+u53v+v/ts7x48e1fv16ORwOTZ06VQUFBYqNjVVMTIzWrVsnp9Op8ePHS5L69++vlJQU5ebmauHChTLGKDc3V6mpqYqNjQ35gACAQC2Gv7q6WpmZmaqqqtL9998vt9utwsJCpaSkSJKmT5+uxsZG5eXlqa6uTgkJCdq4caO6d+/uf4/ly5dr0aJFmjZtmiQpLS1NOTk5IRoJAHAvDmOM6ehN3Ivb7Zb0728WAXcT8/KBDrnup0vSO+S6QHvwrB4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsEyL4V+/fr1+/OMfKzExUY888ohmzpypf/7znwFrjDHKz89XcnKy4uPjNWXKFFVWVgasqaurU2Zmprxer7xerzIzM3XlypXgTgMAaFGL4T9+/LiefvppvfXWWyouLlbnzp2VkZGhy5cv+9cUFhZq48aNys7O1u7du+VyuZSRkaGrV6/618ybN0/l5eUqLCxUUVGRysvLlZWVFZqpAAB35TDGmLa8oL6+XkOHDtWaNWuUlpYmY4xSUlI0efJkzZo1S5L09ddfa8SIEXrppZc0adIknTlzRuPGjdP27dvl9XolSaWlpZo8ebLeffddxcbG3vV6brdbklRRUfFNZ4QFYl4+0CHX/XRJeodcF2iPNt/jr6+v140bN9SjRw9J0vnz51VVVaWkpCT/moiICA0bNkw+n0+S5PP55HQ6lZiY6F/j9XrldDr9awAA4dHm8C9evFgDBgyQx+ORJFVVVUmSoqKiAtZFRkaqurpaklRdXS2XyyWHw+E/73A45HK5/GsAAOFxX1sW/+Y3v9Hf/vY37dixQ507dw44d2vU7+RO540xLb4OABBcrf7E//rrr+vAgQMqLi5Wnz59/Mejo6Ml/f8n/5tqamr8PwVERUWppqZGt/51gjFGtbW1ioyMbNcAAIC2aVX4Fy1apP3796u4uFj9+/cPONe7d29FR0erpKTEf6yxsVGlpaX+20Eej0cNDQ0B9/N9Pp8aGhr8awAA4dHirZ4FCxbonXfe0Zo1a9SjRw//J3un06lu3brJ4XBo6tSpKigoUGxsrGJiYrRu3To5nU6NHz9ektS/f3+lpKQoNzdXCxculDFGubm5Sk1Nvec3egAAwdfi1zlvfp3yP73wwgv65S9/Kenft21Wr16tnTt3qq6uTgkJCcrJyVFcXJx//eXLl7Vo0SIdOnRIkpSWlqacnBz/t4Nauj5f58S98HVOoPXa/D3+cCP8aA3CD7Qez+oBAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwTKvC/9FHH2nmzJlKSUmR2+3W22+/HXDeGKP8/HwlJycrPj5eU6ZMUWVlZcCauro6ZWZmyuv1yuv1KjMzU1euXAneJACAVmlV+BsaGhQXF6f58+crIiLitvOFhYXauHGjsrOztXv3brlcLmVkZOjq1av+NfPmzVN5ebkKCwtVVFSk8vJyZWVlBW8SAECrtCr8o0aN0ty5czV27Fh16hT4EmOMtmzZohkzZmjMmDGKi4vT0qVLVV9fr/3790uSzpw5oyNHjigvL0+JiYnyeDxasGCBDh8+rLNnzwZ/KgDAXbX7Hv/58+dVVVWlpKQk/7GIiAgNGzZMPp9PkuTz+eR0OpWYmOhf4/V65XQ6/WsAAOHR7vBXVVVJkqKiogKOR0ZGqrq6WpJUXV0tl8slh8PhP+9wOORyufxrAADhEbRv9dwa9daeN8a0+DoAQHC1O/zR0dGS/v+T/001NTX+nwKioqJUU1MjY4z/vDFGtbW1ioyMbO8WAABt0O7w9+7dW9HR0SopKfEfa2xsVGlpqTwejyTJ4/GooaEh4H6+z+dTQ0ODfw0AIDzua82i+vp6nTt3TpJ048YNXbhwQadOnVLPnj31ve99T1OnTlVBQYFiY2MVExOjdevWyel0avz48ZKk/v37KyUlRbm5uVq4cKGMMcrNzVVqaqpiY2NDNx0A4DYOc+v9l7s4duyYpk6detvxCRMmaMmSJTLGaPXq1dq5c6fq6uqUkJCgnJwcxcXF+ddevnxZixYt0qFDhyRJaWlpysnJUY8ePe55bbfbLUmqqKho02CwS8zLBzrkup8uSe+Q6wLt0arwdyTCj9Yg/EDr8aweALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAy4Q9/Nu2bVNaWpoGDx6siRMnqrS0NNxbAACrhTX8f/jDH/T6669r5syZ2rt3rzwej6ZPn64LFy6EcxsAYLWwhn/Tpk2aMGGCfvrTn6p///7Kzs5WdHS0duzYEc5tAIDV7gvXhZqamnTy5ElNmzYt4HhSUpJ8Pl+Lr3e73aHaGv4HdO2g67r3zO2gKwMtq6iouOPxsH3ir62t1fXr1xUVFRVwPDIyUlVVVeHaBgBYL2yf+G9yOBytOnbT3f4XCwDwzYTtE/8DDzygzp073/bpvqam5rafAgAAoRO28Hfp0kUPP/ywSkpKAo6XlJTI4/GEaxsAYL2w3urJyMhQVlaW4uPjlZiYqB07dujSpUuaNGlSOLcBAFYLa/jHjRun2tparVu3TpcuXVJcXJw2bNig73//++HcBgBYzWGMMR29CQBA+PCsHgCwDOEHAMsQ/lb4pg+WKy0t1cCBAzV+/PjbzhUXF2vs2LGKj4/XyJEjtWDBAtXX1wd763cU7Hmam5u1evVqjR49WoMHD9aPfvQj/eUvfwnF1m/TllmOHTsmt9t9268zZ84ErPvTn/6kcePGadCgQRo3bpwOHjwY6jH8gj1PZWWlfvWrX+nRRx+V2+1Wfn5+OMaQFPxZdu3apaefflrDhw/X0KFDNWXKlLA95DHYs7z77ruaOHGihg4dqiFDhuiJJ57Qnj17wjHKvxnc04EDB8zAgQPNzp07zenTp01eXp4ZMmSI+eKLL+75usuXL5u0tDQzbdo0k56eHnBu37595uGHHzZ79uwxn3/+uSkpKTGpqanmlVdeCeUoxpjQzLNs2TKTlJRkDh8+bM6dO2e2bdtmBg8ebE6ePBnKUdo8y4cffmji4uJMZWWluXTpkv/XtWvX/GtOnDhhBgwYYNauXWtOnz5t1q5dawYMGGDKyspCOkuo5vn444/NkiVLzL59+0xaWppZtWpVyOcI1Sxz5841W7duNSdPnjRnzpwx2dnZJiEhwXzyySffullKSkrMwYMHzenTp81nn31mNm/ebAYMGGA++OCDkM5yE+FvwVNPPWXmz58fcOyxxx4zy5cvv+frnn/+eZOfn29WrVp1WygXLFhgJk+eHHBs5cqVt60LhVDMk5SUZDZv3hxw7IUXXjDz5s0Lzqbvoq2z3PwDWVNTc9f3nD17tnnuuecCjj377LNmzpw57d9wC0Ixz63S09PDFv5Qz2KMMTdu3DA/+MEPzJYtW9q115aEYxZjjHnyySdb/HMYLNzquYebD5ZLSkoKON7Sg+W2bdum6upqzZo1647nvV6vTp06pbKyMknShQsXdOjQIY0cOTJ4m7+DUM3T3Nysrl0DH5PWtWtXnThxov2bvotvOoskPfXUU0pOTtazzz6rDz/8MOBcWVnZbe+ZnJzcqgcJtkeo5ukI4ZqlublZjY2N6tGjR7v3fDfhmMUYo6NHj+qTTz7R0KFDg7LvloT9WT3fJvd6sNx//gvkmyoqKrRmzRrt3LlTnTt3vuOa9PR0Xb58Wc8884yMMbp27ZqeeOIJZWZmBn2GW4VqnuTkZBUXF2v48OGKiYnR0aNHdfDgQV2/fj3oM9z0TWaJjo7Wa6+9psGDB6u5uVnvvPOOnnvuOW3dulXDhg2TJFVXV9/2nlFRUSF/kGCo5ukI4ZrlzTfflNPp1KOPPhr0GW4K5SxfffWVRo4cqaamJnXq1Ek5OTkaNWpUyGa5FeFvhdY+WK6pqUlz585VVlaW+vTpc9f3O378uNauXavc3FzFx8fr3LlzWrx4sVatWqXZs2cHde93Eux55s+fr1dffVXp6elyOBzq06ePJk6cqLfffjuo+76Ttjz0LzY2VrGxsf7fezweffHFF/rtb397z1AaY+75IMFgCsc84RLKWYqLi7Vz505t3rxZ3bt3D96m7yIUs3Tr1k179+5VQ0ODjh49qiVLlqh3794aMWJE8Af4D9zquYe2Plju0qVLOn36tH79619r4MCBGjhwoNasWaPKykoNHDhQf/3rXyVJK1asUHp6un7yk5/I7Xbrscce05w5c1RUVKRr16596+ZxuVxau3atysrKdPjwYf3xj3+U0+lU7969/2tmuZuEhAR99tln/t9HRUWpurq6Xe/5TYRqno4Q6lmKi4u1YsUKbdiwQfHx8e3e772EcpZOnTqpb9++GjBggKZNm6axY8eqoKAgKPtuCeG/h7Y+WK5Xr176/e9/r7179/p/TZo0SX379vX/X01K0tdff33bbZPOnTvLhPgfUYdqnpu6du2qXr166dq1a3rvvfdC+iN4sB76d+rUKUVHR/t/P2TIkA55kGCo5ukIoZxl06ZN/uiH4354OP+73LhxQ01NTd9on23FrZ4WtPRguaysLEnSsmXL9J3vfEdxcXEBr4+MjFSXLl0CjqempmrTpk0aNGiQ/1bPypUrlZqaqvvuC+1/klDM8/HHH+vixYsaMGCALl68qPz8fN24cUM///nP/2tmkaTNmzerd+/eeuihh9Tc3Kx9+/bp/fffD/hu+9SpU/XMM89o/fr1Gj16tN5//30dO3ZM27dvD+ksoZqnqanJ//3xxsZGVVVV6dSpU3I6nerbt++3apaioiKtWLFCy5YtU0xMjP9TeEREhO6///5v1Szr1q1TQkKC+vTpo6amJv35z3/Wvn379Oqrr4ZsjlsR/ha09GC5f/3rX21+z1mzZsnhcGjlypX68ssv9cADDyg1NVVz5swJ9vZvE4p5GhsbtWLFCn3++edyOp0aNWqUli1bFtJvW0htn6W5uVlLly7VxYsXFRERoYceekgbNmwI+Au1xMREvfHGG1qxYoXy8/PVp08fvfnmm0pISAjpLKGa59KlS3ryySf9vz937px27typ4cOHa+vWrd+qWbZv367m5ubb/pxMmDBBS5Ys+VbN0tDQoNdee01ffvmlIiIiFBsbq6VLl97xH3uGAg9pAwDLcI8fACxD+AHAMoQfACxD+AHAMoQfACxD+AHAMoQfACxD+AHAMv8H6vVuOeXWMb4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(STtrainaurocs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXWklEQVR4nO3df0xV9/3H8dfVTsnVanqBuG4aERtutAperpo6QAe10Yhdq+s2U6stZjpNuxm10HYWqKidGtOq+AOFqWjUaly1TtetNtrNBatlXppFHENta62rAkGskALq5/vH4v165w+g3HtZ93k+Ev/wnM+95/NO43OX4/XMYYwxAgBYo1NHbwAAEF6EHwAsQ/gBwDKEHwAs818ffrfbLbfb3dHbAID/Gf/14QcABBfhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsMx9Hb0BIBhiXj7QIdf9dEl6h1wXaA8+8QOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZdoc/oKCArndbuXl5fmPGWOUn5+v5ORkxcfHa8qUKaqsrAx4XV1dnTIzM+X1euX1epWZmakrV660fwIAQJu0KfxlZWXatWuX3G53wPHCwkJt3LhR2dnZ2r17t1wulzIyMnT16lX/mnnz5qm8vFyFhYUqKipSeXm5srKygjMFAKDVWh3+r776Si+++KIWL16snj17+o8bY7RlyxbNmDFDY8aMUVxcnJYuXar6+nrt379fknTmzBkdOXJEeXl5SkxMlMfj0YIFC3T48GGdPXs2+FMBAO6q1eHPzs7WmDFjNGLEiIDj58+fV1VVlZKSkvzHIiIiNGzYMPl8PkmSz+eT0+lUYmKif43X65XT6fSvAQCER6vCv2vXLp07d06zZ8++7VxVVZUkKSoqKuB4ZGSkqqurJUnV1dVyuVxyOBz+8w6HQy6Xy78GABAe97W04OzZs3rjjTe0bds2denS5a7rbo16a88bY1p8HQAguFoMf1lZmWpra/X444/7j12/fl0fffSR3nrrLf99/KqqKj344IP+NTU1Nf6fAqKiolRTUxMQemOMamtrFRkZGdSBAAD31mL4R48erUGDBgUce+WVVxQTE6Nf/OIX6tevn6Kjo1VSUqL4+HhJUmNjo0pLS/3f2vF4PGpoaJDP5/Pf5/f5fGpoaJDH4wn2TACAe2gx/D169FCPHj0CjjmdTvXs2VNxcXGSpKlTp6qgoECxsbGKiYnRunXr5HQ6NX78eElS//79lZKSotzcXC1cuFDGGOXm5io1NVWxsbEhGAsAcDcthr81pk+frsbGRuXl5amurk4JCQnauHGjunfv7l+zfPlyLVq0SNOmTZMkpaWlKScnJxiXBwC0gcMYYzp6E/dy8x+LVVRUdPBO8N8s5uUDHXLdT5ekd8h1gfbgWT0AYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWaTH827Zt0+OPP67ExEQlJibqZz/7mT744AP/eWOM8vPzlZycrPj4eE2ZMkWVlZUB71FXV6fMzEx5vV55vV5lZmbqypUrQR8GANCyFsPfq1cvvfjii9qzZ49+97vf6ZFHHtHzzz+vf/zjH5KkwsJCbdy4UdnZ2dq9e7dcLpcyMjJ09epV/3vMmzdP5eXlKiwsVFFRkcrLy5WVlRW6qQAAd9Vi+EePHq1Ro0apb9++6tevn+bMmaNu3bqprKxMxhht2bJFM2bM0JgxYxQXF6elS5eqvr5e+/fvlySdOXNGR44cUV5enhITE+XxeLRgwQIdPnxYZ8+eDfmAAIBAbbrHf/36dR04cEANDQ3yeDw6f/68qqqqlJSU5F8TERGhYcOGyefzSZJ8Pp+cTqcSExP9a7xer5xOp38NACB87mvNooqKCk2aNEmNjY1yOp1avXq13G63Tpw4IUmKiooKWB8ZGalLly5Jkqqrq+VyueRwOPznHQ6HXC6XqqurgzUHAKCVWhX+fv36ae/evbpy5Yree+89vfTSS9q6dav//K1Rv5M7nTfGtPg6AEDwtSr8Xbp0Ud++fSVJgwcP1t///ndt3rxZs2bNkiRVVVXpwQcf9K+vqanx/xQQFRWlmpqagNAbY1RbW6vIyMigDgMAaNk3+h7/jRs31NTUpN69eys6OlolJSX+c42NjSotLZXH45EkeTweNTQ0BNzP9/l8/r8nAACEV4uf+JcvX64f/vCH+u53v+v/ts7x48e1fv16ORwOTZ06VQUFBYqNjVVMTIzWrVsnp9Op8ePHS5L69++vlJQU5ebmauHChTLGKDc3V6mpqYqNjQ35gACAQC2Gv7q6WpmZmaqqqtL9998vt9utwsJCpaSkSJKmT5+uxsZG5eXlqa6uTgkJCdq4caO6d+/uf4/ly5dr0aJFmjZtmiQpLS1NOTk5IRoJAHAvDmOM6ehN3Ivb7Zb0728WAXcT8/KBDrnup0vSO+S6QHvwrB4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsEyL4V+/fr1+/OMfKzExUY888ohmzpypf/7znwFrjDHKz89XcnKy4uPjNWXKFFVWVgasqaurU2Zmprxer7xerzIzM3XlypXgTgMAaFGL4T9+/LiefvppvfXWWyouLlbnzp2VkZGhy5cv+9cUFhZq48aNys7O1u7du+VyuZSRkaGrV6/618ybN0/l5eUqLCxUUVGRysvLlZWVFZqpAAB35TDGmLa8oL6+XkOHDtWaNWuUlpYmY4xSUlI0efJkzZo1S5L09ddfa8SIEXrppZc0adIknTlzRuPGjdP27dvl9XolSaWlpZo8ebLeffddxcbG3vV6brdbklRRUfFNZ4QFYl4+0CHX/XRJeodcF2iPNt/jr6+v140bN9SjRw9J0vnz51VVVaWkpCT/moiICA0bNkw+n0+S5PP55HQ6lZiY6F/j9XrldDr9awAA4dHm8C9evFgDBgyQx+ORJFVVVUmSoqKiAtZFRkaqurpaklRdXS2XyyWHw+E/73A45HK5/GsAAOFxX1sW/+Y3v9Hf/vY37dixQ507dw44d2vU7+RO540xLb4OABBcrf7E//rrr+vAgQMqLi5Wnz59/Mejo6Ml/f8n/5tqamr8PwVERUWppqZGt/51gjFGtbW1ioyMbNcAAIC2aVX4Fy1apP3796u4uFj9+/cPONe7d29FR0erpKTEf6yxsVGlpaX+20Eej0cNDQ0B9/N9Pp8aGhr8awAA4dHirZ4FCxbonXfe0Zo1a9SjRw//J3un06lu3brJ4XBo6tSpKigoUGxsrGJiYrRu3To5nU6NHz9ektS/f3+lpKQoNzdXCxculDFGubm5Sk1Nvec3egAAwdfi1zlvfp3yP73wwgv65S9/Kenft21Wr16tnTt3qq6uTgkJCcrJyVFcXJx//eXLl7Vo0SIdOnRIkpSWlqacnBz/t4Nauj5f58S98HVOoPXa/D3+cCP8aA3CD7Qez+oBAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwTKvC/9FHH2nmzJlKSUmR2+3W22+/HXDeGKP8/HwlJycrPj5eU6ZMUWVlZcCauro6ZWZmyuv1yuv1KjMzU1euXAneJACAVmlV+BsaGhQXF6f58+crIiLitvOFhYXauHGjsrOztXv3brlcLmVkZOjq1av+NfPmzVN5ebkKCwtVVFSk8vJyZWVlBW8SAECrtCr8o0aN0ty5czV27Fh16hT4EmOMtmzZohkzZmjMmDGKi4vT0qVLVV9fr/3790uSzpw5oyNHjigvL0+JiYnyeDxasGCBDh8+rLNnzwZ/KgDAXbX7Hv/58+dVVVWlpKQk/7GIiAgNGzZMPp9PkuTz+eR0OpWYmOhf4/V65XQ6/WsAAOHR7vBXVVVJkqKiogKOR0ZGqrq6WpJUXV0tl8slh8PhP+9wOORyufxrAADhEbRv9dwa9daeN8a0+DoAQHC1O/zR0dGS/v+T/001NTX+nwKioqJUU1MjY4z/vDFGtbW1ioyMbO8WAABt0O7w9+7dW9HR0SopKfEfa2xsVGlpqTwejyTJ4/GooaEh4H6+z+dTQ0ODfw0AIDzua82i+vp6nTt3TpJ048YNXbhwQadOnVLPnj31ve99T1OnTlVBQYFiY2MVExOjdevWyel0avz48ZKk/v37KyUlRbm5uVq4cKGMMcrNzVVqaqpiY2NDNx0A4DYOc+v9l7s4duyYpk6detvxCRMmaMmSJTLGaPXq1dq5c6fq6uqUkJCgnJwcxcXF+ddevnxZixYt0qFDhyRJaWlpysnJUY8ePe55bbfbLUmqqKho02CwS8zLBzrkup8uSe+Q6wLt0arwdyTCj9Yg/EDr8aweALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAy4Q9/Nu2bVNaWpoGDx6siRMnqrS0NNxbAACrhTX8f/jDH/T6669r5syZ2rt3rzwej6ZPn64LFy6EcxsAYLWwhn/Tpk2aMGGCfvrTn6p///7Kzs5WdHS0duzYEc5tAIDV7gvXhZqamnTy5ElNmzYt4HhSUpJ8Pl+Lr3e73aHaGv4HdO2g67r3zO2gKwMtq6iouOPxsH3ir62t1fXr1xUVFRVwPDIyUlVVVeHaBgBYL2yf+G9yOBytOnbT3f4XCwDwzYTtE/8DDzygzp073/bpvqam5rafAgAAoRO28Hfp0kUPP/ywSkpKAo6XlJTI4/GEaxsAYL2w3urJyMhQVlaW4uPjlZiYqB07dujSpUuaNGlSOLcBAFYLa/jHjRun2tparVu3TpcuXVJcXJw2bNig73//++HcBgBYzWGMMR29CQBA+PCsHgCwDOEHAMsQ/lb4pg+WKy0t1cCBAzV+/PjbzhUXF2vs2LGKj4/XyJEjtWDBAtXX1wd763cU7Hmam5u1evVqjR49WoMHD9aPfvQj/eUvfwnF1m/TllmOHTsmt9t9268zZ84ErPvTn/6kcePGadCgQRo3bpwOHjwY6jH8gj1PZWWlfvWrX+nRRx+V2+1Wfn5+OMaQFPxZdu3apaefflrDhw/X0KFDNWXKlLA95DHYs7z77ruaOHGihg4dqiFDhuiJJ57Qnj17wjHKvxnc04EDB8zAgQPNzp07zenTp01eXp4ZMmSI+eKLL+75usuXL5u0tDQzbdo0k56eHnBu37595uGHHzZ79uwxn3/+uSkpKTGpqanmlVdeCeUoxpjQzLNs2TKTlJRkDh8+bM6dO2e2bdtmBg8ebE6ePBnKUdo8y4cffmji4uJMZWWluXTpkv/XtWvX/GtOnDhhBgwYYNauXWtOnz5t1q5dawYMGGDKyspCOkuo5vn444/NkiVLzL59+0xaWppZtWpVyOcI1Sxz5841W7duNSdPnjRnzpwx2dnZJiEhwXzyySffullKSkrMwYMHzenTp81nn31mNm/ebAYMGGA++OCDkM5yE+FvwVNPPWXmz58fcOyxxx4zy5cvv+frnn/+eZOfn29WrVp1WygXLFhgJk+eHHBs5cqVt60LhVDMk5SUZDZv3hxw7IUXXjDz5s0Lzqbvoq2z3PwDWVNTc9f3nD17tnnuuecCjj377LNmzpw57d9wC0Ixz63S09PDFv5Qz2KMMTdu3DA/+MEPzJYtW9q115aEYxZjjHnyySdb/HMYLNzquYebD5ZLSkoKON7Sg+W2bdum6upqzZo1647nvV6vTp06pbKyMknShQsXdOjQIY0cOTJ4m7+DUM3T3Nysrl0DH5PWtWtXnThxov2bvotvOoskPfXUU0pOTtazzz6rDz/8MOBcWVnZbe+ZnJzcqgcJtkeo5ukI4ZqlublZjY2N6tGjR7v3fDfhmMUYo6NHj+qTTz7R0KFDg7LvloT9WT3fJvd6sNx//gvkmyoqKrRmzRrt3LlTnTt3vuOa9PR0Xb58Wc8884yMMbp27ZqeeOIJZWZmBn2GW4VqnuTkZBUXF2v48OGKiYnR0aNHdfDgQV2/fj3oM9z0TWaJjo7Wa6+9psGDB6u5uVnvvPOOnnvuOW3dulXDhg2TJFVXV9/2nlFRUSF/kGCo5ukI4ZrlzTfflNPp1KOPPhr0GW4K5SxfffWVRo4cqaamJnXq1Ek5OTkaNWpUyGa5FeFvhdY+WK6pqUlz585VVlaW+vTpc9f3O378uNauXavc3FzFx8fr3LlzWrx4sVatWqXZs2cHde93Eux55s+fr1dffVXp6elyOBzq06ePJk6cqLfffjuo+76Ttjz0LzY2VrGxsf7fezweffHFF/rtb397z1AaY+75IMFgCsc84RLKWYqLi7Vz505t3rxZ3bt3D96m7yIUs3Tr1k179+5VQ0ODjh49qiVLlqh3794aMWJE8Af4D9zquYe2Plju0qVLOn36tH79619r4MCBGjhwoNasWaPKykoNHDhQf/3rXyVJK1asUHp6un7yk5/I7Xbrscce05w5c1RUVKRr16596+ZxuVxau3atysrKdPjwYf3xj3+U0+lU7969/2tmuZuEhAR99tln/t9HRUWpurq6Xe/5TYRqno4Q6lmKi4u1YsUKbdiwQfHx8e3e772EcpZOnTqpb9++GjBggKZNm6axY8eqoKAgKPtuCeG/h7Y+WK5Xr176/e9/r7179/p/TZo0SX379vX/X01K0tdff33bbZPOnTvLhPgfUYdqnpu6du2qXr166dq1a3rvvfdC+iN4sB76d+rUKUVHR/t/P2TIkA55kGCo5ukIoZxl06ZN/uiH4354OP+73LhxQ01NTd9on23FrZ4WtPRguaysLEnSsmXL9J3vfEdxcXEBr4+MjFSXLl0CjqempmrTpk0aNGiQ/1bPypUrlZqaqvvuC+1/klDM8/HHH+vixYsaMGCALl68qPz8fN24cUM///nP/2tmkaTNmzerd+/eeuihh9Tc3Kx9+/bp/fffD/hu+9SpU/XMM89o/fr1Gj16tN5//30dO3ZM27dvD+ksoZqnqanJ//3xxsZGVVVV6dSpU3I6nerbt++3apaioiKtWLFCy5YtU0xMjP9TeEREhO6///5v1Szr1q1TQkKC+vTpo6amJv35z3/Wvn379Oqrr4ZsjlsR/ha09GC5f/3rX21+z1mzZsnhcGjlypX68ssv9cADDyg1NVVz5swJ9vZvE4p5GhsbtWLFCn3++edyOp0aNWqUli1bFtJvW0htn6W5uVlLly7VxYsXFRERoYceekgbNmwI+Au1xMREvfHGG1qxYoXy8/PVp08fvfnmm0pISAjpLKGa59KlS3ryySf9vz937px27typ4cOHa+vWrd+qWbZv367m5ubb/pxMmDBBS5Ys+VbN0tDQoNdee01ffvmlIiIiFBsbq6VLl97xH3uGAg9pAwDLcI8fACxD+AHAMoQfACxD+AHAMoQfACxD+AHAMoQfACxD+AHAMv8H6vVuOeXWMb4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(STtestaurocs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#only 3 areas get predicted in train set, subset to look at those\n",
    "STtestpred_sub = STtestpredmatrix.loc[:,STtestpredmatrix.sum() > 0]\n",
    "STtrainpred_sub = STtrainpredmatrix.loc[:,STtrainpredmatrix.sum() > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "STtestaurocs = matrixtoauroc(STtestpred_sub, ytest2)\n",
    "STtrainaurocs = matrixtoauroc(STtrainpred_sub, ytrain2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAXWklEQVR4nO3df0xV9/3H8dfVTsnVanqBuG4aERtutAperpo6QAe10Yhdq+s2U6stZjpNuxm10HYWqKidGtOq+AOFqWjUaly1TtetNtrNBatlXppFHENta62rAkGskALq5/vH4v165w+g3HtZ93k+Ev/wnM+95/NO43OX4/XMYYwxAgBYo1NHbwAAEF6EHwAsQ/gBwDKEHwAs818ffrfbLbfb3dHbAID/Gf/14QcABBfhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsMx9Hb0BIBhiXj7QIdf9dEl6h1wXaA8+8QOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZQg/AFiG8AOAZdoc/oKCArndbuXl5fmPGWOUn5+v5ORkxcfHa8qUKaqsrAx4XV1dnTIzM+X1euX1epWZmakrV660fwIAQJu0KfxlZWXatWuX3G53wPHCwkJt3LhR2dnZ2r17t1wulzIyMnT16lX/mnnz5qm8vFyFhYUqKipSeXm5srKygjMFAKDVWh3+r776Si+++KIWL16snj17+o8bY7RlyxbNmDFDY8aMUVxcnJYuXar6+nrt379fknTmzBkdOXJEeXl5SkxMlMfj0YIFC3T48GGdPXs2+FMBAO6q1eHPzs7WmDFjNGLEiIDj58+fV1VVlZKSkvzHIiIiNGzYMPl8PkmSz+eT0+lUYmKif43X65XT6fSvAQCER6vCv2vXLp07d06zZ8++7VxVVZUkKSoqKuB4ZGSkqqurJUnV1dVyuVxyOBz+8w6HQy6Xy78GABAe97W04OzZs3rjjTe0bds2denS5a7rbo16a88bY1p8HQAguFoMf1lZmWpra/X444/7j12/fl0fffSR3nrrLf99/KqqKj344IP+NTU1Nf6fAqKiolRTUxMQemOMamtrFRkZGdSBAAD31mL4R48erUGDBgUce+WVVxQTE6Nf/OIX6tevn6Kjo1VSUqL4+HhJUmNjo0pLS/3f2vF4PGpoaJDP5/Pf5/f5fGpoaJDH4wn2TACAe2gx/D169FCPHj0CjjmdTvXs2VNxcXGSpKlTp6qgoECxsbGKiYnRunXr5HQ6NX78eElS//79lZKSotzcXC1cuFDGGOXm5io1NVWxsbEhGAsAcDcthr81pk+frsbGRuXl5amurk4JCQnauHGjunfv7l+zfPlyLVq0SNOmTZMkpaWlKScnJxiXBwC0gcMYYzp6E/dy8x+LVVRUdPBO8N8s5uUDHXLdT5ekd8h1gfbgWT0AYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWIfwAYBnCDwCWaTH827Zt0+OPP67ExEQlJibqZz/7mT744AP/eWOM8vPzlZycrPj4eE2ZMkWVlZUB71FXV6fMzEx5vV55vV5lZmbqypUrQR8GANCyFsPfq1cvvfjii9qzZ49+97vf6ZFHHtHzzz+vf/zjH5KkwsJCbdy4UdnZ2dq9e7dcLpcyMjJ09epV/3vMmzdP5eXlKiwsVFFRkcrLy5WVlRW6qQAAd9Vi+EePHq1Ro0apb9++6tevn+bMmaNu3bqprKxMxhht2bJFM2bM0JgxYxQXF6elS5eqvr5e+/fvlySdOXNGR44cUV5enhITE+XxeLRgwQIdPnxYZ8+eDfmAAIBAbbrHf/36dR04cEANDQ3yeDw6f/68qqqqlJSU5F8TERGhYcOGyefzSZJ8Pp+cTqcSExP9a7xer5xOp38NACB87mvNooqKCk2aNEmNjY1yOp1avXq13G63Tpw4IUmKiooKWB8ZGalLly5Jkqqrq+VyueRwOPznHQ6HXC6XqqurgzUHAKCVWhX+fv36ae/evbpy5Yree+89vfTSS9q6dav//K1Rv5M7nTfGtPg6AEDwtSr8Xbp0Ud++fSVJgwcP1t///ndt3rxZs2bNkiRVVVXpwQcf9K+vqanx/xQQFRWlmpqagNAbY1RbW6vIyMigDgMAaNk3+h7/jRs31NTUpN69eys6OlolJSX+c42NjSotLZXH45EkeTweNTQ0BNzP9/l8/r8nAACEV4uf+JcvX64f/vCH+u53v+v/ts7x48e1fv16ORwOTZ06VQUFBYqNjVVMTIzWrVsnp9Op8ePHS5L69++vlJQU5ebmauHChTLGKDc3V6mpqYqNjQ35gACAQC2Gv7q6WpmZmaqqqtL9998vt9utwsJCpaSkSJKmT5+uxsZG5eXlqa6uTgkJCdq4caO6d+/uf4/ly5dr0aJFmjZtmiQpLS1NOTk5IRoJAHAvDmOM6ehN3Ivb7Zb0728WAXcT8/KBDrnup0vSO+S6QHvwrB4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsAzhBwDLEH4AsEyL4V+/fr1+/OMfKzExUY888ohmzpypf/7znwFrjDHKz89XcnKy4uPjNWXKFFVWVgasqaurU2Zmprxer7xerzIzM3XlypXgTgMAaFGL4T9+/LiefvppvfXWWyouLlbnzp2VkZGhy5cv+9cUFhZq48aNys7O1u7du+VyuZSRkaGrV6/618ybN0/l5eUqLCxUUVGRysvLlZWVFZqpAAB35TDGmLa8oL6+XkOHDtWaNWuUlpYmY4xSUlI0efJkzZo1S5L09ddfa8SIEXrppZc0adIknTlzRuPGjdP27dvl9XolSaWlpZo8ebLeffddxcbG3vV6brdbklRRUfFNZ4QFYl4+0CHX/XRJeodcF2iPNt/jr6+v140bN9SjRw9J0vnz51VVVaWkpCT/moiICA0bNkw+n0+S5PP55HQ6lZiY6F/j9XrldDr9awAA4dHm8C9evFgDBgyQx+ORJFVVVUmSoqKiAtZFRkaqurpaklRdXS2XyyWHw+E/73A45HK5/GsAAOFxX1sW/+Y3v9Hf/vY37dixQ507dw44d2vU7+RO540xLb4OABBcrf7E//rrr+vAgQMqLi5Wnz59/Mejo6Ml/f8n/5tqamr8PwVERUWppqZGt/51gjFGtbW1ioyMbNcAAIC2aVX4Fy1apP3796u4uFj9+/cPONe7d29FR0erpKTEf6yxsVGlpaX+20Eej0cNDQ0B9/N9Pp8aGhr8awAA4dHirZ4FCxbonXfe0Zo1a9SjRw//J3un06lu3brJ4XBo6tSpKigoUGxsrGJiYrRu3To5nU6NHz9ektS/f3+lpKQoNzdXCxculDFGubm5Sk1Nvec3egAAwdfi1zlvfp3yP73wwgv65S9/Kenft21Wr16tnTt3qq6uTgkJCcrJyVFcXJx//eXLl7Vo0SIdOnRIkpSWlqacnBz/t4Nauj5f58S98HVOoPXa/D3+cCP8aA3CD7Qez+oBAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwDOEHAMsQfgCwTKvC/9FHH2nmzJlKSUmR2+3W22+/HXDeGKP8/HwlJycrPj5eU6ZMUWVlZcCauro6ZWZmyuv1yuv1KjMzU1euXAneJACAVmlV+BsaGhQXF6f58+crIiLitvOFhYXauHGjsrOztXv3brlcLmVkZOjq1av+NfPmzVN5ebkKCwtVVFSk8vJyZWVlBW8SAECrtCr8o0aN0ty5czV27Fh16hT4EmOMtmzZohkzZmjMmDGKi4vT0qVLVV9fr/3790uSzpw5oyNHjigvL0+JiYnyeDxasGCBDh8+rLNnzwZ/KgDAXbX7Hv/58+dVVVWlpKQk/7GIiAgNGzZMPp9PkuTz+eR0OpWYmOhf4/V65XQ6/WsAAOHR7vBXVVVJkqKiogKOR0ZGqrq6WpJUXV0tl8slh8PhP+9wOORyufxrAADhEbRv9dwa9daeN8a0+DoAQHC1O/zR0dGS/v+T/001NTX+nwKioqJUU1MjY4z/vDFGtbW1ioyMbO8WAABt0O7w9+7dW9HR0SopKfEfa2xsVGlpqTwejyTJ4/GooaEh4H6+z+dTQ0ODfw0AIDzua82i+vp6nTt3TpJ048YNXbhwQadOnVLPnj31ve99T1OnTlVBQYFiY2MVExOjdevWyel0avz48ZKk/v37KyUlRbm5uVq4cKGMMcrNzVVqaqpiY2NDNx0A4DYOc+v9l7s4duyYpk6detvxCRMmaMmSJTLGaPXq1dq5c6fq6uqUkJCgnJwcxcXF+ddevnxZixYt0qFDhyRJaWlpysnJUY8ePe55bbfbLUmqqKho02CwS8zLBzrkup8uSe+Q6wLt0arwdyTCj9Yg/EDr8aweALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAyxB+ALAM4QcAy4Q9/Nu2bVNaWpoGDx6siRMnqrS0NNxbAACrhTX8f/jDH/T6669r5syZ2rt3rzwej6ZPn64LFy6EcxsAYLWwhn/Tpk2aMGGCfvrTn6p///7Kzs5WdHS0duzYEc5tAIDV7gvXhZqamnTy5ElNmzYt4HhSUpJ8Pl+Lr3e73aHaGv4HdO2g67r3zO2gKwMtq6iouOPxsH3ir62t1fXr1xUVFRVwPDIyUlVVVeHaBgBYL2yf+G9yOBytOnbT3f4XCwDwzYTtE/8DDzygzp073/bpvqam5rafAgAAoRO28Hfp0kUPP/ywSkpKAo6XlJTI4/GEaxsAYL2w3urJyMhQVlaW4uPjlZiYqB07dujSpUuaNGlSOLcBAFYLa/jHjRun2tparVu3TpcuXVJcXJw2bNig73//++HcBgBYzWGMMR29CQBA+PCsHgCwDOEHAMsQ/lb4pg+WKy0t1cCBAzV+/PjbzhUXF2vs2LGKj4/XyJEjtWDBAtXX1wd763cU7Hmam5u1evVqjR49WoMHD9aPfvQj/eUvfwnF1m/TllmOHTsmt9t9268zZ84ErPvTn/6kcePGadCgQRo3bpwOHjwY6jH8gj1PZWWlfvWrX+nRRx+V2+1Wfn5+OMaQFPxZdu3apaefflrDhw/X0KFDNWXKlLA95DHYs7z77ruaOHGihg4dqiFDhuiJJ57Qnj17wjHKvxnc04EDB8zAgQPNzp07zenTp01eXp4ZMmSI+eKLL+75usuXL5u0tDQzbdo0k56eHnBu37595uGHHzZ79uwxn3/+uSkpKTGpqanmlVdeCeUoxpjQzLNs2TKTlJRkDh8+bM6dO2e2bdtmBg8ebE6ePBnKUdo8y4cffmji4uJMZWWluXTpkv/XtWvX/GtOnDhhBgwYYNauXWtOnz5t1q5dawYMGGDKyspCOkuo5vn444/NkiVLzL59+0xaWppZtWpVyOcI1Sxz5841W7duNSdPnjRnzpwx2dnZJiEhwXzyySffullKSkrMwYMHzenTp81nn31mNm/ebAYMGGA++OCDkM5yE+FvwVNPPWXmz58fcOyxxx4zy5cvv+frnn/+eZOfn29WrVp1WygXLFhgJk+eHHBs5cqVt60LhVDMk5SUZDZv3hxw7IUXXjDz5s0Lzqbvoq2z3PwDWVNTc9f3nD17tnnuuecCjj377LNmzpw57d9wC0Ixz63S09PDFv5Qz2KMMTdu3DA/+MEPzJYtW9q115aEYxZjjHnyySdb/HMYLNzquYebD5ZLSkoKON7Sg+W2bdum6upqzZo1647nvV6vTp06pbKyMknShQsXdOjQIY0cOTJ4m7+DUM3T3Nysrl0DH5PWtWtXnThxov2bvotvOoskPfXUU0pOTtazzz6rDz/8MOBcWVnZbe+ZnJzcqgcJtkeo5ukI4ZqlublZjY2N6tGjR7v3fDfhmMUYo6NHj+qTTz7R0KFDg7LvloT9WT3fJvd6sNx//gvkmyoqKrRmzRrt3LlTnTt3vuOa9PR0Xb58Wc8884yMMbp27ZqeeOIJZWZmBn2GW4VqnuTkZBUXF2v48OGKiYnR0aNHdfDgQV2/fj3oM9z0TWaJjo7Wa6+9psGDB6u5uVnvvPOOnnvuOW3dulXDhg2TJFVXV9/2nlFRUSF/kGCo5ukI4ZrlzTfflNPp1KOPPhr0GW4K5SxfffWVRo4cqaamJnXq1Ek5OTkaNWpUyGa5FeFvhdY+WK6pqUlz585VVlaW+vTpc9f3O378uNauXavc3FzFx8fr3LlzWrx4sVatWqXZs2cHde93Eux55s+fr1dffVXp6elyOBzq06ePJk6cqLfffjuo+76Ttjz0LzY2VrGxsf7fezweffHFF/rtb397z1AaY+75IMFgCsc84RLKWYqLi7Vz505t3rxZ3bt3D96m7yIUs3Tr1k179+5VQ0ODjh49qiVLlqh3794aMWJE8Af4D9zquYe2Plju0qVLOn36tH79619r4MCBGjhwoNasWaPKykoNHDhQf/3rXyVJK1asUHp6un7yk5/I7Xbrscce05w5c1RUVKRr16596+ZxuVxau3atysrKdPjwYf3xj3+U0+lU7969/2tmuZuEhAR99tln/t9HRUWpurq6Xe/5TYRqno4Q6lmKi4u1YsUKbdiwQfHx8e3e772EcpZOnTqpb9++GjBggKZNm6axY8eqoKAgKPtuCeG/h7Y+WK5Xr176/e9/r7179/p/TZo0SX379vX/X01K0tdff33bbZPOnTvLhPgfUYdqnpu6du2qXr166dq1a3rvvfdC+iN4sB76d+rUKUVHR/t/P2TIkA55kGCo5ukIoZxl06ZN/uiH4354OP+73LhxQ01NTd9on23FrZ4WtPRguaysLEnSsmXL9J3vfEdxcXEBr4+MjFSXLl0CjqempmrTpk0aNGiQ/1bPypUrlZqaqvvuC+1/klDM8/HHH+vixYsaMGCALl68qPz8fN24cUM///nP/2tmkaTNmzerd+/eeuihh9Tc3Kx9+/bp/fffD/hu+9SpU/XMM89o/fr1Gj16tN5//30dO3ZM27dvD+ksoZqnqanJ//3xxsZGVVVV6dSpU3I6nerbt++3apaioiKtWLFCy5YtU0xMjP9TeEREhO6///5v1Szr1q1TQkKC+vTpo6amJv35z3/Wvn379Oqrr4ZsjlsR/ha09GC5f/3rX21+z1mzZsnhcGjlypX68ssv9cADDyg1NVVz5swJ9vZvE4p5GhsbtWLFCn3++edyOp0aNWqUli1bFtJvW0htn6W5uVlLly7VxYsXFRERoYceekgbNmwI+Au1xMREvfHGG1qxYoXy8/PVp08fvfnmm0pISAjpLKGa59KlS3ryySf9vz937px27typ4cOHa+vWrd+qWbZv367m5ubb/pxMmDBBS5Ys+VbN0tDQoNdee01ffvmlIiIiFBsbq6VLl97xH3uGAg9pAwDLcI8fACxD+AHAMoQfACxD+AHAMoQfACxD+AHAMoQfACxD+AHAMv8H6vVuOeXWMb4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(STtrainaurocs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAUuklEQVR4nO3de0zV9/3H8RdilbKpVUCTFqJDwxko11J3ASVimxlZJ6IzpnPlknWzWbvGbqM6dVWjFkltERRXogIzQl0bYWy6VU3dzII31tKlhqCSVlFabsFkxZWLfH9/7MeJyOWcA+dA/ez5SPjjfM/3e87nnc5nT78cP/OyLMsSAMBY48Z6AQAAzyL0AGA4Qg8AhiP0AGC4r1TobTabbDbbWC8DAIzylQo9AMD9CD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhHIb+rbfe0ooVKxQTE6Nvf/vbWrt2ra5cueLwhWtra7VmzRpFRERowYIF2rt3r9g/DQBGn8PQX7x4Uc8884zefvttFRcXy9vbW+np6bp9+/ag13zxxRfKyMiQn5+f3n33XW3atEkHDx5UYWGhWxcPAHDMy9Vtitvb2xUbG6t9+/YpMTFxwHNKSkr0+uuvq7KyUj4+PpKk/Px8lZaW6uzZs/Ly8hrwut6/FVtbW+vKkgAAQ3D5Hn17e7t6eno0efLkQc+prq5WbGysPfKSFB8fr6amJt28eXN4KwUADMt4Vy/YsWOHQkNDFR0dPeg5LS0tmjFjRp9j/v7+9ueCgoJcfVsAGDWz1h8fk/f9NCvJI6/rUuhfe+01/fOf/1Rpaam8vb2HPPf+2zO9d4gGu20DAPAMp0O/c+dOnThxQsXFxQ4/kfv7+6ulpaXPsdbWVkmSn5/fMJYJABgup+7Rb9++XX/+859VXFys2bNnOzw/KipKVVVV6ujosB+rrKzU9OnTFRgYOPzVAgBc5jD0W7du1bFjx7R7925NnjxZzc3Nam5uVnt7u/2c3bt3KzU11f746aef1sMPP6z169frypUrOnnypAoKCpSens6tGwAYZQ5v3ZSUlEiS0tLS+hx/4YUX9OKLL0qSmpubVV9fb39u0qRJOnTokLZt26YVK1ZoypQpysjIUHp6uhuXDgBwhsvfo/ckvkcP4KvAtG/dsNcNABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABiO0AOA4Qg9ABjOqdBfunRJa9eu1YIFC2Sz2XTs2LEhz79586ZsNlu/n7Nnz7pl0QAA54135qQ7d+4oJCREycnJeuWVV5x+8QMHDuib3/ym/fGUKVNcXyEAYEScCn1CQoISEhIkSRs2bHD6xR955BEFBAQMb2UAALdwKvTD9eKLL6qjo0MzZ85UWlqalixZ4sm3AwAMwCOh9/X11SuvvKKYmBh5e3vr/fff17p169TR0aFly5Z54i0BAIPwSOinTZumjIwM++Pw8HC1tbXpwIEDhB4ARtmofb0yMjJS169fH623AwD8v1ELfU1NDb+YBYAx4NStm/b2dt24cUOS1NPTo4aGBtXU1GjKlCl69NFHtXv3bv3rX/9ScXGxJKmsrEzjx49XWFiYvLy8dObMGZWUlOhXv/qV5yYBAAzIqdB//PHHevbZZ+2P8/LylJeXp+XLlysrK0vNzc2qr6/vc83+/fvV0NCgcePGadasWdqxYwf35wFgDHhZlmWN9SJ62Ww2SVJtbe0YrwTA/7JZ64+Pyft+mpXkkddlrxsAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMByhBwDDEXoAMJxTob906ZLWrl2rBQsWyGaz6dixYw6vqa2t1Zo1axQREaEFCxZo7969sixrxAsGALjGqdDfuXNHISEh2rhxo3x8fBye/8UXXygjI0N+fn569913tWnTJh08eFCFhYUjXjAAwDXjnTkpISFBCQkJkqQNGzY4PL+iokL/+c9/tGvXLvn4+CgkJER1dXUqLCxUenq6vLy8RrZqAIDTnAq9q6qrqxUbG9vn0398fLz27NmjmzdvKigoyBNvK0matf64x157KJ9mJY3J+wKAIx75ZWxLS4v8/Pz6HPP397c/BwAYPR771s39t2d6fxHLbRsAGF0eCb2/v3+/T+6tra2S1O+TPgDAszwS+qioKFVVVamjo8N+rLKyUtOnT1dgYKAn3hIAMAinQt/e3q6amhrV1NSop6dHDQ0NqqmpUUNDgyRp9+7dSk1NtZ//9NNP6+GHH9b69et15coVnTx5UgUFBXzjBgDGgFOh//jjj5WcnKzk5GR9+eWXysvLU3JysnJzcyVJzc3Nqq+vt58/adIkHTp0SE1NTVqxYoW2bdumjIwMpaene2YKAMCgnPp65be+9S3V1tYO+nxWVla/YzabTUeOHBn+ygAAbsFeNwBgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIZzOvRHjhxRYmKiwsPDlZKSoqqqqkHPvXDhgmw2W7+furo6tywaAOC88c6cdOLECe3cuVOvvvqqHn/8cZWUlOi5557T8ePH9eijjw563fHjxzVlyhT742nTpo18xQAAlzj1ib6wsFDLly/XqlWrNHv2bG3evFkBAQEqLS0d8rpp06YpICDA/uPt7e2WRQMAnOcw9J2dnbp8+bLi4uL6HI+Li9OHH3445LUrV65UfHy8UlNTdf78+ZGtFAAwLA5D39bWprt378rf37/PcT8/PzU3Nw94TUBAgLZs2aLc3Fzl5eXpG9/4htLS0nTp0iX3rBoA4DSn7tFLkpeXl1PHJCk4OFjBwcH2x9HR0bp165YOHjyoJ554YhjLBAAMl8NP9FOnTpW3t3e/T++tra39PuUPJTIyUtevX3d9hQCAEXEY+gkTJmju3LmqrKzsc7yyslLR0dFOv1FNTY0CAgJcXyEAYEScunWTnp6uzMxMRUREKCYmRqWlpWpqatLq1aslSZmZmZKk7OxsSVJRUZECAwM1Z84cdXV1qaKiQqdPn1ZeXp6HxgAADMap0C9dulRtbW3av3+/mpqaFBISooKCAj322GOSpM8++6zP+V1dXdq1a5caGxvl4+OjOXPmqKCgQAkJCe6fAAAwJC/LsqyxXkQvm80mSaqtrR32a8xaf9xdy3HJp1lJY/K+ANzPtI6w1w0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGI7QA4DhCD0AGM7p0B85ckSJiYkKDw9XSkqKqqqqhjz/4sWLSklJUXh4uBYvXqzS0tIRLxYA4DqnQn/ixAnt3LlTa9euVXl5uaKjo/Xcc8+poaFhwPPr6+v105/+VNHR0SovL9fPfvYzbd++Xe+9955bFw8AcMyp0BcWFmr58uVatWqVZs+erc2bNysgIGDQT+lvv/22pk+frs2bN2v27NlatWqVkpOTdejQIbcuHgDg2HhHJ3R2dury5cvKyMjoczwuLk4ffvjhgNdUV1crLi6uz7H4+HiVl5erq6tLDz300JDvabPZHC1rUBOHfeXI2MpeHqN3BuBuD2pHamtrBzzu8BN9W1ub7t69K39//z7H/fz81NzcPOA1LS0t8vPz63PM399f3d3damtrc3bNAAA3cPiJvpeXl5dTxwZ7zrIsh9cM9m8jAMDwOfxEP3XqVHl7e/f79N7a2trvU34vf39/tbS09Dt//PjxeuSRR0awXACAqxyGfsKECZo7d64qKyv7HK+srFR0dPSA10RFRQ14/rx58xzenwcAuJdT37pJT09XWVmZ3nnnHdXV1Wn79u1qamrS6tWrJUmZmZnKzMy0n7969Wo1NjZqx44dqqur0zvvvKOysrJ+v9AFAHieU/foly5dqra2Nu3fv19NTU0KCQlRQUGBHnvsMUnSZ5991uf8oKAgFRQU6LXXXlNpaammT5+ujRs36nvf+577JwAADMnL6v0tKQDASOx1AwCGI/QAYDhCPwBXN3DrVVVVpbCwMH3/+9/v91xxcbGWLFmiiIgILVy4UFu3blV7e7u7lz4gd8/T1dWlvXv36sknn1R4eLh+8IMf6OzZs55Yej+uzHLhwgXZbLZ+P3V1dX3Oe++997R06VLNmzdPS5cu1alTpzw9hiT3z3L16lX94he/0OLFi2Wz2ZSXlzcaY9i5e54//OEPeuaZZzR//nzFxsbqxz/+sdP/2x0pd8/yl7/8RSkpKYqNjVVUVJSWLVumsrKy0Rjlvyz0cfz4cSssLMw6evSode3aNWvbtm1WVFSUdevWrSGvu337tpWYmGhlZGRYSUlJfZ6rqKiw5s6da5WVlVn19fVWZWWltWjRImvDhg2eHMWyLM/Mk52dbcXFxVlnzpyxbty4YR05csQKDw+3Ll++7MlRXJ7l/PnzVkhIiHX16lWrqanJ/tPd3W0/54MPPrBCQ0Ot/Px869q1a1Z+fr4VGhpqVVdXP3CzfPTRR1ZWVpZVUVFhJSYmWrm5uR6d4V6emOfll1+2Dh8+bF2+fNmqq6uzNm/ebEVGRlqffPLJAzdLZWWlderUKevatWvW9evXraKiIis0NNT629/+5tFZehH6+6xcudLauHFjn2NPPfWU9frrrw953c9//nMrLy/Pys3N7RfGrVu3Wj/60Y/6HNuzZ0+/8zzBE/PExcVZRUVFfY698MIL1i9/+Uv3LHoQrs7S+wewtbV10Nd86aWXrLS0tD7HUlNTrXXr1o18wUPwxCz3SkpKGtXQe3oey7Ksnp4e67vf/a71+9//fkRrdWQ0ZrEsy0pOTnb459BduHVzj94N3O7fkG2oDdyk//5nXktLi55//vkBn3/88cdVU1Oj6upqSVJDQ4Pef/99LVy40H2LH4Cn5unq6tLEiX23fZo4caI++OCDkS96EMOdRZJWrlyp+Ph4paam6vz5832eG2wDPkevORKemmWsjNY8XV1d6ujo0OTJk0e85sGMxiyWZencuXP65JNPFBsb65Z1O+L0Xjf/C4bawO3+v+nbq7a2Vvv27dPRo0fl7e094DlJSUm6ffu21qxZI8uy1N3drWXLlunXv/6122e4l6fmiY+PV3FxsebPn69Zs2bp3LlzOnXqlO7evev2GXoNZ5aAgABt2bJF4eHh6urq0h//+EelpaXp8OHDeuKJJyT9dwO++1/T399/0A373MFTs4yV0ZrnzTfflK+vrxYvXuz2GXp5cpZ///vfWrhwoTo7OzVu3Dj99re/VUJCgsdmuRehH4CzG7h1dnbq5ZdfVmZmpoKCggZ9vYsXLyo/P1+vvvqqIiIidOPGDe3YsUO5ubl66aWX3Lr2gbh7no0bN2rTpk1KSkqSl5eXgoKClJKSomPHjrl13QNxZXO94OBgBQcH2x9HR0fr1q1bOnjw4JBxtCxryM333GU0ZhlNnpynuLhYR48eVVFRkb7+9a+7b9GD8MQsX/va11ReXq47d+7o3LlzysrKUmBgoL7zne+4f4D7cOvmHq5u4NbU1KRr167pN7/5jcLCwhQWFqZ9+/bp6tWrCgsL0z/+8Q9JUk5OjpKSkvTDH/5QNptNTz31lNatW6cDBw6ou7v7gZtn2rRpys/PV3V1tc6cOaO//vWv8vX1VWBg4FdmlsFERkbq+vXr9seDbcDnymu6ylOzjBVPz1NcXKycnBwVFBQoIiJixOsdiidnGTdunGbOnKnQ0FBlZGRoyZIl+t3vfueWdTtC6O/h6gZuM2bM0J/+9CeVl5fbf1avXq2ZM2fa/y8XJenLL7/sdxvE29vbvnWzp3hqnl4TJ07UjBkz1N3drZMnT3r0P6mHs7neQGpqahQQEGB/PNgGfK68pqs8NctY8eQ8hYWF9siPxv3s0fxn09PTo87OzmGt01XcurlPenq6MjMzFRERoZiYGJWWlvbbwE2SsrOz9dBDDykkJKTP9X5+fpowYUKf44sWLVJhYaHmzZtnv3WzZ88eLVq0SOPHe/YfgSfm+eijj9TY2KjQ0FA1NjYqLy9PPT09+slPfvKVmUWSioqKFBgYqDlz5qirq0sVFRU6ffp0n++XP/vss1qzZo3eeustPfnkkzp9+rQuXLigkpKSB26Wzs5O+3e3Ozo61NzcrJqaGvn6+mrmzJkP3DwHDhxQTk6OsrOzNWvWLPunbB8fH02aNOmBmmX//v2KjIxUUFCQOjs79fe//10VFRXatGmTx+a4F6G/j6sbuDnj+eefl5eXl/bs2aPPP/9cU6dO1aJFi7Ru3Tp3L78fT8zT0dGhnJwc1dfXy9fXVwkJCcrOzvbotyEk12fp6urSrl271NjYKB8fH82ZM0cFBQV9fgEWExOjN954Qzk5OcrLy1NQUJDefPNNRUZGPnCzNDU1KTk52f74xo0bOnr0qObPn6/Dhw8/cPOUlJSoq6ur35+T5cuXKysr64Ga5c6dO9qyZYs+//xz+fj4KDg4WLt27RrwL1d6ApuaAYDhuEcPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABgOEIPAIYj9ABguP8DZdkXA3RURAAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(STtestaurocs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}