synthetic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "from catboost import Pool, CatBoostRegressor\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assume that we have two categorical features with 9 values each, so there are 81 possible feature combinations. The target y depends on the features x as y = mean(x) + eps(x), where mean(x) is some unknown fixed value and eps(x) is a normally distributed noise with mean 0 and variance var(x). We let mean(x) to be randomly generated and var(x) to have two values - 0.01 and 0.04. Also, there will be an \"unknown\" region without any training examples. To indicate this region on the figure, we set var(x) = 0 for these feature combinations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for generating mean and variance\n",
    "def gen_parameters(noise=0.01, seed=0):\n",
    "    \n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    mean = np.random.rand(9, 9)\n",
    "    \n",
    "    figure = np.array([\n",
    "        [1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
    "        [1, 1, 4, 1, 1, 1, 4, 1, 1],\n",
    "        [1, 4, 0, 4, 1, 4, 0, 4, 1],\n",
    "        [4, 0, 0, 0, 4, 0, 0, 0, 4],\n",
    "        [1, 4, 0, 0, 0, 0, 0, 4, 1],\n",
    "        [1, 1, 4, 0, 0, 0, 4, 1, 1],\n",
    "        [1, 1, 1, 4, 0, 4, 1, 1, 1],\n",
    "        [1, 1, 1, 1, 4, 1, 1, 1, 1],\n",
    "        [1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
    "    ])\n",
    "\n",
    "    var = figure*noise\n",
    "    \n",
    "    return mean, var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True data uncertainty (variance). The white region will not contain any traing data.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAD8CAYAAACihcXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAW7klEQVR4nO3df5BdZX3H8fdnNxAFNDjUOpggSUvUhtpBoUGrdaQpNqg1OoUxOK3USV2dmqp12orVItIfIx0KtQO1LoJFag2alnari9gB2mqLcQNCIUHqEkE2/uJXg4gIIZ/+cQ96vd299+7u/XHO2c9r5gznnh/3+9zZ5btPvud5nivbRETEcI0MuwEREZFkHBFRCknGERElkGQcEVECScYRESWQZBwRUQJJxhERc5C0UdLtkqYlnTnL+eWSrijO75C0uji+WtL3Jd1UbH/TKdayLhrzXGATsLI4tBeYsH3bfD5URESVSBoFLgJOBmaAKUkTtnc3XbYFeMD2MZI2A+cCryvO3WH7uG7jte0ZS3oXsA0Q8KViE/CJ2f5KRETUyHpg2vYe24/SyIWbWq7ZBFxW7G8HNkjSQoJ16hlvAY61/VjzQUnnA7uAD8x2k6QxYAzgQ+/eevzYa09ZSNsiYokZ+flXLCiRNXuLntr1tOIP8903U+Sqwrjt8WJ/JXB307kZ4MSWt/jhNbb3S9oHHFGcWyPpy8CDwHttf75dWzol4wPAM4G7Wo4fWZybVfFhxgEOTE1mvnVElFJzruqxbwLPsn2fpOOBf5J0rO0H57qhUzJ+B3CNpK/yo78QzwKOAbb2osUREb3Uw1EJe4Gjml6vKo7Nds2MpGXACuA+Nxb9+QGA7Rsk3QE8G9g5V7C2ydj2ZyU9m0btpPkB3pTtx7v+SBERA7JsYSXb2UwBayWtoZH3NgOvb7lmAjgDuB44FbjWtiU9Hbjf9uOSfgpYC+xp2+5OrbF9APjivD9GRMQQjPQoFxc14K3A1cAocKntXZLOAXbangAuAS6XNA3cTyNhA7wUOEfSYzRKum+xfX+7eOr3EpqpGUdEt3rxAO+doyu6zjnnP76vZ93oxerYM46IqJKR3pUpBirJOCJqparTipOMI6JWelUzHrQk44ioldGUKSIihi9lioiIEkiZIiKiBNIzDq57xZuGEvekyYuHEncpyc+2OjK0LSKiBJZVMxcnGUdEvaRMERFRAiNUs2ucZBwRtZLRFBERJZAyRURECaRnHBFRAj1cXH6gkowjolZSpoiIKIGUKSIiSiBD2yIiSiA944iIEhhNMo6IGL6qlikW/OBR0ht72ZCIiF4YUfdbmSxmFMj75zohaUzSTkk7x6+8ahEhIiLmZ2QeW5m0LVNI+u+5TgHPmOs+2+PAOMCBqUkvuHUREfNUsg5v1zrVjJ8B/ArwQMtxAf/VlxZFRCxCXReX/zRwmO2bWk9I+re+tCgiYhHKVn7oVttkbHtLm3Ov731zIiIWp5r94gxti4iaUU3LFBERlVLNVJxkHBE1U8uacURE1VS0SpFkHBH1UtXp0EnGEVEr1UzFScYRUTNlW3OiW0nGEVErqmjfuJbJ+LpXvGkocTfctXsoca85et1Q4p40efFQ4g7j55ufbXX0MhVL2gh8EBgFPmL7Ay3nlwMfA44H7gNeZ/vOpvPPAnYDZ9s+r12sqo4CiYiYVa+W0JQ0ClwEnAKsA06X1PrXcQvwgO1jgAuAc1vOnw90tXRlknFE1MoI6nrrYD0wbXuP7UeBbcCmlms2AZcV+9uBDSqmAEp6DfA1YFd37Y6IqBHNZ2tae73YxpreaiVwd9PrmeIYs11jez+wDzhC0mHAu2iz7nurWtaMI2Lpms+kj+a113vsbOAC2w91u1ZGknFE1EoPH+DtBY5qer2qODbbNTOSlgEraDzIOxE4VdKfA4cDByQ9YvvCuYIlGUdErfRwaNsUsFbSGhpJdzPQunTwBHAGcD1wKnCtbQO/+MP2SGcDD7VLxJBkHBE1M9qjXGx7v6StwNU0hrZdanuXpHOAnbYngEuAyyVNA/fTSNgLkmQcEbXSy3HGtieByZZjZzXtPwKc1uE9zu4mVpJxRNRKZuBFRJRAltCMiCiBqk6e6NhuSc+VtKEYxNx8fGP/mhURsTDzmfRRJm2TsaS3Af8M/A5wq6TmqYB/1s+GRUQsxIjU9VYmnXrGbwKOt/0a4GXAH0l6e3Fuzk/SPMVw/Mqu1siIiOiJqvaMO9WMR2w/BGD7TkkvA7ZLOpo2n6V5iuGBqUn3qK0RER11O/24bDr1jL8t6bgnXhSJ+VXATwDP62fDIiIWoldLaA5ap57xG4D9zQeKlYneIOnDfWtVRMQCqWxZtkttk7HtmTbn/rP3zYmIWJyRio5tyzjjiKiVqtaMk4wjolYqmouTjCOiXtIzjogogYrm4iTjiKiXss2s61aScUTUykgdh7ZFRFSNMrQtImL4qvoAT43vzuufa56+cuBrU2y4a/egQy5J1xy9bihx8/Ptv6H9bO/Zu+hMese6tV3nnJ/e/dXSZO70jCOiVqraM04yjohaqWguTjKOiHoZzWiKiIjhS5kiIqIEKpqLk4wjol6SjCMiSqCWi8tHRFRNHuBFRJRAyhQRESWQ0RQRESVQ0VzcORlLWg/Y9pSkdcBG4Cu2J/veuoiIeaplz1jS+4BTgGWS/hU4EbgOOFPS823/6QDaGBHRtYrm4o4941OB44DlwLeAVbYflHQesAOYNRlLGgPGAN5x2Ape9aRDe9fiiIg2RkarmY07JeP9th8HHpZ0h+0HAWx/X9KBuW6yPQ6Mw3CW0IyIpauqZYpOa+I/KumQYv/4Jw5KWgHMmYwjIoZmRN1vHUjaKOl2SdOSzpzl/HJJVxTnd0haXRxfL+mmYrtZ0ms7NrvD+ZfafhjAdnPyPQg4o+MniYgYNKn7re3baBS4iMZzs3XA6cUghmZbgAdsHwNcAJxbHL8VOMH2cTQGPXxYUttKRNtkbPsHcxy/1/YtbT9JRMQQSOp662A9MG17j+1HgW3AppZrNgGXFfvbgQ2SZPth2/uL408COpZrK/rVfRERcxgd6XqTNCZpZ9M21vROK4G7m17PFMeY7Zoi+e4DjgCQdKKkXcAtwFuakvOsMukjImplPgsFNQ826DXbO4BjJf0McJmkq2w/Mtf16RlHRL30qGYM7AWOanq9qjg26zVFTXgFcF/zBbZvAx4CfrZdsCTjiKgVjajrrYMpYK2kNZIOBjYDEy3XTPCjwQynAtfadnHPMgBJRwPPBe5sFyxlioiolx6NM7a9X9JW4GpgFLjU9i5J5wA7bU8AlwCXS5oG7qeRsAFeQmOm8mM0hgH/tu1728VLMo6IeunhesbFGjyTLcfOatp/BDhtlvsuBy6fT6wk44ioFY1Ws/qaZBwR9VLR6dB9T8YnTV7c7xD/zzVHt06SGYwNd+0eStxhWWqfdxiG9bs8jP9ve0XV7BinZxwRNZOecUTE8OXboSMiyiA944iI4ctoioiIMkiZIiKiBFKmiIgYvqp+7VKScUTUS8oUERHDlwd4ERFlkDJFRMTwVXXSx7z785I+1o+GRET0RO++6WOg2vaMJbWuai/gJEmHA9h+db8aFhGxIDXtGa8CHgTOB/6i2L7btD+r5m9cHb/yql61NSKiI0ldb2XSqWZ8AvB24D3A79u+SdL3bf97u5uav3H1wNSke9LSiIhu1HE0he0DwAWSPlX899ud7omIGKqS9Xi71VVitT0DnCbplTTKFhER5VTnZPwE258BPtOntkRELN5IDcsUERGVsxR6xhERpZdkHBFRAqOjw27BgiQZR0S9pGccEVECScYRESWQZBwRUQIZ2hYRUQJJxuVx0uTFQ4l7zdHrhhJ3w127hxJ3KRnWz3ZYv8uVljJFRMTwKT3jiIgSSM84IqIEkowjIkogyTgiogQqOh26mpXuiIi59PALSSVtlHS7pGlJZ85yfrmkK4rzOyStLo6fLOkGSbcU//2lTrGSjCOiXnqUjCWNAhcBpwDrgNMltY5x3AI8YPsY4ALg3OL4vcCv2n4ecAZweadmJxlHRL2MjHS/tbcemLa9x/ajwDZgU8s1m4DLiv3twAZJsv1l298oju8Cnixpedtmz+tDRkSU3Tx6xs3fZF9sY03vtBK4u+n1THGM2a6xvR/YBxzRcs2vATfa/kG7ZucBXkTUyzxGUzR/k31/mqJjaZQuXt7p2iTjiKiX3o2m2Asc1fR6VXFstmtmJC0DVgD3AUhaBVwJvMH2HZ2CzatMIeklkt4pqWOWj4gYit6NppgC1kpaI+lgYDMw0XLNBI0HdACnAtfatqTDaXx585m2/7ObZrdNxpK+1LT/JuBC4CnA+2Yb5hERMXQ9SsZFDXgrcDVwG/BJ27sknSPp1cVllwBHSJoG3gk8kRe3AscAZ0m6qdh+sl28TmWKg5r2x4CTbd8j6Tzgi8AHZrupKIKPAXzo3VsZe+0pHcJERPRIDxcKsj0JTLYcO6tp/xHgtFnu+xPgT+YTq1MyHpH0NBo9aNm+pwj0PUn757qpuSh+YGrS82lQRMSi1HQ69ArgBkCAJR1p+5uSDiuORUSUy0g1p0O3Tca2V89x6gDw2p63JiJisUaq2U9c0NA22w8DX+txWyIiFk/VnMuWccYRUS81rRlHRFRLvnYpIqIE0jOOiCiBOo6miIionJQpIiJKIGWKiIgSyNC2iIgSWEqTPmJ2J01ePJS41xzd+rVcg7Hhrt1DiTuMzzusn20sQB7gRUSUQMoUERElkDJFREQJZDRFREQJpEwREVECKVNERJRARlNERJRAyhQRESWQMkVERAmkZxwRUQIZ2hYRUQJZQjMiogQqOpqi7Z8QSSdKemqx/2RJ75f0L5LOlbRiME2MiJgHqfutRDr15y8FHi72PwisAM4tjn10rpskjUnaKWnn+JVX9aShERFdGRnpfiuRTmWKEdv7i/0TbL+g2P+CpJvmusn2ODAOcGBq0otvZkREl0rW4+1Wpz8Nt0p6Y7F/s6QTACQ9G3isry2LiFgIjXS/lUinnvFvAR+U9F7gXuB6SXcDdxfnIiLKpaIP8NomY9v7gN8sHuKtKa6fsf3tQTQuImLe6jwDz/aDwM19bktExOKVrPzQrYwzjoh6qegDvCTjiKiXivaMq9nqiIg5SOp66+K9Nkq6XdK0pDNnOb9c0hXF+R2SVhfHj5B0naSHJF3YTbuTjCOiXkaWdb+1IWkUuAg4BVgHnC5pXctlW4AHbB8DXEBjUhzAI8AfAb/XdbO7vTAiohJG1P3W3npg2vYe248C24BNLddsAi4r9rcDGyTJ9vdsf4FGUu6u2d1eGBFRCfOY9NG8dEOxjTW900oacyqeMFMcY7ZritnK+4AjFtLsPMCLiHqZx2iK5qUbhi3JOCLqpXejKfYCRzW9XlUcm+2aGUnLaCymdt9CgiUZ18BJkxcPJe41R7c+yxiMYX3eqIjejTOeAtZKWkMj6W4GXt9yzQRwBnA9cCpwre0FLY6WZBwR9TLam7UpbO+XtBW4GhgFLrW9S9I5wE7bE8AlwOWSpoH7aSRsACTdCTwVOFjSa4CX2949V7wk44iolx5O+rA9CUy2HDuraf8R4LQ57l09n1hJxhFRL5kOHRFRAhWdDp1kHBH1kp5xREQJjFYzrVWz1RERc+hmAaAySjKOiHpJzTgiogTSM46IKIH0jCMiSqCiPeO2f0IkvU3SUe2uiYgoldHR7rcS6dSf/2Ngh6TPS/ptSU8fRKMiIhZsHusZl0mn1uyhsWzcHwPHA7slfVbSGZKeMtdNzQs2j195VQ+bGxHRgdT9ViKdasa2fQD4HPA5SQfR+D6o04HzgFl7ys0LNh+YmlzQcnIREQtTriTbrU7J+Mc+le3HaKzfOSHpkL61KiJioUrW4+1Wp2T8urlO2H64x22JiFi8OiZj2/8zqIZERPREyR7MdSvjjCOiXqrZMU4yjoi6qWY2TjKOiHqpY804IqJykowjIkogD/AiIsogPeOIiOFLmSIiogSSjGOpOWny4mE3IWIWScYREUOXLySNiCiDjKaIiCiB9IwjIkogyTgiogySjCMihi8944iIEqhmLk4yjoiayWiKiIgSSJkiIqIMqpmMq9mfj4iYi9T91vGttFHS7ZKmJZ05y/nlkq4ozu+QtLrp3LuL47dL+pVOsdomY0kHS3qDpF8uXr9e0oWS3irpoI6fJCJi0HqUjCWNAhcBpwDrgNMlrWu5bAvwgO1jgAuAc4t71wGbgWOBjcBfF+83p05lio8W1xwi6QzgMOAfgQ3AeuCMDvdHRAxW7x7grQembe8BkLQN2ATsbrpmE3B2sb8duFCNxTE2Adts/wD4mqTp4v2unytYp2T8PNs/J2kZsBd4pu3HJf0dcPNcN0kaA8aKl2+2Pd4hzpzvs9B7F2MpxV1Kn3WpxV1Kn/XHHLKi66JxS64CGG9q+0rg7qZzM8CJLW/xw2ts75e0DziiOP7FlntXtmtLpz8hI5IOBp4CHAKsKI4vB+YsU9get31CsS3mhzLW+ZK+WEpxl9JnXWpxl9JnXZCWXLXYfLUonXrGlwBfAUaB9wCfkrQHeCGwrc9ti4gYpr3AUU2vVxXHZrtmpqggrADu6/LeH9O2Z2z7AuAlwIts/xXwa8DVwBbb7+/4USIiqmsKWCtpTVEh2AxMtFwzwY+enZ0KXGvbxfHNxWiLNcBa4EvtgnUcZ2z7G037/0ujSD0ow/onw1KKu5Q+61KLu5Q+a88VNeCtNDqgo8CltndJOgfYaXuCRvXg8uIB3f00EjbFdZ+k8bBvP/BW24+3i6dGEo+IiGHKpI+IiBJIMo6IKIHSJuNO0xD7FPNSSd+RdOsg4hUxj5J0naTdknZJevuA4j5J0pck3VzEHdgDWUmjkr4s6dODilnEvVPSLZJukrRzQDEPl7Rd0lck3SbpRQOI+ZziMz6xPSjpHf2OW8T+3eL36VZJn5D0pEHErYNS1oyLaYP/A5xMY7D0FHC67d1tb1x83JcCDwEfs/2z/YzVFPNI4EjbN0p6CnAD8JoBfFYBh9p+qJja/gXg7ba/2OHWXsR+J3AC8FTbr+p3vKa4dwIn2L53gDEvAz5v+yPFE/lDigfhg4o/SmNI1Ym27+pzrJU0fo/W2f5+8QBr0vbf9jNuXZS1Z/zDaYi2H6UxpnlTv4Pa/g8aT0QHxvY3bd9Y7H8XuI0OM3V6FNe2HypeHlRsff/LLGkV8ErgI/2ONWySVgAvpfHEHduPDjIRFzYAd/Q7ETdZBjy5GHN7CPCNDtdHoazJeLZpiH1PUMNWrPj0fGDHgOKNSroJ+A7wr7YHEfcvgT8ADgwgVisDn5N0QzENtt/WAPcAHy3KMh+RdOgA4jbbDHxiEIFs7wXOA74OfBPYZ/tzg4hdB2VNxkuOpMOAfwDeYfvBQcS0/bjt42jMDlovqa+lGUmvAr5j+4Z+xmnjJbZfQGMVrrcWZal+Wga8APiQ7ecD3wMG8vwDGqsuAq8GPjWgeE+j8S/YNcAzgUMl/fogYtdBWZPxvKcSVllRs/0H4OO2/3HQ8Yt/Ol9HY6m/fnox8OqidrsN+KVi0amBKHpu2P4OcCWNclg/zQAzTf/i2E4jOQ/KKcCNtr89oHi/DHzN9j22H6OxwuMvDCh25ZU1GXczDbEWigdplwC32T5/gHGfLunwYv/JNB6WfqWfMW2/2/Yq26tp/EyvtT2QnpOkQ4sHpBSlgpcDfR01Y/tbwN2SnlMc2sCPL7/Yb6czoBJF4evACyUdUvxeb6DxDCS6UMqvXZprGmK/40r6BPAy4CckzQDvs31Jn8O+GPgN4Jaifgvwh7Yn+xz3SOCy4mn7CPBJ2wMdajZgzwCubOQIlgF/b/uzA4j7O8DHi07FHuCNA4j5xB+ck4E3DyIegO0dkrYDN9KYAvxlajI1ehBKObQtImKpKWuZIiJiSUkyjogogSTjiIgSSDKOiCiBJOOIiBJIMo6IKIEk44iIEvg/4dRQO7diVgEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generate parameters and plot variance\n",
    "mean, var = gen_parameters()\n",
    "print(\"True data uncertainty (variance). The white region will not contain any traing data.\")\n",
    "sns.heatmap(var, cmap=\"Reds\", vmax=0.05)\n",
    "plt.savefig(\"heart_data_gt.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for generating train and validation sets\n",
    "def generate_training_data(n_samples, mean, var, num_cat=9, seed=0):\n",
    "    np.random.seed(seed)\n",
    "    train = []\n",
    "    target = []\n",
    "    val = []\n",
    "    val_target = []\n",
    "    for i in range(num_cat):\n",
    "        for j in range(num_cat):\n",
    "            if var[i, j] == 0:\n",
    "                continue\n",
    "            for _ in range(n_samples):\n",
    "                train.append([i, j])\n",
    "                val.append([i, j])\n",
    "                target.append(np.random.normal(mean[i, j], math.sqrt(var[i, j])))\n",
    "                val_target.append(np.random.normal(mean[i, j], math.sqrt(var[i, j])))\n",
    "                \n",
    "    train = np.asarray(train)\n",
    "    target = np.asarray(target)\n",
    "\n",
    "    return train, target, val, val_target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate train and validation datasets\n",
    "train, target, val, val_target = generate_training_data(1000, mean, var)\n",
    "\n",
    "train_pool = Pool(train, target, cat_features = [0, 1])\n",
    "val_pool = Pool(val, val_target, cat_features = [0, 1])\n",
    "\n",
    "# generate test, consisting of all possible feature combinations\n",
    "num_cat = 9\n",
    "test = np.asarray([[i, j] for i in range(num_cat) for j in range(num_cat)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let’s generate an ensemble of SGLB models (for this, we use the option posterior_sampling):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best iteration = 994\n",
      "best iteration = 999\n",
      "best iteration = 995\n",
      "best iteration = 937\n",
      "best iteration = 984\n",
      "best iteration = 974\n",
      "best iteration = 996\n",
      "best iteration = 955\n",
      "best iteration = 996\n",
      "best iteration = 999\n"
     ]
    }
   ],
   "source": [
    "def ensemble(train_pool, val_pool, num_samples=10, iters=1000, lr=0.3):\n",
    "    ens_preds = []\n",
    "    for seed in range(num_samples):\n",
    "        model = CatBoostRegressor(random_seed=seed, iterations=iters, learning_rate=lr, \n",
    "                                  loss_function='RMSEWithUncertainty', posterior_sampling=True,\n",
    "                                  bootstrap_type='No', verbose=False)\n",
    "        model.fit(train_pool, eval_set=val_pool, use_best_model=False)\n",
    "        print(\"best iteration =\", model.get_best_iteration()) # check that best iteration is reasonable\n",
    "        ens_preds.append(model.predict(test))\n",
    "    return ens_preds\n",
    "\n",
    "ens_preds = ensemble(train_pool, val_pool, num_samples=10, iters=1000, lr=0.3)\n",
    "ens_preds = np.asarray(ens_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For regression, data uncertainty is the average variance predicted by the models, while knowledge uncertainty can be obtained by measuring the variance of the mean across multiple models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average predicted data uncertainty:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAD8CAYAAACihcXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAX+0lEQVR4nO3dfbRddZ3f8ffn3oRoBAIrWosJmnTAsWGchUDDjFKWGsGgDGFqWCauqRkXNVplBuuaVqwziMx0lsyiUrtg1CzAYsYSNEAnQAR0QlsfICQ8FQIyDZGRoAPyIBF5DPfTP86OHm7vebj3nnP2Pvt+XmvtxT774Xx/hyTf+7vf/fv9jmwTERHlGim7ARERkWQcEVEJScYRERWQZBwRUQFJxhERFZBkHBFRAUnGEREtSFou6X5JOyWdNcH5OZKuKM5vlbSoOL5I0rOS7iy2L3eKNauLxrwJWAEsKA49DGyyfd9kPlRExDCRNApcBJwA7Aa2Sdpk+96my04HnrR9mKRVwHnA+4tzD9g+stt4bXvGkj4FbAAE3FpsAi6f6KdERESNLAV22t5l+wUauXDFuGtWAJcV+xuBZZI0lWCdesanA0fYfrH5oKQvADuAz090k6S1wFqAL33mzKPX/qv3TKVtU/fS3sHG22dktJy4L75QTtzZ+5UTt4zPO6vjL5H9oZIqiWMvlRJ25F+8Z0qJrNlHdWDX04q/wi8+QpGrCutsryv2FwAPNZ3bDRw77i1+dY3tvZKeAuYX5xZLugPYA/yp7e+2a0unv2FjwOuAfxh3/JDi3ISKD7MOYOz2GzPfOiIqqTlX9dhPgdfbflzS0cD/kHSE7T2tbuiUjD8B/J2k/8uvf0K8HjgMOKMXLY6I6KUe/i7xMHBo0+uFxbGJrtktaRYwD3jcjUV/ngewfZukB4A3AttbBWubjG1fL+mNNGonzQ/wttku5/eYiIg2Zk2tZDuRbcDhkhbTyHurgA+Mu2YTsAa4GVgJbLFtSa8BnrD9kqR/BhwO7Grb7k6tsT0G3DLpjxERUYKRHuXiogZ8BnADMApcanuHpHOB7bY3AZcA6yXtBJ6gkbABjgfOlfQijZLuR20/0S5eSU8lIiL6o5ePPG1vBjaPO3Z20/5zwGkT3HclcOVkYiUZR0StjPSuTDFQScYRUSvDOq04yTgiaqVXNeNBSzKOiFoZTZkiIqJ8KVNERFRAyhQRERWQnnErZS3aMzr4nzNXvPMPBh4T4P1/t76UuKUtUFSCG5afXkrcd3/7a6XE9bO/KCVuL2RoW5WUkIgjohpmDWcurmkyjogZK2WKiIgKGGE4u8ZJxhFRKxlNERFRASlTRERUQHrGEREV0MPF5QcqyTgiaiVlioiICkiZIiKiAjK0LSKiAtIzjoiogNEk44iI8g1rmWLKDx4lfaiXDYmI6IURdb9VyXRGgXyu1QlJayVtl7R93dXfmkaIiIjJGZnEViVtyxSS/k+rU8BrW91nex2wDmBs22ZPuXUREZNUsQ5v1zrVjF8LvBt4ctxxAT/oS4siIqahrovLXwvsb/vO8Sck/c++tCgiYhqqVn7oVttkbLvld83Y/kDvmxMRMT3D2S/O0LaIqBnVtEwRETFUhjMVJxlHRM3UsmYcETFshrRKkWQcEfUyrNOhk4wjolaGMxUnGUdEzVRtzYluJRlHRK1oSPvG/U/GI6N9DzHeTe8uZ0G5PS+NlRL3B+9pOTenr956zZdLiXvdSR8eeMx3nfjGgccEuOnENaXEfceNl5UStxd6mYolLQe+CIwCF9v+/Ljzc4CvAUcDjwPvt/1g0/nXA/cC59g+v12sYR0FEhExoV4toSlpFLgIOAlYAqyWtGTcZacDT9o+DLgAOG/c+S8AXS1dmWQcEbUygrreOlgK7LS9y/YLwAZgxbhrVgD7fo3YCCxTMQVQ0qnAj4Ad3bU7IqJGNJmtae31Ylvb9FYLgIeaXu8ujjHRNbb3Ak8B8yXtD3yKNuu+j5cHeBFRK5OZ9NG89nqPnQNcYPvpbtfKSDKOiFrp4QO8h4FDm14vLI5NdM1uSbOAeTQe5B0LrJT0V8BBwJik52xf2CpYknFE1EoPh7ZtAw6XtJhG0l0FjF86eBOwBrgZWAlssW3gX/6qPdI5wNPtEjEkGUdEzYz2KBfb3ivpDOAGGkPbLrW9Q9K5wHbbm4BLgPWSdgJP0EjYU5JkHBG10stxxrY3A5vHHTu7af854LQO73FON7GSjCOiVjIDLyKiArKEZkREBQzr5ImO7Zb0JknLikHMzceX969ZERFTM5lJH1XSNhlL+mPgb4E/Au6R1DwV8C/72bCIiKkYkbreqqRTz/jDwNG2TwXeDvyZpDOLcy0/SfMUw3VXbW51WUREzw1rz7hTzXjE9tMAth+U9HZgo6Q30OazNE8xHLvtBveorRERHXU7/bhqOvWMH5F05L4XRWI+GXg18OZ+NiwiYip6tYTmoHXqGX8Q2Nt8oFiZ6IOSvtK3VkVETJGqlmW71DYZ297d5tz3e9+ciIjpGRnSsW0ZZxwRtTKsNeMk44iolSHNxUnGEVEv6RlHRFTAkObiJOOIqJeqzazrVpJxRNTKSB2HtkVEDBtlaFtERPnyAK+F6078w36H+P8c+xsHDzwmwNs/c0Ypccty03s/Ukrck+/4zsBj6uDXDjwmwDt+fF8pca86bmUpcVc++ci032NIc3F6xhFRL+kZR0RUwJDm4iTjiKiX0YymiIgoX8oUEREVMKS5OMk4IuolyTgiogJqubh8RMSwyQO8iIgKSJkiIqICMpoiIqIChjQXd07GkpYCtr1N0hJgOfBD25v73rqIiEmqZc9Y0meBk4BZkr4NHAvcBJwl6S22/9MA2hgR0bUhzcUde8YrgSOBOcA/Agtt75F0PrAVmDAZS1oLrAX42NwDWf6Kub1rcUREGyOjw5mNOyXjvbZfAp6R9IDtPQC2n5U01uom2+uAdQDXzD/EPWttREQHw1qm6LQm/guS9nVrj953UNI8oGUyjogozYi63zqQtFzS/ZJ2SjprgvNzJF1RnN8qaVFxfKmkO4vtLkm/37HZHc4fb/sZANvNyXc2sKbjJ4mIGDSp+63t22gUuIjGc7MlwOpiEEOz04EnbR8GXACcVxy/BzjG9pE0Bj18RVLbSkTbZGz7+RbHH7N9d9tPEhFRAkldbx0sBXba3mX7BWADsGLcNSuAy4r9jcAySbL9jO29xfFXAB3LtUP61X0RES2MjnS9SVoraXvTtrbpnRYADzW93l0cY6JriuT7FDAfQNKxknYAdwMfbUrOE8qkj4iolcksFNQ82KDXbG8FjpD0z4HLJH3L9nOtrk/POCLqpUc1Y+Bh4NCm1wuLYxNeU9SE5wGPN19g+z7gaeC32gVLMo6IWtGIut462AYcLmmxpP2AVcCmcdds4teDGVYCW2y7uGcWgKQ3AG8CHmwXLGWKiKiXHo0ztr1X0hnADcAocKntHZLOBbbb3gRcAqyXtBN4gkbCBjiOxkzlF2kMA/6Y7cfaxUsyjoh66eF6xsUaPJvHHTu7af854LQJ7lsPrJ9MrCTjiKgVjQ5n9TXJOCLqZUinQ8vu79IRY7deO/C1KX7wex8ddEgA/ubRPaXEjf676DsXlRL3u6s/XUrc478zqd+we2bkt98x7Uz6zPve1nXOmXvl9yuTudMzjoh6GdKecZJxRNRKvh06IqIK0jOOiChfRlNERFRByhQRERWQMkVERPmG9WuXkowjol5SpoiIKF8e4EVEVEHKFBER5RvWSR+T7s9L+lo/GhIR0RO9+6aPgWrbM5Y0flV7Ae+QdBCA7VP61bCIiCmpac94IbAH+ALwn4vtF037E2r+xtV1V1/fq7ZGRHQkqeutSjrVjI8BzgQ+A/x723dKetb2/2p3U/M3rpaxhGZEzGB1HE1hewy4QNI3i/8+0umeiIhSVazH262uEqvt3cBpkt5Lo2wREVFNdU7G+9i+DriuT22JiJi+kRqWKSIihs5M6BlHRFReknFERAWMjpbdgilJMo6IeknPOCKiApKMIyIqIMk4IqICMrQtIqICkoxbGBv80hRvvXbdwGMCcPLaUsIeddwbSol77ZadpcTd8vNnBx7zypV/MvCYACu3fL2UuDz3y3Li9kLKFBER5VN6xhERFZCecUREBSQZR0RUQJJxREQFDOl06OGsdEdEtNLDLySVtFzS/ZJ2SjprgvNzJF1RnN8qaVFx/ARJt0m6u/jvOzvFSjKOiHrpUTKWNApcBJwELAFWS1oy7rLTgSdtHwZcAJxXHH8M+D3bbwbWAOs7NTvJOCLqZWSk+629pcBO27tsvwBsAFaMu2YFcFmxvxFYJkm277D9k+L4DuCVkua0bfakPmRERNVNomfc/E32xdY8c2sB8FDT693FMSa6xvZe4Clg/rhr3gfcbvv5ds3OA7yIqJdJjKZo/ib7/jRFR9AoXZzY6dok44iol96NpngYOLTp9cLi2ETX7JY0C5gHPA4gaSFwNfBB2w90CjapMoWk4yR9UlLHLB8RUYrejabYBhwuabGk/YBVwKZx12yi8YAOYCWwxbYlHUTjy5vPsv39bprdNhlLurVp/8PAhcABwGcnGuYREVG6HiXjogZ8BnADcB/wDds7JJ0r6ZTiskuA+ZJ2Ap8E9uXFM4DDgLMl3Vls/6RdvE5litlN+2uBE2z/TNL5wC3A5ye6qSiCrwX40qc+xtpTl3cIExHRIz1cKMj2ZmDzuGNnN+0/B5w2wX1/AfzFZGJ1SsYjkg6m0YOW7Z8VgX4paW+rm5qL4mO3XDP4NTQjYuaq6XToecBtgABLOsT2TyXtXxyLiKiWkeGcDt02Gdte1OLUGPD7PW9NRMR0jQxnP3FKQ9tsPwP8qMdtiYiYPg3nXLaMM46IeqlpzTgiYrjka5ciIiogPeOIiAqo42iKiIihkzJFREQFpEwREVEBGdoWEVEBM2nSx6SU8T9m1n6Djwm89ZovlxL31lP+bSlxj114UClx/+l+sztf1GPHXdu39cfbG3upnLgl/RvqiTzAi4iogJQpIiIqIGWKiIgKyGiKiIgKSJkiIqICUqaIiKiAjKaIiKiAlCkiIiogZYqIiApIzzgiogIytC0iogKyhGZERAUM6WiKtj9CJB0r6cBi/5WSPifpGknnSZo3mCZGREyC1P1WIZ3685cCzxT7XwTmAecVx77a6iZJayVtl7R93dXX96ShERFdGRnpfquQTmWKEdt7i/1jbB9V7H9P0p2tbrK9DlgHMHbrtZ5+MyMiulSxHm+3Ov1ouEfSh4r9uyQdAyDpjcCLfW1ZRMRUaKT7rUI69Yz/DfBFSX8KPAbcLOkh4KHiXEREtQzpA7y2ydj2U8AfFg/xFhfX77b9yCAaFxExaXWegWd7D3BXn9sSETF9FSs/dCvjjCOiXob0AV6ScUTUy5D2jIez1RERLUjqeuvivZZLul/STklnTXB+jqQrivNbJS0qjs+XdJOkpyVd2E27k4wjol5GZnW/tSFpFLgIOAlYAqyWtGTcZacDT9o+DLiAxqQ4gOeAPwP+pOtmd3thRMRQGFH3W3tLgZ22d9l+AdgArBh3zQrgsmJ/I7BMkmz/0vb3aCTl7prd7YUREUNhEpM+mpduKLa1Te+0gMacin12F8eY6JpitvJTwPypNDsP8CKiXiYxmqJ56YayJRlHRL30bjTFw8ChTa8XFscmuma3pFk0FlN7fCrB+p+My5ia+Pyzg48JMFrONMyl111cStzbTv5wKXGP+9svDTymn/75wGMC6ICDS4nLyOxy4vZC78YZbwMOl7SYRtJdBXxg3DWbgDXAzcBKYIvtKS2Olp5xRNRLjzpFtvdKOgO4ARgFLrW9Q9K5wHbbm4BLgPWSdgJP0EjYAEh6EDgQ2E/SqcCJtu9tFS/JOCLqpYeTPmxvBjaPO3Z20/5zwGkt7l00mVhJxhFRL5kOHRFRAUM6HTrJOCLqJT3jiIgKGB3OtDacrY6IaKGbBYCqKMk4IuolNeOIiApIzzgiogLSM46IqIAh7Rm3/REi6Y8lHdrumoiIShkd7X6rkE79+T8Htkr6rqSPSXrNIBoVETFlk1jPuEo6tWYXjWXj/hw4GrhX0vWS1kg6oNVNzQs2r7vqWz1sbkREB1L3W4V0qhnb9hhwI3CjpNk0vg9qNXA+MGFPuXnB5rHt35rScnIREVNTrSTbrU7J+GWfyvaLNNbv3CRpbt9aFRExVRXr8XarUzJ+f6sTtp/pcVsiIqavjsnY9t8PqiERET1RsQdz3co444iol+HsGCcZR0TdDGc2TjKOiHqpY804ImLoJBlHRFRAHuBFRFRBesYREeVLmSIiogKSjFtwCUtTzN5v8DEBxl4qJ25Jjrr8r8oJPDL4f2ya9+qBxwTg+WfLiTvUkowjIkqXLySNiKiCjKaIiKiA9IwjIiogyTgiogqSjCMiypeecUREBQxnLk4yjoiayWiKiIgKSJkiIqIKhjMZD2d/PiKiFan7reNbabmk+yXtlHTWBOfnSLqiOL9V0qKmc58ujt8v6d2dYrVNxpL2k/RBSe8qXn9A0oWSPi5pdsdPEhExaD1KxpJGgYuAk4AlwGpJS8ZddjrwpO3DgAuA84p7lwCrgCOA5cBfF+/XUqcyxVeLa+ZKWgPsD1wFLAOWAms63B8RMVi9e4C3FNhpexeApA3ACuDepmtWAOcU+xuBC9VYHGMFsMH288CPJO0s3u/mVsE6JeM32/5tSbOAh4HX2X5J0t8Ad7W6SdJaYG3x8iO213WI0/J9pnrvdMykuDPps860uDPps77M3HldF43H5SqAdU1tXwA81HRuN3DsuLf41TW290p6CphfHL9l3L0L2rWl04+QEUn7AQcAc4F5xfE5QMsyhe11to8ptun8oaztfElfzKS4M+mzzrS4M+mzTsm4XDXdfDUtnXrGlwA/BEaBzwDflLQL+B1gQ5/bFhFRpoeBQ5teLyyOTXTN7qKCMA94vMt7X6Ztz9j2BcBxwO/a/q/A+4AbgNNtf67jR4mIGF7bgMMlLS4qBKuATeOu2cSvn52tBLbYdnF8VTHaYjFwOHBru2Adxxnb/knT/s9pFKkHpaxfGWZS3Jn0WWda3Jn0WXuuqAGfQaMDOgpcanuHpHOB7bY30agerC8e0D1BI2FTXPcNGg/79gIft932q4DkMr4WKSIiXiaTPiIiKiDJOCKiAiqbjDtNQ+xTzEslPSrpnkHEK2IeKukmSfdK2iHpzAHFfYWkWyXdVcQd2ANZSaOS7pB07aBiFnEflHS3pDslbR9QzIMkbZT0Q0n3SfrdAcT8zeIz7tv2SPpEv+MWsf9d8ffpHkmXS3rFIOLWQSVrxsW0wb8HTqAxWHobsNr2vW1vnH7c44Gnga/Z/q1+xmqKeQhwiO3bJR0A3AacOoDPKuBVtp8uprZ/DzjT9i0dbu1F7E8CxwAH2j653/Ga4j4IHGP7sQHGvAz4ru2Liyfyc4sH4YOKP0pjSNWxtv+hz7EW0Ph7tMT2s8UDrM22/1s/49ZFVXvGv5qGaPsFGmOaV/Q7qO3/TeOJ6MDY/qnt24v9XwD30WGmTo/i2vbTxcvZxdb3n8ySFgLvBS7ud6yySZoHHE/jiTu2XxhkIi4sAx7odyJuMgt4ZTHmdi7wkw7XR6GqyXiiaYh9T1BlK1Z8eguwdUDxRiXdCTwKfNv2IOL+F+A/AGMDiDWegRsl3VZMg+23xcDPgK8WZZmLJb1qAHGbrQIuH0Qg2w8D5wM/Bn4KPGX7xkHEroOqJuMZR9L+wJXAJ2zvGURM2y/ZPpLG7KClkvpampF0MvCo7dv6GaeN42wfRWMVro8XZal+mgUcBXzJ9luAXwIDef4BjVUXgVOAbw4o3sE0foNdDLwOeJWkPxhE7DqoajKe9FTCYVbUbK8Evm77qkHHL351vonGUn/99DbglKJ2uwF4Z7Ho1EAUPTdsPwpcTaMc1k+7gd1Nv3FspJGcB+Uk4Hbbjwwo3ruAH9n+me0Xaazw+NYBxR56VU3G3UxDrIXiQdolwH22vzDAuK+RdFCx/0oaD0t/2M+Ytj9te6HtRTT+TLfYHkjPSdKrigekFKWCE4G+jpqx/Y/AQ5J+szi0jJcvv9hvqxlQiaLwY+B3JM0t/l4vo/EMJLpQya9dajUNsd9xJV0OvB14taTdwGdtX9LnsG8D/jVwd1G/BfiPtjf3Oe4hwGXF0/YR4Bu2BzrUbMBeC1zdyBHMAv677esHEPePgK8XnYpdwIcGEHPfD5wTgI8MIh6A7a2SNgK305gCfAc1mRo9CJUc2hYRMdNUtUwRETGjJBlHRFRAknFERAUkGUdEVECScUREBSQZR0RUQJJxREQF/D89rX2GNGPTcAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knowledge uncertainty via ensemble:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD8CAYAAABekO4JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAafElEQVR4nO3dfZRdVZ3m8e9TVQR5jS5sGEwYSQ/xBegZeengjIxLTYNhtUOwhUWQEVTaagfT6rh6ZtLTA45o9xrWcsZXVkuESGBE0CjLmjYSdMCxddqQ8CYkgFNE7FSgQV46CBEh8Mwf9wRvKlX33Erde+vUuc/HdVadu+8+57fPovxl1z777CPbREREdQ3MdAMiIqK1JOqIiIpLoo6IqLgk6oiIikuijoiouCTqiIiKS6KOiKi4obIKkl4HLAXmFUXbgBHb93azYRER0dCyRy3pPwHXAQJuLTYBX5O0ovvNi4gItXoyUdLPgGNsPz+ufA6wyfbCSY4bBoYBLv/CZ08Yfv97O9bgiKix/edquqf4oA5u+3HrL/mpacfrhbKhjxeBVwG/GFd+ePHdhGyvBFYCsGN7nlGPiJiGskT9UeB/S/p/wNai7J8CRwHLu9mwiIi9UccZEi0Tte0bJb0GWMTuNxM32H6h242LiJiqIc2K0YwpKZ31YftF4Cc9aEtExLQNdDBPS1oCfA4YBK6w/d/Gfb8vcDVwAvA4cLbtByUtYtfwb2MCxn+1fUNxzIPAr4AXgJ22TyxrR2mijoiYTTo19CFpELgMOAUYAzZIGrG9uanaBcCTto+StAy4FDgbuAc40fZOSYcDd0n6X7Z3Fse91fZj7baljsM5EdHHBqS2txKLgFHbW2w/R2Oq8tJxdZYCq4v9NcBiSbK9oykpvwyY1qSKJOqIqJWBKWyShiVtbNqGm041j99OooBGr3oeu3upTpGYtwOH0Dj3SZI2AXcDH2xK3AZuknTbuHiTytBHRNTKVMaod5tK3GG21wPHSHo9sFrSd20/C5xse5ukQ4HvSbrP9g9bnSs96oiolUGp7a3ENuCIps/zi7IJ60gaAubSuKn4kmK5jaeBY4vP24qfjwI30BhiaSmJOiJqZSpDHyU2AAslLSiexl4GjIyrMwKcX+yfCdxs28UxQwCSXg28DnhQ0gGSDirKDwBOpXHjsaUMfURErXRqel4xY2M5sI7G9LxVtjdJugTYaHsEuBK4RtIo8ASNZA5wMrBC0vM0nuK+0PZjkn4XuEGN3vwQcK3tG8va0nKtj47II+QR0a4OrPXxiX1f0XbO+fhvnpwVT8ekRx0RtdLGtLtZJ4k6ImplqH55Ook6IuqljjMkkqgjolYGqF+XOok6Imqlk4syVUUSdUTUSoY+IiIqLj3qiIiK68sXB0REzCYZ+oiIqLgMfUREVFym50VEVFx61BERFTeYRB0RUW11HPrY6xukkt7XyYZERHTCgNrfZovpzGT5xGRfNL8wcuWqq6YRIiJiajr4hpfKaDn0Iemnk30FHDbZcbu9MDIvDoiIHppFHeW2lY1RHwa8HXhyXLmA/9uVFkVETEM/vjjgb4ADbd85/gtJP+hKiyIipmE2DWm0q2Witn1Bi+/e3fnmRERMT/3605meFxE1oxoOfdTxr4SI6GOawlZ6LmmJpPsljUpaMcH3+0q6vvh+vaQji/JFku4strskvbPdc04kiToiaqVT0/MkDQKXAacBRwPnSDp6XLULgCdtHwV8Bri0KL8HONH2G4AlwOWShto854TXFBFRG1L7W4lFwKjtLbafA64Dlo6rsxRYXeyvARZLku0dtncW5S8Ddk1Tbuece0iijohaGUBtb80P5xXbcNOp5gFbmz6PFWVMVKdIzNuBQwAknSRpE3A38MHi+3bOuYfcTIyIWpnKrcTdHs7rMNvrgWMkvR5YLem7e3uu9KgjolY6uNbHNuCIps/zi7IJ60gaAuYCjzdXsH0v8DRwbJvn3POaSpsaETGLaAr/K7EBWChpgaQ5wDJgZFydEeD8Yv9M4GbbLo4ZApD0auB1wINtnnMPGfrooBe+ffmMxB1c+iczEref3PrPfm9G4i564O4ZiTubdWoWte2dkpYD64BBYJXtTZIuATbaHgGuBK6RNAo8QSPxApwMrJD0PPAicKHtxwAmOmdZW5KoI6JWOrl8qe21wNpxZRc37T8LnDXBcdcA17R7zjJJ1BFRK3V8cUASdUTUSv3SdBJ1RNRMDZf6SKKOiHqpYZ5Ooo6Iemlj2t2sk0QdEbUyWL88nUQdEfVSwzydRB0R9ZKhj4iIisusj4iIiqvjAkal1yTpdZIWSzpwXPmS7jUrImLvdPJVXFXRMlFL+jDwbeBPgXskNb+J4K+62bCIiL0xILW9zRZlPeoPACfYPgN4C3CRpI8U3016lc1vTVi56qqONDQioh117FGXjVEP2H4awPaDkt4CrCnWV530Ond7a8KO7Z6sXkREp2kW9ZTbVdajfkTSG3Z9KJL2O4BXAjOzQG9ERAsdfMNLZZT1qM8DdjYXFC9oPE/SzKySHxHRgmZTBm5Ty0Rte6zFdz/ufHMiIqZnoIbz8zKPOiJqpY5j1EnUEVErNczTSdQRUS/pUUdEVFwN83QtH4uPiD7WyScTJS2RdL+kUUkrJvh+X0nXF9+vl3RkUX6KpNsk3V38fFvTMT8oznlnsR1a1o70qCOiVgY6ND1P0iBwGXAKMAZskDRie3NTtQuAJ20fJWkZcClwNvAY8G9sPyTpWGAdMK/puHNtb2y3LelRR0StaKD9rcQiYNT2FtvPAdcBS8fVWQqsLvbXAIslyfYdth8qyjcB+0nad2+vKYk6ImpFUttbiXnA1qbPY+zeK96tTvEw4HbgkHF13gXcbvs3TWVfKYY9LlIbDanl0McLV83Qwn5z5sxI2Beu/fSMxB1895/NSNxvzHtNz2Mef/jBPY8J8K35r52RuH80dv+MxO2EqdxMlDQMDDcVrSzWKupQW3QMjeGQU5uKz7W9TdJBwDeB9wBXtzpPLRN1RPSvqUzP220BuT1tA45o+jy/KJuozpikIWAu8HjRjvnADcB5th9oirmt+PkrSdfSGGJpmagz9BERtSK1v5XYACyUtEDSHGAZMDKuzghwfrF/JnCzbUt6OfAdYEXzchuShiS9stjfh8Yid/eUNSQ96oiolcEOzfqwvVPSchozNgaBVbY3SboE2Gh7BLgSuEbSKPAEjWQOsBw4CrhY0sVF2anAM8C6IkkPAt8HvlzWliTqiKiVTj6ZaHstsHZc2cVN+88CZ01w3KeAT01y2hOm2o4k6oiolTo+mZhEHRG1kkQdEVFxfffigIiI2aZTNxOrJIk6ImolQx8RERWX9agjIiquhnm6PFFLWgTY9gZJRwNLgPuK+YUREZXSdz1qSR8HTgOGJH0POAm4BVgh6Tjbf9mDNkZEtK2Gebp0rY8zgTcBbwY+BJxh+5PA22ksjj0hScOSNkrauHLVVZ1qa0REqYFBtb3NFmVDHzttvwDskPSA7acAbP9a0ouTHbTbilQ7trtTjY2IKNN3Qx/Ac5L2t72DpufTJc0FJk3UEREzpg/nUb9511sJbDcn5n347dJ+ERHV0W896nGvjmkuf4zGyxsjIiqlH4c+IiJml8H6vQ8liToiaiWLMkVEVF2GPiIiqi096oiIqkuPOiKi4tKjjoioNtVw1kf9rigi+pvU/lZ6Ki2RdL+kUUkrJvh+X0nXF9+vl3RkUX6KpNsk3V38fFvTMScU5aOSPq82Jn7L7vJSHH201seW3//9mW5CX9hnqPf9i7/86UM9jwnwpWe2zkjcGbP/3GmPW+x415vazjn7f/PHk8aTNAj8DDgFGAM2AOfY3txU50Lgn9v+oKRlwDttny3pOOAR2w9JOhZYZ3teccytwIeB9cBa4PO2v9uqnelRR0S9dK5HvQgYtb3F9nPAdcDScXWWAquL/TXAYkmyfYftXf+6bwL2K3rfhwMH2/6JG73kq4EzyhqSRB0RtaIBtb2VmAc0/0kzVpRNWMf2TmA7cMi4Ou8Cbi+W5JhXnKfVOfeQm4kRUS9TmJ4naRgYbipaWSzT3KGm6BjgUuDU6ZwniToiamUqsz52Wzt/T9uAI5o+zy/KJqozJmkImAs8DiBpPnADcJ7tB5rqzy855x4y9BER9TKg9rfWNgALJS2QNAdYBoyMqzPCb5d8PhO42bYlvRz4DrDC9o93Vbb9MPCUpDcWsz3OA75dekntXHdExKzRoZuJxZjzcmAdcC/wddubJF0i6fSi2pXAIZJGgY8Bu6bwLQeOAi6WdGexHVp8dyFwBTAKPAC0nPEBGfqIiJrp5HrUttfSmELXXHZx0/6zwFkTHPcp4FOTnHMjcOxU2pFEHRH1kkfIIyKqrY6PkCdRR0S9ZPW8iIhqq+N61FP+G0HS1d1oSERER3RwUaaqaNmjljR+zqCAtxZzBLF9+p5HRUTMoBr2qMuGPuYDm2nM+TONRH0i8N9bHdT8WOblX/gsw+9/77QbGhHRjk5Oz6uKskR9IvAR4C+A/2D7Tkm/tv1/Wh2022OZfbTMaURUQL/N+rD9IvAZSd8ofj5SdkxExIzqwx41ALbHgLMk/SHwVHebFBExDf2aqHex/R0aC41ERFTTQJ8NfUREzDr93qOOiKi8JOqIiIobHJzpFnRcEnVE1Et61BERFZdEHRFRcUnUEREVl+l5EREVl0Qdrfzuhg0zEtdP/sOMxH349DNmJO6hbzm65zG/9Hfrex4z9lKGPiIiqk017FHX74oior918MUBkpZIul/SqKQVE3y/r6Tri+/XSzqyKD9E0i2Snpb0xXHH/KA4553FdmhZO9Kjjoh66dDQh6RB4DLgFGAM2CBpxPbmpmoXAE/aPkrSMuBS4GzgWeAi4NhiG+9c2xvbbUt61BFRL53rUS8CRm1vsf0ccB2wdFydpcDqYn8NsFiSbD9j+0c0Eva0JVFHRL0MDra/tTYP2Nr0eawom7CO7Z3AduCQNlr5lWLY4yK18UqaJOqIqJcp9KglDUva2LQN96CF59r+PeBfF9t7yg7IGHVE1MsUxqh3e23gnrYBRzR9nl+UTVRnTNIQMBd4vCTmtuLnryRdS2OI5epWx6RHHRH1MjDQ/tbaBmChpAWS5gDLgJFxdUaA84v9M4GbbU/6nlhJQ5JeWezvA7wDuKesIelRR0S9dGjWh+2dkpYD64BBYJXtTZIuATbaHgGuBK6RNAo8QSOZF83Qg8DBwBxJZwCnAr8A1hVJehD4PvDlsrYkUUdEvXTwyUTba4G148oubtp/FjhrkmOPnOS0J0y1HUnUEVEv/f7iAEkn0xj4vsf2Td1pUkTENNRwrY+Wo+mSbm3a/wDwReAg4OMTPU4ZETHjOvgIeVWU3fbcp2l/GDjF9idoDIqfO9lBzXMTV666avqtjIhoV+dmfVRG2dDHgKRX0Ejosv1LANvPSNo52UG7zU3csX3SqSoRER03i3rK7SpL1HOB2wABlnS47YclHViURURUy0Cf3UxsMb3kReCdHW9NRMR0DdSvD7lX0/Ns7wB+3uG2RERMn2bP2HO7Mo86IuqlD8eoIyJml1k0m6NdSdQRUS/pUUdEVFy/zfqIiJh1MvQREVFxGfqIiKi4TM+LiKi4PPASVaRX/JMZiXvYx86bkbiD77xwRuLGLJGbiRERFZehj4iIisvQR0RExWXWR0RExWXoIyKi4mo49FG/f3oior8NDLa/lZC0RNL9kkYnek+spH0lXV98v17SkUX5IZJukfS0pC+OO+YESXcXx3xeKh+rSaKOiHrRQPtbq9NIg8BlwGnA0cA5ko4eV+0C4EnbRwGfAS4typ8FLgL+bIJT/zXwAWBhsS0pu6Qk6oiolwG1v7W2CBi1vcX2c8B1wNJxdZYCq4v9NcBiSbL9jO0f0UjYL5F0OHCw7Z/YNnA1cEbpJZVedETEbDKFHrWkYUkbm7bhpjPNA7Y2fR4rypioju2dwHbgkBatm1ecp9U595CbiRFRL1OYnmd7JbCye43pjCTqiKiXzi1zug04ounz/KJsojpjkoaAucDjJeecX3LOPWToIyLqpXOzPjYACyUtkDQHWAaMjKszApxf7J8J3FyMPU/I9sPAU5LeWMz2OA/4dllDWvaoJZ0E3Gv7KUn7ASuA44HNwF/Z3l4WICKipzr0ZKLtnZKWA+uAQWCV7U2SLgE22h4BrgSukTQKPEEjmRfN0IPAwcAcSWcAp9reDFwIXAXsB3y32FoqG/pYBfyLYv9zwA4a008WA18B/miig4oB+WGAy7/wWYbf/96ydkREdEYH3/Biey2wdlzZxU37zwJnTXLskZOUbwSOnUo7yhL1QHEnE+BE28cX+z+SdOdkB+02QL9j+6R/BkREdFwN1/oo+6fnHknvK/bvknQigKTXAM93tWUREXujQw+8VElZj/qPgc9J+i/AY8DfSdpKY97gH3e7cRERU9ZvLw4obha+V9LBwIKi/pjtR3rRuIiIKavhokxtzaO2/RRwV5fbEhExfbNoSKNdeeAlIuqlhjcTk6gjol7So46IqLY2lneedZKoI6JeBuqX1up3RRHR3/p11kdExKyRMeqIiIrLGHVERMWlRx0tTb4MbXfNUA9i8Ix/NyNxZ8QLO8vrdMNg/i86ZelRR0RU3GCfrfURETHrZOgjIqLiMvQREVFx6VFHRFRcetQRERVXw5ky9buiiOhrdVyUqX6DORHR3zr4zkRJSyTdL2lU0ooJvt9X0vXF9+slHdn03Z8X5fdLentT+YOS7pZ0p6SN7VxSetQRUS8d6lFLGgQuA04BxoANkkZsb26qdgHwpO2jJC0DLgXOlnQ0sAw4BngV8H1Jr7H9QnHcW20/1m5b0qOOiHrpXI96ETBqe4vt54DrgKXj6iwFVhf7a4DFaoy9LAWus/0b2z8HRovz7ZUk6oioF6ntTdKwpI1N23DTmeYBW5s+jxVlTFTH9k5gO3BIybEGbpJ027h4k2o59CHpw8ANtre2qhcRURlTeITc9kpgZfcaM6GTbW+TdCjwPUn32f5hqwPKetSfBNZL+ltJF0r6nY41NSKiGzo39LENOKLp8/yibMI6koaAucDjrY61vevno8ANtDEkUtbSLUWATwInAJsl3SjpfEkHTXZQ858TK1ddVdaGiIjOmcLQR4kNwEJJCyTNoXFzcGRcnRHg/GL/TOBm2y7KlxWzQhYAC4FbJR2wK3dKOgA4FbinrCFlsz5s+0XgJhpjKvsApwHnAJ8GJuxh7/bnxI7tM7T2Z0T0p87M+rC9U9JyYB0wCKyyvUnSJcBG2yPAlcA1kkaBJ2gkc4p6Xwc2AzuBD9l+QdJhwA3FXO8h4FrbN5ZekVusoSzpDtvHTfLd/rZ3lF5tPyXqPluPuq+uN+tR98b+c6f9H9dbN7f9i6kjjp4VT8eU/RacPdkXbSXpiIheq+GTiS0Tte2f9aohEREdkdXzIiIqrn4d6iTqiKib+mXqJOqIqJd+G6OOiJh1kqgjIiouNxMjIqouPeqIiGrL0EdERMUlUUdLNfwFaamfrnemHuXup8f0O2Y2t31iSdQRUSt1fLltEnVE1EtmfUREVFx61BERFZdEHRFRdUnUERHVlh51RETF1S9PJ1FHRM1k1kdERMVl6CMiouqSqCMiqq3fetSS5gDLgIdsf1/Su4F/BdwLrLT9fA/aGBHRvhomarnFoi+Svkojme8P/CNwIPAtYHFx7PmlEXZsn6FVZSJqoN8WZdp/7vQDTyXndCJeL9iedAN+WvwcAh4BBovP2vXdJMcNAxuLbbhVjJL4e33sdLZ+ittP19pvcfvpWuu+lfWo7wGOBw4A/h54te0nJL0MuMP26zvzz8Wk8TfaPrGbMfo9bj9da7/F7adrrbuym4lXAvcBg8BfAN+QtAV4I3Bdl9sWERGUJGrbn5F0fbH/kKSrgT8Avmz71l40MCKi35VOz7P9UNP+PwJrutqi3a3sYax+jdtP19pvcfvpWmut5Rh1RETMvPo9FB8RUTNJ1BERFVfZRC1piaT7JY1KWtGjmKskPVpMS+wJSUdIukXSZkmbJH2kR3FfJulWSXcVcT/Ri7hF7EFJd0j6m17FLOI+KOluSXdK2tijmC+XtEbSfZLulfQvexDztcU17tqekvTRbsctYv/74vfpHklfK6byxjRVcoxa0iDwM+AUYAzYAJxje3OX474ZeBq42vax3YzVFPNw4HDbt0s6CLgNOKMH1yrgANtPS9oH+BHwEds/6WbcIvbHgBOBg22/o9vxmuI+CJxo+7EexlwN/K3tK4olGfYvbsr3Kv4gsA04yfYvuhxrHo3fo6Nt/1rS14G1tq/qZtx+UNUe9SJg1PYW28/RmLO9tNtBbf8QeKLbccbFfNj27cX+r2isozKvB3Ft++ni4z7F1vV/tSXNB/4QuKLbsWaapLnAm2k8j4Dt53qZpAuLgQe6naSbDAH7Sdq19MRDJfWjDVVN1POArU2fx+hB8pppko4EjgPW9yjeoKQ7gUeB79nuRdzPAv8ReLEHscYzcJOk2yQN9yDeAuCXwFeKoZ4rJB3Qg7jNlgFf60Ug29uAT9N4ivlhYLvtm3oRu+6qmqj7jqQDgW8CH7X9VC9i2n7B9huA+cAiSV0d7pH0DuBR27d1M04LJ9s+HjgN+FAx1NVNQzSWYPhr28cBzwA9ud8CL61+eTrwjR7FewWNv3wXAK8CDpD0b3sRu+6qmqi3AUc0fZ5flNVSMUb8TeCrtr/V6/jFn+O3AEu6HOpNwOnFWPF1wNsk/c8ux3xJ0ePD9qPADTSG2LppDBhr+ktlDY3E3SunAbfbfqRH8f4A+LntX7qxBPK3aCyLHNNU1US9AVgoaUHTmtgjM9ymrihu6l0J3Gv7f/Qw7u9Ienmxvx+NG7f3dTOm7T+3Pd/2kTT+m95suyc9LkkHFDdrKYYfTgW6OrvH9j8AWyW9tihaDHT1JvE459CjYY/C3wNvlLR/8Xu9mMY9l5imSr7hxfZOScuBdTQWhFple1O340r6GvAW4JWSxoCP276yy2HfBLwHuLsYLwb4z7bXdjnu4cDqYlbAAPB12z2dLtdjhwE3NPIHQ8C1tm/sQdw/Bb5adDi2AO/rQcxd/xidAvxJL+IB2F4vaQ1wO7ATuIM8Tt4RlZyeFxERv1XVoY+IiCgkUUdEVFwSdURExSVRR0RUXBJ1RETFJVFHRFRcEnVERMX9f6Jt2rY9lKn4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = np.mean(ens_preds, axis=0)[:, 1] # average estimated data uncertainty\n",
    "knowledge = np.var(ens_preds, axis=0)[:, 0] # estimated knowledge uncertainty\n",
    "\n",
    "print(\"Average predicted data uncertainty:\")            \n",
    "sns.heatmap(data.reshape([num_cat, num_cat]), cmap=\"Reds\", vmin=0, vmax=0.05)\n",
    "plt.savefig(\"heart_data_sglb.pdf\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Knowledge uncertainty via ensemble:\")            \n",
    "sns.heatmap(knowledge.reshape([num_cat, num_cat]), cmap=\"Reds\")\n",
    "plt.savefig(\"heart_knowledge_sglb.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us estimate data and knowledge uncertainty with a virtual ensemble. Here we can also use prediction_type='TotalUncertainty': in this case, CatBoost calculates and returns the following statistics using the virtual ensemble: Mean Predictions, Knowledge Uncertainty, Data Uncertainty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CatBoostRegressor(iterations=1000, learning_rate=0.3, loss_function='RMSEWithUncertainty', \n",
    "                          posterior_sampling=True, bootstrap_type='No', random_seed=0, verbose=False)\n",
    "model.fit(train_pool, eval_set=val_pool, use_best_model=False)\n",
    "preds = model.virtual_ensembles_predict(test, prediction_type='TotalUncertainty', virtual_ensembles_count=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average predicted data uncertainty:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAD8CAYAAACihcXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAYC0lEQVR4nO3df7RdZX3n8ffn3pBA+BEsdVyYYJMpQSdUByUGHa1LG9FQGKLLMCbMjIyTenVVrI7zo3RsEWk7S1xUxllQy61gkan8MEq9ajC0gjPawZDwayD8aEOkkkhRIANGCPn1mT/OBo/Xe885995zzt5n5/Naay/22fvZ5/vslfA9T579PM+WbSIiolxDZVcgIiKSjCMiKiHJOCKiApKMIyIqIMk4IqICkowjIiogyTgiYhKSVkh6UNJWSedNcH6OpOuK8xslLSyOL5T0rKS7iu3P2sWa1UFlXgGsBOYXh3YAY7bvn8pNRUQMEknDwGXAqcB2YJOkMdv3NRVbC+y0fbyk1cBFwLuLcw/ZPqnTeC1bxpJ+F7gWEHBbsQm4ZqJfiYiIGlkGbLW9zfYeGrlw5bgyK4Griv11wHJJmk6wdi3jtcCJtvc2H5T0aWAL8MmJLpI0AowAfPZjHzl55F2nT6du07d/b/syvaCSen327ikn7pzDyom7+6f9jznrkP7HBJg9p5y4e54rJezQa39zWoms2Qd0VMfTii/nJ++nyFWFUdujxf584JGmc9uBU8Z9xQtlbO+T9BRwTHFukaQ7gaeB37f9nVZ1aZeMDwAvBf5h3PFji3MTKm5mFODAnX+T+dYRUUnNuarLHgVeZvsJSScDfyXpRNtPT3ZBu2T8EeBbkv6en/1CvAw4Hji3GzWOiOimLv77dAdwXNPnBcWxicpslzQLmAc84caiP88B2L5d0kPACcDmyYK1TMa2vynpBBp9J80P8DbZ3t/xLUVE9Mms6XXZTmQTsFjSIhp5bzVw9rgyY8A5wK3AKuBm25b0YuBJ2/sl/VNgMbCtZb3b1cb2AeB7U76NiIgSDHUpFxd9wOcCG4Bh4ErbWyRdCGy2PQZcAVwtaSvwJI2EDfAm4EJJe2l06X7A9pOt4rVNxhERg6Sbj9FtrwfWjzt2ftP+buCsCa77MvDlqcRKMo6IWhnqXjdFXyUZR0StDOq04iTjiKiVbvUZ91uScUTUynC6KSIiypduioiICkg3RUREBaRlPJl9JS1iM9z/35k/ecOavscE+I/f/WIpcdm7u5y4JSzItOG03+p7TIC33/QXpcT1npL+bLsgQ9uqpIREHBHVMGswc3FNk3FEHLTSTRERUQFDDGbTOMk4ImoloykiIiog3RQRERWQlnFERAV0cXH5vkoyjohaSTdFREQFpJsiIqICMrQtIqIC0jKOiKiA4STjiIjyDWo3xbQfPEp6bzcrEhHRDUPqfKuSmYwC+cRkJySNSNosafPoDTfOIERExNQMTWGrkpbdFJL+72SngJdMdp3tUWAU4MCm9Z527SIipqhiDd6OteszfgnwdmDnuOMC/k9PahQRMQN1XVz+68ARtu8af0LSt3tSo4iIGaha90OnWiZj22tbnDu7+9WJiJiZwWwXZ2hbRNSMatpNERExUAYzFScZR0TN1LLPOCJi0AxoL0WScUTUy6BOh04yjohaGcxUnGQcETVTtTUnOpVkHBG1ogFtG/c+GQ8N9zzEeN9eUc6Cch9616tKifvt0/59KXHf/I0/LyXu107v//2u/MEDfY8J8K2FJ5YS9y1/84VS4nZDN1OxpBXAZ4Bh4HO2Pznu/BzgC8DJwBPAu20/3HT+ZcB9wAW2L24Va1BHgURETKhbS2hKGgYuA04DlgBrJC0ZV2wtsNP28cAlwEXjzn8a6GjpyiTjiKiVIdTx1sYyYKvtbbb3ANcCK8eVWQlcVeyvA5armAIo6R3A94EtndU7IqJGNJWtae31Yhtp+qr5wCNNn7cXx5iojO19wFPAMZKOAH6XFuu+j5cHeBFRK1OZ9NG89nqXXQBcYntXp2tlJBlHRK108QHeDuC4ps8LimMTldkuaRYwj8aDvFOAVZI+BRwNHJC02/alkwVLMo6IWuni0LZNwGJJi2gk3dXA+KWDx4BzgFuBVcDNtg38+gv1kS4AdrVKxJBkHBE1M9ylXGx7n6RzgQ00hrZdaXuLpAuBzbbHgCuAqyVtBZ6kkbCnJck4Imqlm+OMba8H1o87dn7T/m7grDbfcUEnsZKMI6JWMgMvIqICsoRmREQFDOrkibb1lvQKScuLQczNx1f0rloREdMzlUkfVdIyGUv6HeCrwIeAeyU1TwX8b72sWETEdAxJHW9V0q5l/D7gZNvvAN4M/IGkDxfnJr2T5imGo19ZP1mxiIiuG9SWcbs+4yHbuwBsPyzpzcA6Sb9Ci3tpnmJ44PYN7lJdIyLa6nT6cdW0axk/Jumk5z8UifkM4JeBV/ayYhER09GtJTT7rV3L+D3AvuYDxcpE75F0ec9qFRExTapalu1Qy2Rse3uLc3/b/epERMzM0ICObcs444iolUHtM04yjohaGdBcnGQcEfWSlnFERAUMaC5OMo6IeqnazLpOJRlHRK0M1XFoW0TEoFGGtkVElG9QH+Cp8e683rn+RS/p+9oUZz24sd8hAdDco0qJW5bbT3h1KXFPvqeE+UaHHt7/mIAf3VZK3Otf+5ulxH33zsdmnEkfWrK445zzq/f9fWUyd1rGEVErg9oyTjKOiFoZ0FycZBwR9TKc0RQREeVLN0VERAUMaC5OMo6IekkyjoiogFouLh8RMWjyAC8iogLSTRERUQEZTRERUQEDmovbJ2NJywDb3iRpCbACeMD2+p7XLiJiimrZMpb0ceA0YJakvwZOAW4BzpP0att/3Ic6RkR0bEBzcduW8SrgJGAO8I/AAttPS7oY2AhMmIwljQAjAO877EjeOuew7tU4IqKFoeHBzMbtlmHeZ3u/7WeAh2w/DWD7WeDAZBfZHrW91PbSJOKI6CdJHW9V0i4Z75E0t9g/+fmDkubRIhlHRJRmSJ1vbUhaIelBSVslnTfB+TmSrivOb5S0sDi+TNJdxXa3pHe2rXab828qWsXYbk6+hwDntL2TiIh+kzrfWn6NhoHLaDw3WwKsKQYxNFsL7LR9PHAJcFFx/F5gqe2TaAx6uFxSy27hlsnY9nOTHH/c9j0t7yQiogRd7KZYBmy1vc32HuBaYOW4MiuBq4r9dcBySbL9jO19xfFDgbZvHxnQV/dFRExieKjjTdKIpM1N20jTN80HHmn6vL04xkRliuT7FHAMgKRTJG0B7gE+0JScJ5RJHxFRK1NZKMj2KDDai3rY3gicKOmfAVdJutH27snKp2UcEfXSpT5jYAdwXNPnBcWxCcsUfcLzgCeaC9i+H9gF/FqrYEnGEVErGlLHWxubgMWSFkmaDawGxsaVGeNngxlWATfbdnHNLABJvwK8Ani4VbB0U0REvXRp/LDtfZLOBTYAw8CVtrdIuhDYbHsMuAK4WtJW4EkaCRvgjTRmKu+lMQz4t20/3ipeknFE1EsX1zMu1uBZP+7Y+U37u4GzJrjuauDqqcRKMo6IWtHwYPa+JhlHRL1UbJpzp3qejFfd9Pleh/gFt77q1/seE+CUj/2rUuKW5c8f3VlK3BP+dduZpV135PXlrBj7rdefWUrcs779xVLidoMGs2GclnFE1ExaxhER5cvboSMiqiAt44iI8mU0RUREFaSbIiKiAtJNERFRvqq9TqlTScYRUS/ppoiIKF8e4EVEVEG6KSIiyjeokz6m3J6X9IVeVCQioiu696aPvmrZMpY0flV7AW+RdDSA7XJWMYmImExNW8YLgKeBTwN/Umw/adqfUPMbV0dvuLFbdY2IaEtSx1uVtOszXgp8GPgY8J9t3yXpWdv/q9VFzW9cPbBpvbtS04iITtRxNIXtA8Alkr5U/PexdtdERJSqYi3eTnWUWG1vB86SdDqNbouIiGqqczJ+nu1vAN/oUV0iImZuqIbdFBERA+dgaBlHRFReknFERAUMD5ddg2lJMo6IeknLOCKiApKMIyIqIMk4IqICMrQtIqICkown4QM9DzHe68Y+2/eYALec/v5S4n7p8Z+UErcsn7p5a99jvvxlJ/Y9JsDZ37m+lLjsfqacuN2QboqIiPIpLeOIiApIyzgiogKSjCMiKiDJOCKiAgZ0OvRg9nRHREymiy8klbRC0oOStko6b4LzcyRdV5zfKGlhcfxUSbdLuqf472+0i5VkHBH10qVkLGkYuAw4DVgCrJG0ZFyxtcBO28cDlwAXFccfB/6l7VcC5wBXt6t2knFE1MvQUOdba8uArba32d4DXAusHFdmJXBVsb8OWC5Jtu+0/cPi+BbgMElzWlZ7SjcZEVF1U2gZN7/JvthGmr5pPvBI0+ftxTEmKmN7H/AUcMy4Mu8C7rD9XKtq5wFeRNTLFEZTNL/JvjdV0Yk0ui7e1q5sknFE1Ev3RlPsAI5r+rygODZRme2SZgHzgCcAJC0AbgDeY/uhdsGm1E0h6Y2SPiqpbZaPiChF90ZTbAIWS1okaTawGhgbV2aMxgM6gFXAzbYt6WgaL28+z/bfdlLtlslY0m1N++8DLgWOBD4+0TCPiIjSdSkZF33A5wIbgPuB621vkXShpDOLYlcAx0jaCnwUeD4vngscD5wv6a5i+yet4rXrpjikaX8EONX2jyVdDHwP+OREFxWd4CMAnz3vg4y8c0WbMBERXdLFhYJsrwfWjzt2ftP+buCsCa77I+CPphKrXTIekvQiGi1o2f5xEeinkvZNdlFzp/iB277uqVQoImJGajodeh5wOyDAko61/aikI4pjERHVMjSY06FbJmPbCyc5dQB4Z9drExExU0OD2U6c1tA2288A3+9yXSIiZk6DOZct44wjol5q2mccETFY8tqliIgKSMs4IqIC6jiaIiJi4KSbIiKiAtJNERFRARnaFhFRAQfTpI8p2b+/5yF+wexD+x8TeMs3rywl7i+dOdK+UA9c/sOdpcQ945eO6HvM137t8r7HBOC5Z8uJOzzA7bQ8wIuIqIB0U0REVEC6KSIiKiCjKSIiKiDdFBERFZBuioiICshoioiICkg3RUREBaSbIiKiAtIyjoiogAxti4iogCyhGRFRAQM6mqLlT4ikUyQdVewfJukTkr4m6SJJ8/pTxYiIKZA63yqkXXv+SuCZYv8zwDzgouLY5ye7SNKIpM2SNo/+1YauVDQioiNDQ51vFdKum2LI9r5if6nt1xT735V012QX2R4FRgEO3PpVz7yaEREdqliLt1PtfhrulfTeYv9uSUsBJJ0A7O1pzSIipkNDnW8V0q5l/FvAZyT9PvA4cKukR4BHinMREdUyoA/wWiZj208B/654iLeoKL/d9mP9qFxExJTVeQae7aeBu3tcl4iImatY90OnMs44IuplQB/gJRlHRL0MaMt4MGsdETEJSR1vHXzXCkkPStoq6bwJzs+RdF1xfqOkhcXxYyTdImmXpEs7qXeScUTUy9CszrcWJA0DlwGnAUuANZKWjCu2Fthp+3jgEhqT4gB2A38A/KeOq91pwYiIgTCkzrfWlgFbbW+zvQe4Flg5rsxK4Kpifx2wXJJs/9T2d2kk5c6q3WnBiIiBMIVJH81LNxTbSNM3zacxp+J524tjTFSmmK38FHDMdKqdB3gRUS9TGE3RvHRD2ZKMI6JeujeaYgdwXNPnBcWxicpslzSLxmJqT0wnWO+T8SGzex7iF+wvadmMkobU/POxcn7Y157xvlLinvzVP+1/0P37+x8Tyvn/B2B4MKcUA90cZ7wJWCxpEY2kuxo4e1yZMeAc4FZgFXCz7WktjpaWcUTUS5d+SGzvk3QusAEYBq60vUXShcBm22PAFcDVkrYCT9JI2ABIehg4Cpgt6R3A22zfN1m8JOOIqJcu/gvV9npg/bhj5zft7wbOmuTahVOJlWQcEfWS6dARERUwoNOhk4wjol7SMo6IqIDhwUxrg1nriIhJdLIAUBUlGUdEvaTPOCKiAtIyjoiogLSMIyIqYEBbxi1/QiT9jqTjWpWJiKiU4eHOtwpp157/Q2CjpO9I+m1JL+5HpSIipm0K6xlXSbvabKOxbNwfAicD90n6pqRzJB052UXNCzaPfuXGLlY3IqINqfOtQtr1Gdv2AeAm4CZJh9B4H9Qa4GJgwpZy84LNBzbfOK3l5CIipqdaSbZT7ZLxz92V7b001u8ckzS3Z7WKiJiuirV4O9UuGb97shO2n+lyXSIiZq6Oydj23/WrIhERXVGxB3OdyjjjiKiXwWwYJxlHRN0MZjZOMo6Ieqljn3FExMBJMo6IqIA8wIuIqIK0jCMiypduioiICkgynoRLWJpiqKSl8cq4VyjtL99rrvlUKXFLMXtOKWH97K5S4urQw0uJ2x1JxhERpcsLSSMiqiCjKSIiKiAt44iICkgyjoiogiTjiIjypWUcEVEBg5mLk4wjomYymiIiogLSTRERUQWDmYwHsz0fETEZqfOt7VdphaQHJW2VdN4E5+dIuq44v1HSwqZzv1ccf1DS29vFapmMJc2W9B5Jby0+ny3pUkkflHRI2zuJiOi3LiVjScPAZcBpwBJgjaQl44qtBXbaPh64BLiouHYJsBo4EVgB/GnxfZNq103x+aLMXEnnAEcAXwGWA8uAc9pcHxHRX917gLcM2Gp7G4Cka4GVwH1NZVYCFxT764BL1VgcYyVwre3ngO9L2lp8362TBWuXjF9p+1WSZgE7gJfa3i/pfwJ3T3aRpBFgpPj4ftujbeJM+j3TvXYmDqa4B9O9HmxxD6Z7/Tlz53XcaTwuVwGMNtV9PvBI07ntwCnjvuKFMrb3SXoKOKY4/r1x185vVZd2PyFDkmYDRwJzgXnF8TnApN0UtkdtLy22mfyhjLQv0hMHU9yD6V4PtrgH071Oy7hcNdN8NSPtWsZXAA8Aw8DHgC9J2ga8Dri2x3WLiCjTDuC4ps8LimMTldle9CDMA57o8Nqf07JlbPsS4I3A623/D+BdwAZgre1PtL2ViIjBtQlYLGlR0UOwGhgbV2aMnz07WwXcbNvF8dXFaItFwGLgtlbB2o4ztv3Dpv3/R6OTul/K+ifDwRT3YLrXgy3uwXSvXVf0AZ9LowE6DFxpe4ukC4HNtsdo9B5cXTyge5JGwqYodz2Nh337gA/a3t8qnlzWq4IiIuIFmfQREVEBScYRERVQ2WTcbhpij2JeKelHku7tR7wi5nGSbpF0n6Qtkj7cp7iHSrpN0t1F3L49kJU0LOlOSV/vV8wi7sOS7pF0l6TNfYp5tKR1kh6QdL+k1/ch5suLe3x+e1rSR3odt4j9H4q/T/dKukbSof2IWweV7DMupg3+HXAqjcHSm4A1tu9reeHM474J2AV8wfav9TJWU8xjgWNt3yHpSOB24B19uFcBh9veVUxt/y7wYdvfa3NpN2J/FFgKHGX7jF7Ha4r7MLDU9uN9jHkV8B3bnyueyM8tHoT3K/4wjSFVp9j+hx7Hmk/j79ES288WD7DW2/6LXsati6q2jF+Yhmh7D40xzSt7HdT2/6bxRLRvbD9q+45i/yfA/bSZqdOluLa9q/h4SLH1/JdZ0gLgdOBzvY5VNknzgDfReOKO7T39TMSF5cBDvU7ETWYBhxVjbucCP2xTPgpVTcYTTUPseYIqW7Hi06uBjX2KNyzpLuBHwF/b7kfc/w78F+BAH2KNZ+AmSbcX02B7bRHwY+DzRbfM5yQd3oe4zVYD1/QjkO0dwMXAD4BHgads39SP2HVQ1WR80JF0BPBl4CO2n+5HTNv7bZ9EY3bQMkk97ZqRdAbwI9u39zJOC2+0/Roaq3B9sOiW6qVZwGuAz9p+NfBToC/PP6Cx6iJwJvClPsV7EY1/wS4CXgocLunf9CN2HVQ1GU95KuEgK/psvwz8pe2v9Dt+8U/nW2gs9ddLbwDOLPpurwV+o1h0qi+Klhu2fwTcQKM7rJe2A9ub/sWxjkZy7pfTgDtsP9aneG8Fvm/7x7b30ljh8V/0KfbAq2oy7mQaYi0UD9KuAO63/ek+xn2xpKOL/cNoPCx9oJcxbf+e7QW2F9L4M73Zdl9aTpIOLx6QUnQVvA3o6agZ2/8IPCLp5cWh5fz88ou9toY+dVEUfgC8TtLc4u/1chrPQKIDlXzt0mTTEHsdV9I1wJuBX5a0Hfi47St6HPYNwL8F7in6bwH+q+31PY57LHBV8bR9CLjedl+HmvXZS4AbGjmCWcAXbX+zD3E/BPxl0ajYBry3DzGf/8E5FXh/P+IB2N4oaR1wB40pwHdSk6nR/VDJoW0REQebqnZTREQcVJKMIyIqIMk4IqICkowjIiogyTgiogKSjCMiKiDJOCKiAv4/uhyFrWR/NQIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knowledge uncertainty via virtual ensemble:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAdMklEQVR4nO3df5TV9X3n8edrLmJFI3YxmxBgI2chtUN/GPWQZNP1pCGpkGaZ7NEch+4marXjplBj0rOJbrvNidnsWc62i9ZoEgoouq5IiGkmKVWbajduNgrEXxWU7ARUhlh/AIFaLDjw3j/uB70znXu/d2buj5nPvB6e7/Hez/f7/bw/c4H3/czn+/l+vooIzMwsHx3tboCZmTWWE7uZWWac2M3MMuPEbmaWGSd2M7PMOLGbmWXGid3MLDNTig6QdDbQBcxKRXuB3oh4upkNMzOz0anZY5f0eWADIGBL2gTcJena5jfPzMxGSrXuPJX0Y2BBRLw+pHwqsD0i5lc5rwfoAfj6TTec1/PblzWswWZtEcfbFFhtitsmp54x5h/4P+j0um+n/1ocyvIDLhqKOQ68A3huSPnMtG9YEbEaWA3A4YNes8DMrIWKEvs1wF9L+n/AnlT2L4B5wIpmNszMbDQ8I6QgsUfEvZLeBSxk8MXTrRFxrNmNMzMbqSnKcnRlRApnxUTEceDhFrTFzGzMOpzXixO7mdlE4qEYJ3Yzy0yHh2Kc2M0sL+6xO7GbWWY8xu4vNzPLTEmqeysiabGknZL6hrvbXtLJku5O+x+RdFbFvutS+U5JFxbVKWmtpCckPSlpk6TTUvllkl6W9HjarixqtxO7mWWlYwRbLZJKwM3AEqATWCapc8hhVwAHImIesApYmc7tBLqBBcBi4BZJpYI6PxMRvxoRvwI8z+B7he6OiHPStqaez8DMLBsdqn8rsBDoi4hdEXGU8rpZXUOO6QLWp9ebgEWSlMo3RMSRiNgN9KX6qtYZEYcA0vmnAKO+a9+J3cyyMpIeu6QeSdsqtp6Kqmbx5h33AP28eaPmPzkmIgaAg8CMGufWrFPSrcDfAWcDN1Ucd1HFEM2cos8g34unNRY3axpPs8qXJlkf6PjEvbF8JNMdB61rNQ5ExOVpuOYm4BLgVuA7wF0RcUTSVZR/Q/hgrXry/NvajqRuZuPCFNW/FdgLVPaOZ6eyYY+RNAWYDuyrcW5hnWm5lg3ARen9vog4knavAc4ranieid3MJq1GXTwFtgLzJc1NS5V3A71DjukFLk2vLwYeiPJa6L1Ad5o1MxeYT/l5FsPWqbJ58MYY+1LgmfR+ZkW8pUDhQ47yHYoxs0mpo0Fr2EfEgKQVwH1ACVgXEdslXQ9si4heYC1wh6Q+YD/lRE06biOwAxgAlp9YOLFKnR3AekmnU16E/wngU6kpV0tamurZD1xW1PaaD9poiHasx96uoRiPsVsu2jXGfto/G/M/oj+ZNqPuBPD7h/dl+Y/WPXYzy4rHl53YzSwzXlLAid3MMuMHbTixm1lmPBTjxG5mmfFQjBO7mWWmUdMdJzIndjPLinvsTuxmlpmSE7sTu5nlxUMxY7iALOnyRjbEzKwRGrge+4Q1lplBX6y2o3KN49XrbhtDCDOzkWngImATVs2hGElPVtsFvK3aeYPWOG7HWjFmNmll3BGvW9EY+9uAC4EDQ8oF/N+mtMjMbAxG8qCNXBUl9u8Cp0XE40N3SPqbprTIzGwMch5iqVfNxB4RV9TY91uNb46Z2di4v+7pjmaWGXkoxondzPLitO7EbmaZ8Ri7E7uZZcYjMU7sZpYZLyng31rMLDMawVZYl7RY0k5JfZKuHWb/yZLuTvsfkXRWxb7rUvlOSRcW1SlpraQnJD0paZOk04piVOPEbmZZadRaMZJKwM3AEqATWCapc8hhVwAHImIesApYmc7tBLqBBcBi4BZJpYI6PxMRvxoRvwI8D6yoFaPmZ1B0gJnZRKIR/FdgIdAXEbsi4iiwAegackwXsD693gQsUnm+ZRewISKORMRuoC/VV7XOiDgEkM4/BYiCGFVlOsbenuVp4uDLbYmr089sS1zUpn5BtP7P9+CSC1oeE2D6vQ+1JS4dpfbEbYCRjLBL6gF6KopWp7WuAGYBeyr29QPvGVLFG8dExICkg8CMVP7wkHNnpddV65R0K/ARYAfw+wUxXqn2c2Wa2M1sshrJcryDFiwcByLi8jRccxNwCXDraOrxUIyZZaUD1b0V2AvMqXg/O5UNe4ykKcB0YF+NcwvrjIhjlIdoLiqIUZUTu5llpYGzYrYC8yXNlTSV8sXQ3iHH9AKXptcXAw9ERKTy7jSjZS4wH9hSrU6VzYM3xtiXAs8UxKjKQzFmlpVG3aCUxrNXAPcBJWBdRGyXdD2wLSJ6gbXAHZL6gP2UEzXpuI2Ux8oHgOWpJ06VOjuA9ZJOp/yd8wTwqdSUYWPU/AwKEv/YteNBG3G85SEB4lDVaxlN5YunzTfpLp62y7TpY07LD/7zWXX/Bfn1l/ZmeTeTe+xmlpU6pjFmz4ndzLJScl53YjezvDivO7GbWWY8FOPEbmaZ8bK9TuxmlhnfnFPHZyDpbEmLTiwhWVG+uHnNMjMbnUYu2ztR1Uzskq4Gvg38HvCUpMqVzf5rMxtmZjYaHVLdW66Keuy/A5wXER8DPgD8Z0mfTvuqfiqSeiRtk7Rt9brbGtJQM7N6uMdePMbeERGvAkTEs5I+AGyS9E5qfC6DVkxrx52nZjZpFSxVPikU9dhflHTOiTcpyX8UOBP45WY2zMxsNBr1BKWJrKjH/knKC9i8ISIGgE9K+nrTWmVmNkrKOWPXqWZij4j+Gvt+0PjmmJmNTYfnO3oeu5nlxWPsTuxmlhnndSd2M8uMe+xO7GaWGed1J3Yzy0zOd5TWy4ndzLLS4emOTuxmlpd2PYp3PHFiN7Os+OJppon9+NMPtyfwSVPbEjZeer4tcTvmn9+WuK9/dlnLY37+oWdbHhNg1eX/pi1xT7n1O22J2wiNzOtpefIbgRKwJiL+25D9JwO3A+cB+4BLIuLZtO864ArgGHB1RNxXq05JdwLnA68DW4CrIuL1tEbXt4HdKew9EXF9rXb7lxYzy4qkureCekrAzcASoBNYJqlzyGFXAAciYh6wCliZzu0EuoEFwGLgFkmlgjrvBM6mvA7XKcCVFXEeiohz0lYzqYMTu5llRqp/K7AQ6IuIXRFxFNgAdA05pgtYn15vAhap/I3RBWyIiCMRsRvoS/VVrTMiNkdCucc+e7SfgRO7mWWl1KG6t8pnR6Stp6KqWcCeivf9qYzhjkkLJB4EZtQ4t7BOSScBnwDurSh+n6QnJP2lpAVFn0GWY+xmNnmN5OLpoGdHjB+3AN+PiIfS+0eBd0bEq5I+Avw5ML9WBe6xm1lWGjgUsxeYU/F+diob9hhJU4DplC+iVju3Zp2SvgC8FfjsibKIOFTxwKPNwEmSzqzVcCd2M8tKAxP7VmC+pLmSplK+GNo75Jhe4NL0+mLggTRG3gt0SzpZ0lzKPewtteqUdCVwIbAsIo6/+fPo7WncHkkLKeftfbUa7qEYM8tKox60EREDklYA91GemrguIrZLuh7YFhG9wFrgDkl9wH7KiZp03EZgB+WHFS2PiGMAw9WZQn4NeA74YcrjJ6Y1Xgx8StIA8BrQnb48qnJiN7OslBq4pEAa+tg8pOyPKl7/I/DxKud+GfhyPXWm8mHzcUR8BfjKSNrtxG5mWfGNp07sZpYZLyngxG5mmXFeryOxp6uwERFb062vi4Fn0jiRmdm44h57QWJPcyqXAFMk/RXwHuBB4FpJ704XB8zMxg3n9eJ57BcD7wcuAJYDH4uIL1Gea3lJtZMqb9Ndve62RrXVzKxQR0l1b7kqGooZSHMvD0v6SUQcAoiI1yQdr3bSoNt0Dx+sOd/SzKyRPBRTnNiPSpoWEYcprzcMgKTpQNXEbmbWNn40XmFivyAijgBU3uIKnMSbt9GamY0f7rHXTuwnkvow5a8ArzSlRWZmY+ChGM9jN7PclLy2oRO7mWWlUYuATWRO7GaWFw/FOLGbWV7cY3diN7PcuMfuxG5mmXGP3YndzPIiz4pxYjezzHgoBhU8Om/sJtFaMcefeqjdTWip+PsD7Ql88imtj/m977Y+JlD63I1tids206aPOSsfvuj9deecad/8QZbfAu6xm1le3GMvXLbXzGxCUYfq3grrkhZL2impT9K1w+w/WdLdaf8jks6q2HddKt8p6cKiOiXdmcqfkrRO0kmpXJL+NB3/pKRzi9rtxG5meZHq32pWoxJwM+WHDXUCy9JT5CpdARyIiHnAKmBlOrcT6AYWUH7q3C2SSgV13gmcDfwycApwZSpfAsxPWw/w1aKPwIndzLKiUkfdW4GFQF9E7IqIo8AGoGvIMV3A+vR6E7BI5VXIuoANEXEkInYDfam+qnVGxOZIgC3A7IoYt6ddDwNnSJpZq+FO7GaWlw7VvVU+7S1tPRU1zQL2VLzvT2UMd0xEDAAHgRk1zi2sMw3BfAK4dwTtGMQXT80sLyO4eDroaW/jxy3A9yNi1NPsnNjNLCsNXI99LzCn4v3sVDbcMf2SpgDTgX0F51atU9IXgLcCV42wHYN4KMbM8jKCoZgCW4H5kuZKmkr5YmjvkGN6efNpchcDD6Qx8l6gO82amUv5wueWWnVKuhK4EFg25Il1vcAn0+yY9wIHI+KFWg13j93MstKoJQUiYkDSCuA+oASsi4jtkq4HtkVEL7AWuENSH7CfcqImHbcR2AEMAMsj4hjAcHWmkF8DngN+mH7ruCcirgc2Ax+hfAH2MHB5Udud2M0sLw28QSkiNlNOrJVlf1Tx+h+Bj1c598vAl+upM5UPm4/TbwDLR9JuJ3Yzy4rXYx/FGLuk25vREDOzhmjQDUoTWc0eu6ShFwoE/LqkMwAiYmmzGmZmNirusRcOxcymPPi/BgjKif184E9qnZQm+fcAfP2mG+j57cvG3FAzs3o0cLrjhFWU2M8HPg38AfAfI+JxSa9FxP+uddKgSf+TaNleMxsH/KCN2ok9zaVcJekb6f8vFp1jZtZW7rHXl6Qjoh/4uKTfBA41t0lmZmPgxD6y3ndE/AXwF01qi5nZ2HV4KMbDKmaWF/fYndjNLDNO7E7sZpaZUqndLWg7J3Yzy4t77E7sZpYZJ3YndjPLjBO7E7uZZcbTHZ3YzSwzTuyZJvZBT5VqnY5f+tdtiXts3ZfaEjf27Ck+qAk0fXrLY5Y+d2PLY7ZVTOAlnjwUk2liN7NJS+6xO7GbWWbcY3diN7PMOLGP/NF4ZmbjWgMfjSdpsaSdkvokXTvM/pMl3Z32PyLprIp916XynZIuLKpT0opUFpLOrCj/gKSDkh5P2xsP067GPXYzy0uDlhSQVAJuBj4M9ANbJfVGxI6Kw64ADkTEPEndwErgEkmdQDewAHgH8D1J70rnVKvzB8B3gb8ZpjkPRcRH6227e+xmlpfG9dgXAn0RsSsijgIbgK4hx3QB69PrTcAilZ/N1wVsiIgjEbEb6Ev1Va0zIh6LiGfH/gE4sZtZbhqX2GcBlXN6+1PZsMdExABwEJhR49x66hzO+yQ9IekvJS0oOthDMWaWlxFMd5TUA/RUFK1Oz2weTx4F3hkRr0r6CPDnwPxaJzixm1leRjArJiXxaol8LzCn4v3sVDbcMf2SpgDTgX0F5xbVObSNhypeb5Z0i6QzI+KVaud4KMbM8tK4oZitwHxJcyVNpXwxtHfIMb3Apen1xcADERGpvDvNmplLuYe9pc46h/w4ensat0fSQsp5e1+tc9xjN7O8NGhWTEQMSFoB3AeUgHURsV3S9cC2iOgF1gJ3SOoD9lNO1KTjNgI7gAFgeUQcg/K0xqF1pvKrgc8BbweelLQ5Iq6k/IXxKUkDwGtAd/ryqEoF+wcfLP0a5au6T0XE/XWddPhg6xedaNNaMag9vwB5rZjmK13z31ses63atVbMqWeM+e6iYzdcU3fjS9fckOXdTDUzkaQtFa9/B/gK8BbgC8NN1jcza7sG3qA0URV1MU+qeN0DfDgivgj8BvDvqp0kqUfSNknbVq+7beytNDOrV0dH/VumisbYOyT9POUvAEXEywAR8Q9pvGdYg640t2Moxswmr4x74vUqSuzTgR8BAkLSzIh4QdJpqczMbHzpaMzF04msZmKPiLOq7DoO/NuGt8bMbKw63Occ1XTHiDgM7G5wW8zMxq5Ns9PGE89jN7O8eIzdid3MMpPxbJd6ObGbWV7cY3diN7PMeFaME7uZZcZDMU7sZpYZD8U4sZtZZjzd0YndzDLjG5RakNgHjjY9xLgxZWpbwpYu/8O2xI0XftKWuHrHvLbEnVQm8nCGL566x25mmfFQjBO7mWXGQzFO7GaWmYk8jNQgTuxmlhcPxTixm1lmPBTjxG5mmfGsmMJnnpqZTSzqqH8rqkpaLGmnpD5J1w6z/2RJd6f9j0g6q2Lfdal8p6QLi+qUtCKVhaQzK8ol6U/TviclnVvUbid2M8tLh+rfapBUAm4GlgCdwDJJnUMOuwI4EBHzgFXAynRuJ9ANLAAWA7dIKhXU+QPgQ8BzQ2IsAeanrQf4auFHUHSAmdmE0rge+0KgLyJ2RcRRYAPQNeSYLmB9er0JWCRJqXxDRByJiN1AX6qvap0R8VhEPDtMO7qA26PsYeAMSTNrNdyJ3czyItW9SeqRtK1i66moaRawp+J9fypjuGMiYgA4CMyocW49dQ414nN88dTM8jKCZXsjYjWwunmNaQ8ndjPLS+NmxewF5lS8n53KhjumX9IUYDqwr+DcojpH045Ban61SXqPpNPT61MkfVHSdyStlDS9oDFmZq03gqGYAluB+ZLmSppK+WJo75BjeoFL0+uLgQciIlJ5d5o1M5fyhc8tddY5VC/wyTQ75r3AwYh4odYJRb+zrAMOp9c3Uv42WpnKbq12UuW41epbby8IYWbWQB0d9W81pDHzFcB9wNPAxojYLul6SUvTYWuBGZL6gM8C16ZztwMbgR3AvcDyiDhWrU4ASVdL6qfcI39S0poUYzOwi/IF2D8DfrfoI1D5y6XKTunpiPjF9PrRiDi3Yt/jEXFOUQAOvVw9QG7atGwvNf4MmxrWy/Zao02bPubbRo/99R11/4MoLfpElrepFvXYn5J0eXr9hKTzASS9C3i9qS0zMxuNBt6gNFEVXTy9ErhR0h8CrwA/lLSH8tSbK5vdODOzEfOSArUTe0QcBC5LF1DnpuP7I+LFVjTOzGzEvAhYfdMdI+IQ8EST22JmNnYZD7HUy/PYzSwvftCGE7uZZcY9did2M8uL3GN3YjezzHQ4rfkTMLO8eFaME7uZZcZj7E7sZpYZj7E7sZtZZtxjb0Fib8ftve36g23TYlzt6qFo5r9sS9xJZZL9nWqIidz2BnGP3czyUvJaMU7sZpYXD8U4sZtZZjwU48RuZplxj92J3cwy4x67E7uZZabktOZPwMyy4kXAip95amY2sTTwmaeSFkvaKalP0rXD7D9Z0t1p/yOSzqrYd10q3ynpwqI6Jc1NdfSlOqem8sskvSzp8bQVPpbUid3M8iLVv9WsRiXgZmAJ0Aksk9Q55LArgAMRMQ9YBaxM53YC3cACYDFwi6RSQZ0rgVWprgOp7hPujohz0ram6CNwYjezvDSux74Q6IuIXRFxFNgAdA05pgtYn15vAhapPBbUBWyIiCMRsRvoS/UNW2c654OpDlKdHxvtR+DEbmZ5GUGPXVKPpG0VW09FTbOAPRXv+1MZwx0TEQPAQWBGjXOrlc8AfpbqGC7WRZKelLRJ0pyij6DmxVNJVwPfiog9tY4zMxs3RrCkQESsBlY3rzEN8R3grog4Iukqyr35D9Y6oajH/iXgEUkPSfpdSW9tUEPNzJqjcUMxe4HK3vHsVDbsMZKmANOBfTXOrVa+Dzgj1TEoVkTsi4gjqXwNcF5Rw4t+sl0pwJdSZTsk3SvpUklvqXZS5a83q9etr3aYmVnjNejiKbAVmJ9mq0ylfDG0d8gxvcCl6fXFwAMREam8O82amQvMB7ZUqzOd82Cqg1Tnt8s/jmZWxFsKPF3U8KJ57BERx4H7gfslnUT5au4y4I+BYXvwg369eXV/m9YdNbPJqTHz2CNiQNIK4D6gBKyLiO2Srge2RUQvsBa4Q1IfsJ9yoiYdtxHYAQwAyyPiGMBwdaaQnwc2SPovwGOpboCrJS1N9ewHLitqu6LGes+SHouId1fZNy0iDhcFaEtin2xrRbTrhgyvFd58k+0znjZ9zIFjz466PzTN6czyL1NRj/2SajvqSupmZq02mb74q6iZ2CPix61qiJlZQ0y239iH4bVizCwv7rA7sZtZbpzZndjNLC8eY3diN7PMOLE7sZtZZnzx1IndzHLjHrsTu5nlxUMxTuxmlhkn9hYk9o76l9C0UWrXbefWfE5So+DPzD12M8uKH2btxG5mufGsGCd2M8uMe+xO7GaWGSd2J3Yzy40TuxO7meXFPXYndjPLjPO6E7uZZcazYpzYzSwzHopxYjez3DixO7GbWV7cY6+d2CVNBbqBn0bE9yT9FvCvgKeB1RHxegvaaGZWPyd2FDUWkJJ0J+XkPw34GXAacA+wKJ17aWGEwwe9QlWzTbZFwPwPN1/Tpo/9D3ckOacR8cajiKi6AU+m/08BXgRK6b1O7KtyXg+wLW09tWIUxB/1uWPZJlPcyfSzTra4k+ln9TZ4K+qxPwWcC5wKPA+8MyL2S/o54LGI+MXGfL1Ujb8tIs5vZozJHncy/ayTLe5k+lltsKKLp2uBZ4AS8AfANyTtAt4LbGhy28zMbBRqJvaIWCXp7vT6p5JuBz4E/FlEbGlFA83MbGQKpztGxE8rXv8M2NTUFg22uoWxJmvcyfSzTra4k+lntQo1x9jNzGzi8aIKZmaZcWI3M8vMuE3skhZL2impT9K1LYq5TtJLaZpnS0iaI+lBSTskbZf06RbF/TlJWyQ9keJ+sRVxU+ySpMckfbdVMVPcZyX9raTHJW1rUcwzJG2S9IykpyW9rwUxfyH9jCe2Q5KuaXbcFPsz6e/TU5LuSlOjrcXG5Ri7pBLwY+DDQD+wFVgWETuaHPcC4FXg9oj4pWbGqog5E5gZEY9KegvwI+BjLfhZBZwaEa9KOgn4P8CnI+LhZsZNsT8LnA+cHhEfbXa8irjPAudHxCstjLkeeCgi1qQlOqalSQitil8C9gLviYjnmhxrFuW/R50R8ZqkjcDmiLitmXHtnxqvPfaFQF9E7IqIo5TnzHc1O2hEfB/Y3+w4Q2K+EBGPptd/T3kdnlktiBsR8Wp6e1Lamv4tL2k28JvAmmbHajdJ04ELKN8PQkQcbWVSTxYBP2l2Uq8wBThF0omlSH5acLw1wXhN7LOAPRXv+2lBsms3SWcB7wYeaVG8kqTHgZeAv4qIVsS9AfgccLwFsYYK4H5JP5LU04J4c4GXgVvT0NMaSae2IG6lbuCuVgSKiL3AH1O+S/0F4GBE3N+K2DbYeE3sk46k04BvAtdExKFWxIyIYxFxDjAbWCipqcNPkj4KvBQRP2pmnBp+LSLOBZYAy9PQWzNNobwkx1cj4t3APwAtuV4Eb6zOuhT4Rovi/Tzl36znAu8ATpX071sR2wYbr4l9LzCn4v3sVJalNMb9TeDOiLin1fHT8MCDwOImh3o/sDSNdW8APijpfzY55htSj5KIeAn4FuUhv2bqB/orfhPaRDnRt8oS4NGIeLFF8T4E7I6Il6O8pPc9lJf5thYbr4l9KzBf0tyKNeF729ympkgXMdcCT0fE/2hh3LdKOiO9PoXyhepnmhkzIq6LiNkRcRblP9MHIqIlPTpJp6aL06ThkN8Amjr7KSL+Dtgj6RdS0SKgqRfFh1hGi4ZhkueB90qalv5eL6J8zchabFw+QSkiBiStAO6jvADZuojY3uy4ku4CPgCcKakf+EJErG1y2PcDnwD+No13A/yniNjc5LgzgfVp1kQHsDEiWjr9sMXeBnyrnG+YAvyviLi3BXF/D7gzdVB2AZe3IOaJL68PA1e1Ih5ARDwiaRPwKDAAPIaXF2iLcTnd0czMRm+8DsWYmdkoObGbmWXGid3MLDNO7GZmmXFiNzPLjBO7mVlmnNjNzDLz/wGZEIY0HWuzNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mean_preds = preds[:,0] # mean values predicted by a virtual ensemble\n",
    "\n",
    "data = preds[:,2] # estimated data uncertainty\n",
    "print(\"Average predicted data uncertainty:\")\n",
    "sns.heatmap(data.reshape([num_cat,num_cat]), cmap=\"Reds\", vmin = 0, vmax = 0.05)\n",
    "plt.savefig(\"heart_data_vsglb.pdf\")\n",
    "plt.show()\n",
    "\n",
    "knowledge = preds[:,1] # knowledge uncertainty\n",
    "print(\"Knowledge uncertainty via virtual ensemble:\")\n",
    "sns.heatmap(knowledge.reshape([num_cat,num_cat]), cmap=\"Reds\")\n",
    "plt.savefig(\"heart_knowledge_vsglb.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let us do the same for SGB ensembles. In this case, we do not use posterior_sampling and instead set bootstrap_type='Bernoulli' and subsample=0.5. Note that a smaller learning rate is needed for convergence. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best iteration = 858\n",
      "best iteration = 483\n",
      "best iteration = 702\n",
      "best iteration = 393\n",
      "best iteration = 538\n",
      "best iteration = 419\n",
      "best iteration = 519\n",
      "best iteration = 552\n",
      "best iteration = 474\n",
      "best iteration = 450\n"
     ]
    }
   ],
   "source": [
    "def ensemble(train_pool, val_pool, num_samples=10, iters=1000, lr=0.1):\n",
    "    ens_preds = []\n",
    "    for seed in range(num_samples):\n",
    "        model = CatBoostRegressor(random_seed=seed, iterations=iters, learning_rate=lr, \n",
    "                                  loss_function='RMSEWithUncertainty', subsample=0.5,\n",
    "                                  bootstrap_type='Bernoulli', verbose=False)\n",
    "        model.fit(train_pool, eval_set=val_pool, use_best_model=False)\n",
    "        print(\"best iteration =\", model.get_best_iteration()) # check that best iteration is reasonable\n",
    "        ens_preds.append(model.predict(test))\n",
    "    return ens_preds\n",
    "\n",
    "ens_preds = ensemble(train_pool, val_pool, num_samples=10, iters=1000, lr=0.1)\n",
    "ens_preds = np.asarray(ens_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average predicted data uncertainty:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAD8CAYAAACihcXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAX/0lEQVR4nO3dfbRddZ3f8ffn3kAwAsEVrQsTnKRDHBvGFiUrDGpZagYNPgVrWATXVLQZg0uZwbraytQZRJx2SUuldqCOWYCDqTVolDZgeHAJ9WkgJDwVAjITAjME8YGHASPEkNxP/zgbPdzee865N+fsvc++n9dae2Wf/XC+v5Nkfe/vfvfv9zuyTUREVGuk6gZERESScURELSQZR0TUQJJxREQNJBlHRNRAknFERA0kGUdETELSCkn3S9oh6ZwJzs+WdGVxfoukhcXxhZKelXRnsf1lt1izemjMq4GVwPzi0CPAJtv3TeVDRUQME0mjwCXAScAuYKukTbbvbbtsDfCk7aMlrQYuAE4rzj1g+9he43XsGUv6BLABEHBrsQn46kQ/JSIiGmQZsMP2Ttt7aeXCleOuWQlcUexvBJZL0nSCdesZrwGOsf1c+0FJnwO2A5+d6CZJa4G1AF/45NnHrf0Xb59O26ZvbH+58Z6niqo++5/rfs0gjB5UTdwqPu/IaPkxobr/Ux6rJOzI0pOnlcjafViH9zyt+Iv84kyKXFVYZ3tdsT8feLjt3C7g+HFv8etrbO+T9BQwrzi3SNIdwNPAn9r+fqe2dEvGY8ArgL8bd/zI4tyEig+zDmDs9hsy3zoiaqk9V/XZo8ArbT8u6Tjgf0k6xvbTk93QLRl/DPiOpL/lNz8hXgkcDZzVjxZHRPRTH3+XeAQ4qu31guLYRNfskjQLmAs87taiP78CsH2bpAeAVwHbJgvWMRnbvk7Sq2jVTtof4G21XVEtICJicrOmV7KdyFZgsaRFtPLeauB9467ZBJwB3AysAm60bUkvA56wvV/SPwYWAzs7trtba2yPAbdM+WNERFRgpE+5uKgBnwVcD4wCl9veLul8YJvtTcBlwHpJO4AnaCVsgBOB8yU9R6uk+2HbT3SK1zUZR0QMk34+8rS9Gdg87ti5bft7gFMnuO8bwDemEivJOCIaZaR/ZYpSJRlHRKMM67TiJOOIaJR+1YzLlmQcEY0ymjJFRET1UqaIiKiBlCkiImogPePJ7N838BATmnVw6SE3vHn85JxyrP7O+kri8tzeauJW0PW5bsW/Kj0mwIpvV/Rvu2dPNXH7IEPb6qSCRBwR9TBrOHNxQ5NxRMxYKVNERNTACMPZNU4yjohGyWiKiIgaSJkiIqIG0jOOiKiBPi4uX6ok44holJQpIiJqIGWKiIgayNC2iIgaSM84IqIGRpOMIyKqN6xlimk/eJT0wX42JCKiH0bU+1YnBzIK5NOTnZC0VtI2SdvWXXXtAYSIiJiakSlsddKxTCHp/052Cnj5ZPfZXgesAxjbutnTbl1ExBTVrMPbs24145cDbwOeHHdcwF8PpEUREQegqYvLXwMcavvO8Sck/Z+BtCgi4gDUrfzQq47J2PaaDueq+Y6hiIgOhrNfnKFtEdEwamiZIiJiqAxnKk4yjoiGaWTNOCJi2AxplSLJOCKaZVinQycZR0SjDGcqTjKOiIap25oTvUoyjohG0ZD2jQefjEfLz/c3vfWM0mMCnLbpLyqJe/Pb/7CSuCdc/YVK4l5z8odKj/mue75XekyA7/6zN1cS98Tr/6qSuP3Qz1QsaQXweWAUuNT2Z8ednw18GTgOeBw4zfZDbedfCdwLnGf7wk6xhnUUSETEhPq1hKakUeAS4GRgCXC6pCXjLlsDPGn7aOAi4IJx5z8H9LR0ZZJxRDTKCOp562IZsMP2Ttt7gQ3AynHXrASuKPY3AstVTAGUdArwILC9t3ZHRDSIprK1rb1ebGvb3mo+8HDb613FMSa6xvY+4ClgnqRDgU/QYd338fIALyIaZSqTPtrXXu+z84CLbO/uda2MJOOIaJQ+PsB7BDiq7fWC4thE1+ySNAuYS+tB3vHAKkn/CTgCGJO0x/bFkwVLMo6IRunj0LatwGJJi2gl3dXA+KWDNwFnADcDq4AbbRv4579uj3QesLtTIoYk44homNE+5WLb+ySdBVxPa2jb5ba3Szof2GZ7E3AZsF7SDuAJWgl7WpKMI6JR+jnO2PZmYPO4Y+e27e8BTu3yHuf1EivJOCIaJTPwIiJqIEtoRkTUwLBOnujabkmvlrS8GMTcfnzF4JoVETE9U5n0UScdk7GkPwb+N/BHwD2S2qcC/sdBNiwiYjpGpJ63OunWM/4QcJztU4A3AX8m6ezi3KSfpH2K4bpvbp7ssoiIvhvWnnG3mvGI7d0Ath+S9CZgo6TfosNnaZ9iOHb7De5TWyMiuup1+nHddOsZ/1TSsc+/KBLzO4GXAq8ZZMMiIqajX0tolq1bz/j9wL72A8XKRO+X9MWBtSoiYppUtyzbo47J2PauDud+2P/mREQcmJEhHduWccYR0SjDWjNOMo6IRhnSXJxkHBHNkp5xREQNDGkuTjKOiGap28y6XiUZR0SjjDRxaFtExLBRhrZFRFRvWB/gqfXdeYNz9bwjS1+b4u3/+cyyQwIwsvy9lcStyo1Lq1lF9S13fKf8oLMOKj8m4EcfrCTuVcv/oJK4733iJwecSR9YsrjnnPPb9/5tbTJ3esYR0SjD2jNOMo6IRhnSXJxkHBHNMprRFBER1UuZIiKiBoY0FycZR0SzJBlHRNRAIxeXj4gYNnmAFxFRAylTRETUQEZTRETUwJDm4u7JWNIywLa3SloCrAB+ZHvzwFsXETFFjewZS/oUcDIwS9K3geOBm4BzJL3W9n8ooY0RET0b0lzctWe8CjgWmA38BFhg+2lJFwJbgAmTsaS1wFqAj8w5nBWHzOlfiyMiOhgZHc5s3C0Z77O9H3hG0gO2nwaw/ayksclusr0OWAfVLKEZETPXsJYpuq2Jv1fS893a454/KGkuMGkyjoiozIh637qQtELS/ZJ2SDpngvOzJV1ZnN8iaWFxfJmkO4vtLknv6drsLudPtP0MgO325HsQcEbXTxIRUTap963j22gUuITWc7MlwOnFIIZ2a4AnbR8NXARcUBy/B1hq+1hagx6+KKljJaJjMrb9q0mOP2b77o6fJCKiApJ63rpYBuywvdP2XmADsHLcNSuBK4r9jcBySbL9jO19xfFDgK7l2iH96r6IiEmMjvS8SVoraVvbtrbtneYDD7e93lUcY6JriuT7FDAPQNLxkrYDdwMfbkvOE8qkj4holKksFNQ+2KDfbG8BjpH0T4ArJF1re89k16dnHBHN0qeaMfAIcFTb6wXFsQmvKWrCc4HH2y+wfR+wG/jdTsGSjCOiUTSinrcutgKLJS2SdDCwGtg07ppN/GYwwyrgRtsu7pkFIOm3gFcDD3UKljJFRDRLn8YZ294n6SzgemAUuNz2dknnA9tsbwIuA9ZL2gE8QSthA7yR1kzl52gNA/6I7cc6xUsyjohm6eN6xsUaPJvHHTu3bX8PcOoE960H1k8lVpJxRDSKRoez+ppkHBHNMqTToQeejN9x3WWDDvH/ufmdZ5YeE+CEm06qJG5V3nL396oJvPfZ8mOOHFJ+TOC7b6tmout7bvpKJXH7QcPZMU7POCIaJj3jiIjq5duhIyLqID3jiIjqZTRFREQdpEwREVEDKVNERFRvWL92Kck4IpolZYqIiOrlAV5ERB2kTBERUb1hnfQx5f68pC8PoiEREX3Rv2/6KFXHnrGk8avaC3izpCMAbL97UA2LiJiWhvaMFwBPA58D/kux/aJtf0Lt37i67qpr+9XWiIiuJPW81Um3mvFS4Gzgk8C/tX2npGdtf7fTTe3fuDq2dbP70tKIiF40cTSF7THgIklfL/78abd7IiIqVbMeb696Sqy2dwGnSnoHrbJFREQ9NTkZP8/2t4BvDagtEREHbqSBZYqIiKEzE3rGERG1l2QcEVEDo6NVt2BakowjolnSM46IqIEk44iIGkgyjoiogQxti4iogSTjSezfP/AQ451wzRdLjwnwwzefVknc13/5M5XEZfeT1cQdKf9p+caTPlB6TIBVN36lkrg8+0w1cfshZYqIiOopPeOIiBpIzzgiogaSjCMiaiDJOCKiBoZ0OvRwVrojIibTxy8klbRC0v2Sdkg6Z4LzsyVdWZzfImlhcfwkSbdJurv48y3dYiUZR0Sz9CkZSxoFLgFOBpYAp0taMu6yNcCTto8GLgIuKI4/BrzL9muAM4D13ZqdZBwRzTIy0vvW2TJgh+2dtvcCG4CV465ZCVxR7G8ElkuS7Tts/7g4vh14kaTZHZs9pQ8ZEVF3U+gZt3+TfbGtbXun+cDDba93FceY6Brb+4CngHnjrnkvcLvtX3Vqdh7gRUSzTGE0Rfs32Q+mKTqGVunird2uTTKOiGbp32iKR4Cj2l4vKI5NdM0uSbOAucDjAJIWAFcB77f9QLdgUypTSHqjpI9L6prlIyIq0b/RFFuBxZIWSToYWA1sGnfNJloP6ABWATfatqQjaH158zm2f9hLszsmY0m3tu1/CLgYOAz41ETDPCIiKtenZFzUgM8CrgfuA75me7uk8yW9u7jsMmCepB3Ax4Hn8+JZwNHAuZLuLLZ/1CletzLFQW37a4GTbP9c0oXALcBnJ7qpKIKvBfjCJz7C2lNWdAkTEdEnfVwoyPZmYPO4Y+e27e8BTp3gvj8H/nwqsbol4xFJL6HVg5btnxeBfilp32Q3tRfFx2652lNpUETEAWnodOi5wG2AAEs60vajkg4tjkVE1EsF6133Q8dkbHvhJKfGgPf0vTUREQdqZDj7idMa2mb7GeDBPrclIuLAaTjnsmWccUQ0S0NrxhERwyVfuxQRUQPpGUdE1EATR1NERAydlCkiImogZYqIiBrI0LaIiBqYSZM+pqSKv5hZB5cfE3jDNQNbo7qjW991ZiVxl137pUri/vWKD5Qec9UN1XxWxvZXE/fgjt8QVG95gBcRUQMpU0RE1EDKFBERNZDRFBERNZAyRUREDaRMERFRAxlNERFRAylTRETUQMoUERE1kJ5xREQNZGhbREQNZAnNiIgaGNLRFB1/hEg6XtLhxf6LJH1a0tWSLpA0t5wmRkRMgdT7ViPd+vOXA88U+58H5gIXFMcmXcZK0lpJ2yRtW3fVdX1paERET0ZGet9qpFuZYsT2vmJ/qe3XFfs/kHTnZDfZXgesAxi79RofeDMjInpUsx5vr7r9aLhH0geL/bskLQWQ9CrguYG2LCJiOjTS+1Yj3XrGfwh8XtKfAo8BN0t6GHi4OBcRUS9D+gCvYzK2/RTwgeIh3qLi+l22f1pG4yIipqzJM/BsPw3cNeC2REQcuJqVH3qVccYR0SxD+gAvyTgimmVIe8bD2eqIiElI6nnr4b1WSLpf0g5J50xwfrakK4vzWyQtLI7Pk3STpN2SLu6l3UnGEdEsI7N63zqQNApcApwMLAFOl7Rk3GVrgCdtHw1cRGtSHMAe4M+Af9Nzs3u9MCJiKIyo962zZcAO2ztt7wU2ACvHXbMSuKLY3wgslyTbv7T9A1pJubdm93phRMRQmMKkj/alG4ptbds7zac1p+J5u4pjTHRNMVv5KWDedJqdB3gR0SxTGE3RvnRD1ZKMI6JZ+jea4hHgqLbXC4pjE12zS9IsWoupPT6dYINPxlVMTXyu5zJNf1U0DXPZty6tJO4d71hTSdzXX/2Xpcf07n8oPSaADntJJXEZOaiauP3Qv3HGW4HFkhbRSrqrgfeNu2YTcAZwM7AKuNH2tBZHS884IppltD+dItv7JJ0FXA+MApfb3i7pfGCb7U3AZcB6STuAJ2glbAAkPQQcDhws6RTgrbbvnSxeknFENEsfJ33Y3gxsHnfs3Lb9PcCpk9y7cCqxkowjolkyHToiogaGdDp0knFENEt6xhERNTA6nGltOFsdETGJXhYAqqMk44holtSMIyJqID3jiIgaSM84IqIGhrRn3PFHiKQ/lnRUp2siImpldLT3rUa69ec/A2yR9H1JH5H0sjIaFRExbVNYz7hOurVmJ61l4z4DHAfcK+k6SWdIOmyym9oXbF73zWv72NyIiC6k3rca6VYztu0x4AbgBkkH0fo+qNOBC4EJe8rtCzaPbbt2WsvJRURMT72SbK+6JeMXfCrbz9Fav3OTpDkDa1VExHTVrMfbq27J+LTJTth+ps9tiYg4cE1Mxrb/pqyGRET0Rc0ezPUq44wjolmGs2OcZBwRTTOc2TjJOCKapYk144iIoZNkHBFRA3mAFxFRB+kZR0RUL2WKiIgaSDKehCtYmmL0oPJjAoztryZuRUsBHnvlhZXEZX/5f8+a+9LSYwKwb281cYdaknFEROXyhaQREXWQ0RQRETWQnnFERA0kGUdE1EGScURE9dIzjoiogeHMxUnGEdEwGU0REVEDKVNERNTBcCbj4ezPR0RMRup96/pWWiHpfkk7JJ0zwfnZkq4szm+RtLDt3J8Ux++X9LZusTomY0kHS3q/pN8vXr9P0sWSPiqpogUgIiI66FMyljQKXAKcDCwBTpe0ZNxla4AnbR8NXARcUNy7BFgNHAOsAP578X6T6lam+FJxzRxJZwCHAt8ElgPLgDO63B8RUa7+PcBbBuywvRNA0gZgJXBv2zUrgfOK/Y3AxWotjrES2GD7V8CDknYU73fzZMG6JePX2P6nkmYBjwCvsL1f0v8A7prsJklrgbXFyzNtr+sSZ9L3me69B2ImxZ1Jn3WmxZ1Jn/UF5sztuWg8LlcBrGtr+3zg4bZzu4Djx73Fr6+xvU/SU8C84vgt4+6d36kt3X6EjEg6GDgMmAPMLY7PBiYtU9heZ3tpsR3IP8ra7pcMxEyKO5M+60yLO5M+67SMy1UHmq8OSLee8WXAj4BR4JPA1yXtBH4P2DDgtkVEVOkR4Ki21wuKYxNds6uoIMwFHu/x3hfo2DO2fRHwRuAE2/8NeC9wPbDG9qe7fpSIiOG1FVgsaVFRIVgNbBp3zSZ+8+xsFXCjbRfHVxejLRYBi4FbOwXrOs7Y9o/b9v+BVpG6LFX9yjCT4s6kzzrT4s6kz9p3RQ34LFod0FHgctvbJZ0PbLO9iVb1YH3xgO4JWgmb4rqv0XrYtw/4qO2OX1EjV/G1SBER8QKZ9BERUQNJxhERNVDbZNxtGuKAYl4u6WeS7ikjXhHzKEk3SbpX0nZJZ5cU9xBJt0q6q4hb2gNZSaOS7pB0TVkxi7gPSbpb0p2StpUU8whJGyX9SNJ9kk4oIebvFJ/x+e1pSR8bdNwi9r8u/j/dI+mrkg4pI24T1LJmXEwb/BvgJFqDpbcCp9u+t+ONBx73RGA38GXbvzvIWG0xjwSOtH27pMOA24BTSvisAl5se3cxtf0HwNm2b+lyaz9ifxxYChxu+52DjtcW9yFgqe3HSox5BfB925cWT+TnFA/Cy4o/SmtI1fG2/27AsebT+n+0xPazxQOszbb/apBxm6KuPeNfT0O0vZfWmOaVgw5q+3u0noiWxvajtm8v9n8B3EeXmTp9imvbu4uXBxXbwH8yS1oAvAO4dNCxqiZpLnAirSfu2N5bZiIuLAceGHQibjMLeFEx5nYO8OMu10ehrsl4ommIA09QVStWfHotsKWkeKOS7gR+Bnzbdhlx/yvw74CxEmKNZ+AGSbcV02AHbRHwc+BLRVnmUkkvLiFuu9XAV8sIZPsR4ELg74FHgads31BG7CaoazKecSQdCnwD+Jjtp8uIaXu/7WNpzQ5aJmmgpRlJ7wR+Zvu2Qcbp4I22X0drFa6PFmWpQZoFvA74gu3XAr8ESnn+Aa1VF4F3A18vKd5LaP0Guwh4BfBiSX9QRuwmqGsynvJUwmFW1Gy/AXzF9jfLjl/86nwTraX+BukNwLuL2u0G4C3FolOlKHpu2P4ZcBWtctgg7QJ2tf3GsZFWci7LycDttn9aUrzfBx60/XPbz9Fa4fH1JcUeenVNxr1MQ2yE4kHaZcB9tj9XYtyXSTqi2H8RrYelPxpkTNt/YnuB7YW0/k1vtF1Kz0nSi4sHpBSlgrcCAx01Y/snwMOSfqc4tJwXLr84aKdTUomi8PfA70maU/y/Xk7rGUj0oJZfuzTZNMRBx5X0VeBNwEsl7QI+ZfuyAYd9A/AvgbuL+i3Av7e9ecBxjwSuKJ62jwBfs13qULOSvRy4qpUjmAX8T9vXlRD3j4CvFJ2KncAHS4j5/A+ck4Azy4gHYHuLpI3A7bSmAN9BQ6ZGl6GWQ9siImaaupYpIiJmlCTjiIgaSDKOiKiBJOOIiBpIMo6IqIEk44iIGkgyjoiogf8H3l9+q4EFsKMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knowledge uncertainty via ensemble:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAD8CAYAAACihcXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAU4ElEQVR4nO3df7CmZX3f8ffnnAV0QdaIKSUsCjMQDdoZfzBgq3VsEAMT4pIGR0gb0VC3TiTROmlDmsafaWfoJMGMMq0nsgGJFXTV6VaJaIJJ1EbcBSGyIHZFIguKIrgUVwLLfvvHcy89nOw5z3POeX7cz33er5l79n7uH+d7PbM733Pt976u605VIUmarJlJN0CSZDKWpFYwGUtSC5iMJakFTMaS1AImY0lqAZOxJLXAun4XJHkusAk4tjl0D7Ctqm4fZcMkaS1Zsmec5LeAq4EAX2m2AB9JcvHomydJa0OWmoGX5BvA86rqsQXHDwV2VtVJi9y3GdgM8IH3vffFm3/19UNrsKQOW78hq/0Rb8qRA08r/u/10KrjDUu/MsV+4KeAv1tw/Jjm3EFV1RwwB8DePc63lqQ++iXjtwJ/keT/AHc3x54FnAhcNMqGSdJKTOuohCWTcVV9JslPA6fy5Ad426vq8VE3TpKWa11aU3lYlr6jKapqP/DlMbRFklZtZjpzcf9kLEnTpJNlCkmaNjNdLVNI0jSxZyxJLWDNWJJaYNYyhSRNnmUKSWoByxSS1AL2jCWpBRzaJkktsG46c7HJWFK3WKaQpBaYYTq7xiZjSZ3iaApJagHLFJLUAvaMJakFOru4vCRNE8sUktQClikkqQUc2iZJLWDPWJJaYHZKk/G01rol6aBmyMBbP0nOTHJHkl1JLj7I+cOSXNOcvyHJ8QvOPyvJw0l+s3+7VyjJG1Z6rySNykwG35aSZBa4DDgLOBk4P8nJCy67EHiwqk4ELgUuWXD+D4E/G6jdg1y0iHctdiLJ5iQ7kuyY23LFKkJI0vLMLGPr41RgV1XdWVWPAlcDmxZcswm4stnfCpye9AY6JzkH+Bawc5B2L1kzTvK3i50Cjl7svqqaA+YA2LunBmmIJA3DckrGSTYDm+cdmmvyF8CxwN3zzu0GTlvwI564pqr2JdkDHJXkEeC3gDOAviUK6P8A72jg54AHF34H4H8PEkCSxmk5i8s/qeM4XO8ELq2qhzNge/ol408BR1TVzQtPJPnL5bZOkkZtiKMS7gGOm/d5Y3PsYNfsTrIO2AD8gF4P+twk/xV4OrA/ySNV9f7Fgi2ZjKvqwiXO/fJS90rSJAxxZNt24KQkJ9BLuucBC/PeNuAC4G+Ac4Hrq6qAf/5Ee5J3Ag8vlYjBccaSOmbQskA/TQ34IuA6YBbYUlU7k7wb2FFV24DLgauS7AIeoJewVyS9JD5CPsCTNKj1G1adST/xjH88cM75lw98tzVTROwZS+qUaZ3JZjKW1ClTupyxyVhSt7hqmyS1wHSmYpOxpI5xCU1JaoFMad/YZDxE9dD9E4mbI585kbhryaNvPncicQ+9bOtE4k6z6UzFJmNJHWOZQpJawNEUktQC05mKTcaSOsZJH5LUAlOai03GkrrFoW2S1AKz05mLTcaSumVKc7HJWFK3WKaQpBZwNIUktcC0Li7ft91Jnpvk9CRHLDh+5uiaJUkrk2VsbbJkMk7yG8D/BH4duDXJpnmn/8soGyZJKzGTDLy1Sb+e8RuBF1fVOcArgN9N8pbm3KLfJMnmJDuS7JjbcsVQGipJg5jWnnG/mvFMVT0MUFV3JXkFsDXJs1niu1TVHDAH+HZoSWOVlvV4B9WvZ3xfkhcc+NAk5rOBZwL/ZJQNk6SVmMngW5v06xm/Dtg3/0BV7QNel+QDI2uVJK1Q2pZlB7RkMq6q3Uuc+9LwmyNJqzMzpWPbHGcsqVOmtWZsMpbUKVOai03GkrrFnrEktcCU5mKTsaRuadvMukGZjCV1ykwXh7ZJ0rSJQ9skafJ8gNciD539ionEfezHj00k7qQc9ReTmffzpsOPG3vMy963eewxAf746BMnEveN9+2aSNxhmNJc3M1kLGntsmcsSS0wpbnYZCypW2YdTSFJkzetZYopHQQiSQeXDL71/1k5M8kdSXYlufgg5w9Lck1z/oYkxzfHT01yc7PdkuQX+8UyGUvqlGEl4ySzwGXAWcDJwPlJTl5w2YXAg1V1InApcElz/FbglKp6AXAm8IEkS1YiTMaSOiUzGXjr41RgV1XdWVWPAlcDmxZcswm4stnfCpyeJFW1t3kRB8BTgL6vnzMZS+qU2ZkMvM1/eXKzzR9Qfixw97zPu5tjHOyaJvnuAY4CSHJakp3A14A3zUvOB+UDPEmdspznd096efKQVdUNwPOS/AxwZZI/q6pHFrvenrGkTkky8NbHPcD86Z4bm2MHvaapCW8AfjD/gqq6HXgYeP5SwUzGkjpliKMptgMnJTkhyaHAecC2BddsAy5o9s8Frq+qau5Z12tPng08F7hrqWB9yxRJTgWqqrY3TxLPBL5eVdf2/SqSNGbDGmdcVfuSXARcB8wCW6pqZ5J3AzuqahtwOXBVkl3AA/QSNsDLgIuTPAbsB36tqu5fKt6SyTjJO+gN61iX5HPAacDnmyAvrKr/vOJvKkkjMMw5H02n89oFx94+b/8R4DUHue8q4KrlxOpXpjgXeCnwcuDNwDlV9R7g54DXLnbT/CeUc1uuWE57JGlVZmYz8NYm/coU+6rqcWBvkm9W1UMAVfXjJPsXu+lJTyj37uk7vk6ShmVap0P3S8aPJllfVXuBFx84mGQDvTqIJLVLRxcKenlV/T1AVc1Pvofw/58gSlJ7dLFnfCARH+T4/cCSTwYlaRK6WqaQpOkyO53TJ0zGkjplgAWAWslkLKlbLFNI0uTZM5akNrBnLEktYM9YkiYvjqaQpBaY0jJFqka8dMQaWpvi8S9/atJNGK/7vzuZuA98f+whd/3Bx8YeE+A5t9w0kbgTs37DqjPp3l966cA5Z/3Hv9SazG3PWFK3TGnP2GQsqVMc2iZJbWDPWJImz9EUktQGlikkqQUsU0jS5LmesSS1gWUKSZo8H+BJUhtYppCkyZvWSR/L7s8n+dAoGiJJQ5EMvrXIkj3jJNsWHgL+RZKnA1TVq0fVMElakSntGfcrU2wEbgM+CBS9ZHwK8AdL3ZRkM7AZ4APvey+bf/X1q26oJA2iq0PbTgHeAvwO8O+r6uYkP66qv1rqpqqaA+aANbWEpqQW6OJoiqraD1ya5GPNn/f1u0eSJqqjPWMAqmo38JokPw88NNomSdIqdDkZH1BVnwY+PaK2SNLqzXSwTCFJU2ct9IwlqfVMxpLUArOzk27BipiMJXWLPWNJagGTsSS1wJQm4+kcAyJJi5mZGXzrI8mZSe5IsivJxQc5f1iSa5rzNyQ5vjl+RpIbk3yt+fNn+8WyZyypW4Y0zjjJLHAZcAawG9ieZFtV3TbvsguBB6vqxCTnAZcArwXuB36hqu5N8nzgOuDYpeKZjIdo9iVnTybw3slMitz/7dsnEjfPOHrsMZ/zut8ee0yt0PDKFKcCu6rqzt6PzdXAJnqLpx2wCXhns78VeH+SVNVX512zE3hqksOq6u8XC2aZQlKnZGZm4K2PY4G7533ezT/s3T5xTVXtA/YARy245peAm5ZKxGDPWFLXLKNnPH+538Zcs+rkkJqS59ErXbyq37UmY0ndsoxk/KTlfv+he4Dj5n3e2Bw72DW7k6wDNgA/6DUjG4FPAq+rqm/2a4tlCkndMrzXLm0HTkpyQpJDgfOAhW8/2gZc0OyfC1xfVdW8DenTwMVV9aVBmm0yltQts7ODb0toasAX0RsJcTvw0arameTdSQ68cu5y4Kgku4C3AQeGv10EnAi8PcnNzfaPlopnmUJStwxx0kdVXQtcu+DY2+ftPwK85iD3/R7we8uJZTKW1C1TOgPPZCypW1xcXpJawJ6xJLWAyViSWmAtLC6f5GX05mvfWlWfHU2TJGkVprRnvGSlO8lX5u2/EXg/8DTgHQdbTk6SJm54kz7Gqt9jx0Pm7W8Gzqiqd9GbZ/2vFrspyeYkO5LsmNtyxepbKUmDGuJ6xuPUr0wxk+Qn6CXtVNX3AarqR0n2LXbTk+Z7791TQ2qrJPXXsh7voPol4w3AjUCASnJMVX0nyRHNMUlql5kOPsCrquMXObUf+MWht0aSVmtmOvuJKxraVlV7gW8NuS2StHppVy14UI4zltQtHa0ZS9J0adkoiUGZjCV1iz1jSWqBLo6mkKSpY5lCklrAMoUktYBD2ySpBdbSpA+1zPojJxL2hxe9bSJxn/HnA735XGuVD/AkqQUsU0hSC1imkKQWcDSFJLWAZQpJagHLFJLUAo6mkKQWsEwhSS1gmUKSWsCesSS1gEPbJKkFXEJTklpgSkdTLPkrJMlpSY5s9p+a5F1J/leSS5JsGE8TJWkZksG3FunXn98C7G32/wjYAFzSHPuTxW5KsjnJjiQ75rZcMYx2StJgZmYG31qkX5lipqr2NfunVNWLmv0vJrl5sZuqag6YA2Dvnlp1KyVpUC3r8Q6q36+GW5O8odm/JckpAEl+GnhspC2TpJXIzOBbi/TrGf8b4I+S/CfgfuBvktwN3N2ck6R2mdIHeEsm46raA7y+eYh3QnP97qq6bxyNk6Rlm9IZeAP106vqoaq6papuNBFLarUhlimSnJnkjiS7klx8kPOHJbmmOX9DkuOb40cl+XySh5O8f5Bmt6toIkmrNaShbUlmgcuAs4CTgfOTnLzgsguBB6vqROBSeqPNAB4Bfhf4zUGbbTKW1C3D6xmfCuyqqjur6lHgamDTgms2AVc2+1uB05Okqn5UVV+kl5QHYjKW1ClJlrM9MSei2TbP+1HH0huscMDu5hgHu6YZBrwHOGol7XY6tKRumRk8rT1pTsSEmYwldcvwRlPcAxw37/PG5tjBrtmdZB29Wco/WEkwyxSSumV4NePtwElJTkhyKHAesG3BNduAC5r9c4Hrq2pFs47tGUvqliFNh66qfUkuAq4DZoEtVbUzybuBHVW1DbgcuCrJLuABegm7aUbuAo4EDk1yDvCqqrptsXgmY0ndMsRpzlV1LXDtgmNvn7f/CPCaRe49fjmxTMZasWd87guTbsL47H98MnGndGrvRE3pQkEmY0ndMjudv8BMxpK6pWWrsQ3KZCypWyxTSFIL2DOWpBawZyxJLTA7nWltOlstSYuIPWNJagFrxpLUAvaMJakF7BlLUgtMac94yV8hSX4jyXFLXSNJrTI7O/jWIv368+8BbkjyhSS/luQnx9EoSVqxIb4depz6teZOeqvbvwd4MXBbks8kuSDJ0xa7af57pea2XDG81kpSP0N6O/S4ZalF6ZPcVFUvmvf5EHqvrT4feGVV9e8p792zolXvNQVq/2TiTqJH4xKa47F+w6ozZH1758A5J896Xmsycr8HeE9qaFU9Ru81I9uSrB9ZqyRppVrW4x1Uv2T82sVOVNXeIbdFklavi8m4qr4xroZI0lC07MHcoBxnLKlbprNjbDKW1DXTmY1NxpK6pYs1Y0maOiZjSWoBH+BJUhvYM5akybNMIUktYDLWmjOltbkVWWtrREw1k7EkTZwvJJWkNpjS/7GZjCV1iz1jSWoBk7EktYHJWJImz56xJLXAdOZik7GkjnE0hSS1gGUKSWoDk7EkTV4Xe8ZJDgXOA+6tqj9P8svAPwNuB+aq6rExtFGSBjelyThVtfjJ5MP0EvZ64IfAEcAngNObey/oG2HvnsUDSNJ86zesPpMuJ+cMI96wVNWiG/C3zZ/rgPuA2eZzDpxb5L7NwI5m27xUjD7xV3zvara1FHctfde1FnctfdcubP16xrcCLwIOB74NPLuqHkjyFOCrVfUzw/mVsGj8HVV1yihjrPW4a+m7rrW4a+m7dkG/B3iXA18HZoHfAT6W5E7gJcDVI26bJK0ZSybjqro0yTXN/r1JPgS8EvjjqvrKOBooSWtB36FtVXXvvP0fAltH2qInmxtjrLUady1917UWdy1916m3ZM1YkjQe0zmJW5I6xmQsSS3Q2mSc5MwkdyTZleTiMcXckuR7zZC+sUhyXJLPJ7ktyc4kbxlT3Kck+UqSW5q47xpH3Cb2bJKvJvnUuGI2ce9K8rUkNyfZMaaYT0+yNcnXk9ye5J+OIeZzmu94YHsoyVtHHbeJ/e+af0+3JvlIMwxWA2hlzTjJLPAN4AxgN7AdOL+qbhtx3JcDDwMfqqrnjzLWvJjHAMdU1U1JngbcCJwzhu8a4PCqejjJIcAXgbdU1ZdHGbeJ/TbgFODIqjp71PHmxb0LOKWq7h9jzCuBL1TVB5vlBdY3D8LHFX8WuAc4rar+bsSxjqX37+jkqvpxko8C11bVFaOM2xVt7RmfCuyqqjur6lF6Y5o3jTpoVf018MCo4yyI+Z2quqnZ/7/01v04dgxxq6oebj4e0mwj/82cZCPw88AHRx1r0pJsAF5Ob7w+VfXoOBNx43Tgm6NOxPOsA56a5MAyCvf2uV6NtibjY4G7533ezRgS1KQlOR54IXDDmOLNJrkZ+B7wuaoaR9z3Av8B2D+GWAsV8NkkNybZPIZ4JwDfB/6kKct8MMnhY4g733nAR8YRqKruAX6f3mzd7wB7quqz44jdBW1NxmtOkiOAjwNvraqHxhGzqh6vqhcAG4FTk4y0NJPkbOB7VXXjKOMs4WVV9SLgLODNTVlqlNbRW07gv1XVC4EfAWN5/gFPrLr4auBjY4r3E/T+B3sC8FPA4Un+9Thid0Fbk/E9wHHzPm9sjnVSU7P9OPDhqvrEuOM3/3X+PHDmiEO9FHh1U7u9GvjZJH864phPaHpuVNX3gE/SK4eN0m5g97z/cWyll5zH5Szgpqq6b0zxXgl8q6q+X73ldT9Bb8ldDaCtyXg7cFKSE+atqbxtwm0aieZB2uXA7VX1h2OM+5NJnt7sP5Xew9KvjzJmVf12VW2squPp/Z1eX1Vj6TklObx5QEpTKngVMNJRM1X1XeDuJM9pDp0OjPTB7ALnM6YSRePbwEuSrG/+XZ9O7xmIBtDKN31U1b4kFwHX0VukaEtV7Rx13CQfAV4BPDPJbuAdVXX5iMO+FPgV4GtN/RbgP1bVtSOOewxwZfO0fQb4aFWNdajZmB0NfLKXI1gH/I+q+swY4v468OGmU3En8IYxxDzwC+cM4N+OIx5AVd2QZCtwE7AP+CpOjR5YK4e2SdJa09YyhSStKSZjSWoBk7EktYDJWJJawGQsSS1gMpakFjAZS1IL/D+OZVGZlEan4QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = np.mean(ens_preds, axis=0)[:, 1] # average estimated data uncertainty\n",
    "knowledge = np.var(ens_preds, axis=0)[:, 0] # estimated knowledge uncertainty\n",
    "\n",
    "print(\"Average predicted data uncertainty:\")            \n",
    "sns.heatmap(data.reshape([num_cat, num_cat]), cmap=\"Reds\", vmin=0, vmax=0.05)\n",
    "# plt.savefig(\"heart_data_sgb.pdf\")\n",
    "plt.show()\n",
    "\n",
    "print(\"Knowledge uncertainty via ensemble:\")            \n",
    "sns.heatmap(knowledge.reshape([num_cat, num_cat]), cmap=\"Reds\")\n",
    "# plt.savefig(\"heart_knowledge_sgb.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CatBoostRegressor(iterations=1000, learning_rate=0.1, loss_function='RMSEWithUncertainty', \n",
    "                          subsample=0.5, bootstrap_type='Bernoulli', random_seed=0, verbose=False)\n",
    "model.fit(train_pool, eval_set=val_pool, use_best_model=False)\n",
    "preds = model.virtual_ensembles_predict(test, prediction_type='TotalUncertainty', virtual_ensembles_count=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average predicted data uncertainty:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWMAAAD8CAYAAACihcXDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAYDklEQVR4nO3df7RdZX3n8ffn3kAE0eBC60KCkilYG7QLJQ06Mow1RUNxCF1CCawp6ESjo1SpqzPSsSJiZ5Z0MWacBUUzgEXGIWCUzoVGoBbakQ6EhF8DAakh0HKjBflRECEkIZ/542zwcHvvOefenLP3Pvt+XmvtlX32fvb5PpuE733us5/n2bJNRERUa6TqCkRERJJxREQtJBlHRNRAknFERA0kGUdE1ECScUREDSQZR0RMQdJSSfdL2izpzEnOz5V0RXF+vaSDiuMHSXpO0p3F9rVuseb0UJm3AMuAA4pDW4Ex2/dN56YiIoaJpFHgAuBoYBzYIGnM9r1txVYAT9o+WNJy4FzgpOLcA7YP6zVex5axpM8CawABtxabgMsn+ykREdEgi4HNtrfY3k4rFy6bUGYZcGmxvxZYIkkzCdatZbwCONT2jvaDkr4CbAK+PNlFklYCKwEu/NwZh6/84LEzqdvMeVe58V6KW9Fsxh3bq4k7d69q4j7/XPkx99iz/JhVxq3ivzEw8uu/NaNE1u7jenXP/yN+nZ99jCJXFVbbXl3sHwA83HZuHDhiwle8VMb2TklPAfsV5xZIugN4Gvgj2z/oVJduyXgX8Abg7ycc3784N6niZlYD7Lrj+5lvHRG11J6r+uwnwBttPy7pcODPJR1q++mpLuiWjM8A/krSj/jFT4g3AgcDp/ejxhER/dTHUQlbgQPbPs8vjk1WZlzSHGAe8Lhbi/48D2D7NkkPAG8GNk4VrGMytn2tpDfT6jtpf4C3wfYLPd9SRERJ5sysy3YyG4BDJC2glfeWA6dMKDMGnAbcDJwA3GDbkl4HPGH7BUn/AjgE2NKx3t1qY3sXcMu0byMiogIjfcrFRR/w6cB1wChwie1Nks4BNtoeAy4GLpO0GXiCVsIGOAo4R9IOWl26H7f9RKd4XZNxRMQw6efkCdvrgHUTjp3Vtr8NOHGS674DfGc6sZKMI6JRRvrXTVGqJOOIaJRhnVacZBwRjdKvPuOyJRlHRKOMppsiIqJ66aaIiKiBdFNERNRAWsZT2bFt4CEmtccrSg+56siTuhcagN+/6fJK4rKzogWKKmj6XPv+D5UeE2DpdX9WSdzKFtvqgwxtq5MKEnFE1MOc4czFDU3GETFrpZsiIqIGRhjOpnGScUQ0SkZTRETUQLopIiJqIC3jiIga6OPi8qVKMo6IRkk3RUREDaSbIiKiBjK0LSKiBtIyjoiogdEk44iI6g1rN8WMHzxK+nA/KxIR0Q8j6n2rk90ZBfLFqU5IWilpo6SNq6+6djdCRERMz8g0tjrp2E0h6f9NdQp4/VTX2V4NrAbYdes1nnHtIiKmqWYN3p516zN+PfB+4MkJxwX834HUKCJiNzR1cflrgH1s3znxhKS/HkiNIiJ2Q926H3rVMRnbXtHh3Cn9r05ExO4ZznZxhrZFRMOood0UERFDZThTcZJxRDRMI/uMIyKGzZD2UiQZR0SzDOt06CTjiGiU4UzFScYR0TB1W3OiV0nGEdEoGtK28eCT8egeAw8x0fffd2rpMQHOuP7rlcS96Zh/V0ncI6/+WiVxr1r6kdJjfvBHG0uPCfDgv35vJXHf9K0LK4nbD/1MxZKWAl8FRoGLbH95wvm5wDeBw4HHgZNsP9R2/o3AvcDZts/rFGtYR4FEREyqX0toShoFLgCOARYCJ0taOKHYCuBJ2wcDq4BzJ5z/CvC9nurdS6GIiGExgnreulgMbLa9xfZ2YA2wbEKZZcClxf5aYImKKYCSjgceBDb1Vu+IiAbRdLa2tdeLbWXbVx0APNz2ebw4xmRlbO8EngL2k7QP8Fk6rPs+UR7gRUSjTGfSR/va6312NrDK9jO9rpWRZBwRjdLHB3hbgQPbPs8vjk1WZlzSHGAerQd5RwAnSPoTYF9gl6Rtts+fKliScUQ0Sh+Htm0ADpG0gFbSXQ5MXDp4DDgNuBk4AbjBtoF/9VJ9pLOBZzolYkgyjoiGGe1TLra9U9LpwHW0hrZdYnuTpHOAjbbHgIuByyRtBp6glbBnJMk4Ihqln+OMba8D1k04dlbb/jbgxC7fcXYvsZKMI6JRMgMvIqIGsoRmREQNDOvkia71lvQWSUuKQcztx5cOrloRETMznUkfddIxGUv6FPC/gd8D7pHUPhXwvwyyYhERMzEi9bzVSbeW8UeBw20fD7wH+LykTxfnpryT9imGq7+7bqpiERF9N6wt4259xiO2nwGw/ZCk9wBrJb2JDvfSPsVw123XuU91jYjoqtfpx3XTrWX8iKTDXvxQJOYPAK8F3jbIikVEzES/ltAsW7eW8anAzvYDxcpEp0qqZiX1iIgOVLcs26OOydj2eIdzf9v/6kRE7J6RIR3blnHGEdEow9pnnGQcEY0ypLk4yTgimiUt44iIGhjSXJxkHBHNUreZdb1KMo6IRhlp4tC2iIhhowxti4io3rA+wFPr3XmDc/m+v1T62hS/c8nnyg4JwOiSGb/+aijdsXBxJXHfft/G8oNu31Z+TMDbfl5J3MvfelQlcU/5p0d3O5M+sPCQnnPOL9/7o9pk7rSMI6JRhrVlnGQcEY0ypLk4yTgimmU0oykiIqqXboqIiBoY0lycZBwRzZJkHBFRA41cXD4iYtjkAV5ERA2kmyIiogYymiIiogaGNBd3T8aSFgO2vUHSQmAp8EPb6wZeu4iIaWpky1jSF4BjgDmS/hI4ArgROFPS223/5xLqGBHRsyHNxV1bxicAhwFzgX8E5tt+WtJ5wHpg0mQsaSWwEmDFXvuwZM+9+lfjiIgORkaHMxt3W4Z5p+0XbD8LPGD7aQDbzwG7prrI9mrbi2wvSiKOiDJJ6nmrk27JeLukvYv9w188KGkeHZJxRERlRtT71oWkpZLul7RZ0pmTnJ8r6Yri/HpJBxXHF0u6s9jukvTbXavd5fxRRasY2+3Jdw/gtK53EhFRNqn3rePXaBS4gNZzs4XAycUghnYrgCdtHwysAs4tjt8DLLJ9GK1BD1+X1LFbuGMytv38FMcfs313xzuJiKhAH7spFgObbW+xvR1YAyybUGYZcGmxvxZYIkm2n7W9szj+CqDr20eG9NV9ERFTGB3peZO0UtLGtm1l2zcdADzc9nm8OMZkZYrk+xSwH4CkIyRtAu4GPt6WnCeVSR8R0SjTWSjI9mpg9SDqYXs9cKikXwUulfQ921O+TDEt44holj71GQNbgQPbPs8vjk1apugTngc83l7A9n3AM8BbOwVLMo6IRtGIet662AAcImmBpD2B5cDYhDJj/GIwwwnADbZdXDMHQNKbgLcAD3UKlm6KiGiWPo0ftr1T0unAdcAocIntTZLOATbaHgMuBi6TtBl4glbCBjiS1kzlHbSGAX/C9mOd4iUZR0Sz9HE942INnnUTjp3Vtr8NOHGS6y4DLptOrCTjiGgUjQ5n72uScUQ0S82mOfdq4Mn4pOsvGXSIf2bDcf++9JgAb3v3lZXErcqvXfT5SuLu+IPfLT3m6Ec/VXpMgO//1kcqibv8by6vJG4/aDgbxmkZR0TDpGUcEVG9vB06IqIO0jKOiKheRlNERNRBuikiImog3RQREdWr2+uUepVkHBHNkm6KiIjq5QFeREQdpJsiIqJ6wzrpY9rteUnfHERFIiL6on9v+ihVx5axpImr2gv4DUn7Atg+blAVi4iYkYa2jOcDTwNfAf5rsf2sbX9S7W9cXX3Vtf2qa0REV5J63uqkW5/xIuDTwOeA/2D7TknP2f6bThe1v3F1163XuC81jYjoRRNHU9jeBayS9O3iz0e6XRMRUamatXh71VNitT0OnCjpWFrdFhER9dTkZPwi238B/MWA6hIRsftGGthNERExdGZDyzgiovaSjCMiamB0tOoazEiScUQ0S1rGERE1kGQcEVEDScYRETWQoW0RETWQZDyFXeUvTfHrYxeWHhPgxmM/Vkncbz/2s0rifml8VSVx19zzSOkxX3Plh0qPCXDKD66sJC7bnq0mbj+kmyIionpKyzgiogbSMo6IqIEk44iIGkgyjoiogSGdDj2cPd0REVPp4wtJJS2VdL+kzZLOnOT8XElXFOfXSzqoOH60pNsk3V38+d5usZKMI6JZ+pSMJY0CFwDHAAuBkyUtnFBsBfCk7YOBVcC5xfHHgH9j+23AacBl3aqdZBwRzTIy0vvW2WJgs+0ttrcDa4BlE8osAy4t9tcCSyTJ9h22f1wc3wTsJWlux2pP6yYjIupuGi3j9jfZF9vKtm86AHi47fN4cYzJytjeCTwF7DehzAeB220/36naeYAXEc0yjdEU7W+yH0xVdCitrov3dSubZBwRzdK/0RRbgQPbPs8vjk1WZlzSHGAe8DiApPnAVcCpth/oFmxa3RSSjpT0GUlds3xERCX6N5piA3CIpAWS9gSWA2MTyozRekAHcAJwg21L2pfWy5vPtP23vVS7YzKWdGvb/keB84FXAV+YbJhHRETl+pSMiz7g04HrgPuAK21vknSOpOOKYhcD+0naDHwGeDEvng4cDJwl6c5i+6VO8bp1U+zRtr8SONr2TyWdB9wCfHmyi4pO8JUAF372E6w8fmmXMBERfdLHhYJsrwPWTTh2Vtv+NuDESa77Y+CPpxOrWzIekfQaWi1o2f5pEejnknZOdVF7p/iuW64ufw3NiJi9Gjodeh5wGyDAkva3/RNJ+xTHIiLqZWQ4p0N3TMa2D5ri1C7gt/tem4iI3TUynO3EGQ1ts/0s8GCf6xIRsfs0nHPZMs44IpqloX3GERHDJa9dioiogbSMIyJqoImjKSIihk66KSIiaiDdFBERNZChbRERNTCbJn3U3pw9Kwn7G9d9o5K48479SCVxX3vlmkriHvHOY0uPuWjsa6XHBOD556qJO6QPwYChrXszk3FEzF7ppoiIqIF0U0RE1EBGU0RE1EC6KSIiaiDdFBERNZDRFBERNZBuioiIGkg3RUREDaRlHBFRAxnaFhFRA1lCMyKiBoZ0NEXHHyGSjpD06mJ/L0lflHS1pHMlzSunihER0yD1vtVIt/b8JcCzxf5XgXnAucWxKZcok7RS0kZJG1f/+bV9qWhERE9GRnrfaqRbN8WI7Z3F/iLb7yj2b5J051QX2V4NrAbYdcvV3v1qRkT0qGYt3l51+9Fwj6QPF/t3SVoEIOnNwI6B1iwiYiY00vtWI91axh8Bvirpj4DHgJslPQw8XJyLiKiXIX2A1zEZ234K+FDxEG9BUX7c9iNlVC4iYtqaPAPP9tPAXQOuS0TE7qtZ90OvMs44IpplSB/gJRlHRLMMact4OGsdETEFST1vPXzXUkn3S9os6cxJzs+VdEVxfr2kg4rj+0m6UdIzks7vpd5JxhHRLCNzet86kDQKXAAcAywETpa0cEKxFcCTtg8GVtGaFAewDfg88Ac9V7vXghERQ2FEvW+dLQY2295iezuwBlg2ocwy4NJify2wRJJs/9z2TbSScm/V7rVgRMRQmMakj/alG4ptZds3HUBrTsWLxotjTFammK38FLDfTKqdB3gR0SzTGE3RvnRD1ZKMI6JZ+jeaYitwYNvn+cWxycqMS5pDazG1x2cSbPDJeE4F+X7n9vJjQmVDat5xzf+oJO7t7/pAJXEXXf318oPu3Nm9zCDM3auauEM6iw3o5zjjDcAhkhbQSrrLgVMmlBkDTgNuBk4AbrA9o8XR0jKOiGYZ7c/aFLZ3SjoduA4YBS6xvUnSOcBG22PAxcBlkjYDT9BK2ABIegh4NbCnpOOB99m+d6p4ScYR0Sx9/A3V9jpg3YRjZ7XtbwNOnOLag6YTK8k4Ipol06EjImpgSKdDJxlHRLOkZRwRUQOjw5nWhrPWERFT6GUBoDpKMo6IZkmfcUREDaRlHBFRA2kZR0TUwJC2jDv+CJH0KUkHdioTEVEro6O9bzXSrT3/JWC9pB9I+oSk15VRqYiIGZvGesZ10q02W2gtG/cl4HDgXknXSjpN0qumuqh9webV3/1eH6sbEdGF1PtWI936jG17F3A9cL2kPWi9D+pk4Dxg0pZy+4LNuzZ+b0bLyUVEzEy9kmyvuiXjl92V7R201u8ck7T3wGoVETFTNWvx9qpbMj5pqhO2n+1zXSIidl8Tk7HtvyurIhERfVGzB3O9yjjjiGiW4WwYJxlHRNMMZzZOMo6IZmlin3FExNBJMo6IqIE8wIuIqIO0jCMiqpduioiIGkgynoIrWJpidI/yYwLseqGauCPVLAV42Jo/qSQuO7aXH3PuXuXHBHj+uWri7rFnNXH7Isk4IqJyeSFpREQdZDRFREQNpGUcEVEDScYREXWQZBwRUb20jCMiamA4c3GScUQ0TEZTRETUQLopIiLqYDiT8XC25yMipiL1vnX9Ki2VdL+kzZLOnOT8XElXFOfXSzqo7dwfFsfvl/T+brE6JmNJe0o6VdJvFp9PkXS+pE9KqmgBiIiIDvqUjCWNAhcAxwALgZMlLZxQbAXwpO2DgVXAucW1C4HlwKHAUuBPi++bUrduim8UZfaWdBqwD/BdYAmwGDity/UREeXq3wO8xcBm21sAJK0BlgH3tpVZBpxd7K8FzldrcYxlwBrbzwMPStpcfN/NUwXrlozfZvvXJM0BtgJvsP2CpP8J3DXVRZJWAiuLjx+zvbpLnCm/Z6bX7o7ZFHc23etsizub7vVl9p7Xc6fxhFwFsLqt7gcAD7edGweOmPAVL5WxvVPSU8B+xfFbJlx7QKe6dPsRMiJpT+BVwN7AvOL4XGDKbgrbq20vKrbd+UtZ2b3IQMymuLPpXmdb3Nl0rzMyIVftbr7aLd1axhcDPwRGgc8B35a0BXgnsGbAdYuIqNJW4MC2z/OLY5OVGS96EOYBj/d47ct0bBnbXgUcCbzL9n8HPghcB6yw/cWutxIRMbw2AIdIWlD0ECwHxiaUGeMXz85OAG6w7eL48mK0xQLgEODWTsG6jjO2/eO2/X+i1Uldlqp+ZZhNcWfTvc62uLPpXvuu6AM+nVYDdBS4xPYmSecAG22P0eo9uKx4QPcErYRNUe5KWg/7dgKftN3xVUByFa9FioiIl8mkj4iIGkgyjoiogdom427TEAcU8xJJj0q6p4x4RcwDJd0o6V5JmyR9uqS4r5B0q6S7irilPZCVNCrpDknXlBWziPuQpLsl3SlpY0kx95W0VtIPJd0n6V0lxPyV4h5f3J6WdMag4xaxf7/493SPpMslvaKMuE1Qyz7jYtrg3wFH0xosvQE42fa9HS/c/bhHAc8A37T91kHGaou5P7C/7dslvQq4DTi+hHsV8ErbzxRT228CPm37li6X9iP2Z4BFwKttf2DQ8driPgQssv1YiTEvBX5g+6LiifzexYPwsuKP0hpSdYTtvx9wrANo/TtaaPu54gHWOtt/Nsi4TVHXlvFL0xBtb6c1pnnZoIPa/j+0noiWxvZPbN9e7P8MuI8uM3X6FNe2nyk+7lFsA//JLGk+cCxw0aBjVU3SPOAoWk/csb29zERcWAI8MOhE3GYOsFcx5nZv4Mddykehrsl4smmIA09QVStWfHo7sL6keKOS7gQeBf7Sdhlx/xvwH4FdJcSayMD1km4rpsEO2gLgp8A3im6ZiyS9soS47ZYDl5cRyPZW4DzgH4CfAE/Zvr6M2E1Q12Q860jaB/gOcIbtp8uIafsF24fRmh20WNJAu2YkfQB41PZtg4zTwZG230FrFa5PFt1SgzQHeAdwoe23Az8HSnn+Aa1VF4HjgG+XFO81tH6DXQC8AXilpH9bRuwmqGsynvZUwmFW9Nl+B/iW7e+WHb/41flGWkv9DdK7geOKvts1wHuLRadKUbTcsP0ocBWt7rBBGgfG237jWEsrOZflGOB224+UFO83gQdt/9T2DlorPP7LkmIPvbom416mITZC8SDtYuA+218pMe7rJO1b7O9F62HpDwcZ0/Yf2p5v+yBaf6c32C6l5STplcUDUoqugvcBAx01Y/sfgYcl/UpxaAkvX35x0E6mpC6Kwj8A75S0d/HvegmtZyDRg1q+dmmqaYiDjivpcuA9wGsljQNfsH3xgMO+G/hd4O6i/xbgP9leN+C4+wOXFk/bR4ArbZc61KxkrweuauUI5gD/y/a1JcT9PeBbRaNiC/DhEmK++APnaOBjZcQDsL1e0lrgdlpTgO+gIVOjy1DLoW0REbNNXbspIiJmlSTjiIgaSDKOiKiBJOOIiBpIMo6IqIEk44iIGkgyjoiogf8PnlWKlCZAmJUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knowledge uncertainty via virtual ensemble:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD8CAYAAABuHP8oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAX7ElEQVR4nO3df5BlZX3n8fenewAdlJEgYWGGhKllkIXNriI1uDHlkh2RIVqOqYIw5AeoaMcV4o+tShY2bihxSYXKrprN+mvCDCABBhxh7UQC/sBaNYnDjIgJMzikM6jTYEABB2H41TOf/eM+g3fa7ntud997u8+9n1fVqT73Oec83+fA1Pc+9znPOUe2iYiI+hma7wZERMTsJIFHRNRUEnhERE0lgUdE1FQSeERETSWBR0TUVBJ4RERNLaraQdKJwBpgaSl6EBi1fV83GxYREa217IFL+q/ARkDAXWURcKOkS7rfvIiImI5a3Ykp6X7gZNvPTyo/GNhme8U0x40AIwCf+vOPvnrk7W/tWIMjoo8tXqK5VvEuHdb27eWf9BNzjjefqoZQ9gHHAN+bVH502TYl2+uAdQDs2Z179SMiuqAqgb8P+LKkfwJ2lbJfAI4HLu5mwyIiZmOQZma0TOC2b5d0ArCSAy9ibrG9t9uNi4iYqUWq9ajIjFTOQrG9D/hGD9oSETFnQ4OTv6sTeEREnWQIJSKipoYyhBIRUU/pgUdE1FTGwCMiamo4QygREfWUIZSIiJrKEEpERE2lBx4Rg63FQ+4WukwjjIioqUWDk7+TwCOiv2QIJSKipoYYnC54EnhE9JXMQomIqKkMoURE1FR64BERNZUXOkRE1FSGUCIiamqQhlAG6csqIgbAEGp7qSJptaQdksYkXTLF9kMk3VS2b5Z0XNO2S0v5DklnNpVvkPSIpHsn1fVzkr4o6Z/K38OrzzUioo8Mqf2lFUnDwMeAs4CTgPMknTRptwuBx20fD3wEuLIcexKwFjgZWA18vNQHcE0pm+wS4Mu2VwBfLp9bn2vVDhERdTKs9pcKK4Ex2zttPwdsBNZM2mcNcG1Z3wSskqRSvtH2s7YfAMZKfdj+KvDYFPGa67oWeEtVA5PAI6KvzGQIRdKIpK1Ny0hTVUuBXU2fx0sZU+1jewLYDRzR5rGTHWX7B2X9X4Cjqs511hcxJb3N9tWzPT4iohtmchHT9jpgXdcaM0u2LanykZBz6YF/cLoNzd9q6zZcM4cQEREzMzSDpcKDwLFNn5eVsin3kbQIWAI82uaxkz0s6ehS19HAI1UNbNkDl/QP022iRff+gG+1Pbvr+2DhiKidDs4i3AKskLScRvJdC/zmpH1GgQuAvwfOBu4svedR4AZJHwaOAVYAd1XE21/Xn5S/n6tqYNUQylHAmcDjk8oF/F1V5RERvdapFzrYnpB0MXAHMAxssL1N0uXAVtujwHrgOkljNC5Mri3HbpN0M7AdmAAusr0XQNKNwOnAyyWNA5fZXk8jcd8s6ULge8BvVLVRbvHmDUnrgattf32KbTfYnvxt9LPSA4+on/l6I8+hL5tz9v3M4Ue13fhzHn+41rf9tOyB276wxbbq5B0R0WO1zsgzlFvpI6KvKA+zioiop8FJ30ngEdFnBunuxCTwiOgrAzSCkgQeEf0lLzWOiKipwUnfSeAR0WcG6YUOSeAR0Vc0QH3wJPBO2vv8/MQdPmh+4g6Sp5+cn7gvOnR+4tb4SmB9Wz5zSeAR0VcyhBIRUVOZhRIRUVODk76TwCOiz9R4+H7GksAjoq8MUP5OAo+I/pJphBERNTU8OPk7CTwi+ssA5e8k8IjoLxlCiYioqcxCiYioqUF6oUPluUo6UdIqSS+ZVL66e82KiJgdzWCpu5YJXNJ7gM8BvwfcK2lN0+Y/7mbDIiJmY0hqe6m7qh74O4FX234LcDrw3yW9t2yb9uwljUjaKmnrug3XdKShERHtGKQeeNUY+JDtJwFsf1fS6cAmSb9Ii/O3vQ5YB8Ce3e5MUyMiqqkPetbtquqBPyzplfs/lGT+JuDlwC91s2EREbMxpPaXuqvqgZ8PTDQX2J4Azpf0qa61KiJiltQPmblNLRO47fEW2/62882JiJiboQGaR5h54BHRVzIGHhFRU1L7S3VdWi1ph6QxSZdMsf0QSTeV7ZslHde07dJSvkPSmVV1lvtt7pZ0j6SvSzq+qn1J4BHRVyS1vVTUMwx8DDgLOAk4T9JJk3a7EHjc9vHAR4Ary7EnAWuBk4HVwMclDVfU+Qngt2y/ErgB+EDVuSaBR0Rf6WAPfCUwZnun7eeAjcCaSfusAa4t65uAVWp8M6wBNtp+1vYDwFipr1WdBg4r60uAh6oamDHwiOgrM7nDUtIIMNJUtK7cxwKwFNjVtG0cOG1SFS/sY3tC0m7giFL+jUnHLi3r09X5DuA2SU8DTwCvqWp/EnhE9JWhGUwjPOCmw/n3fuDXbG+W9PvAh2kk9WklgUdEX1HnBoYfBI5t+ryslE21z7ikRTSGPh6tOPZnyiUdCfx725tL+U3A7VUNzBh4RPSVTl3EBLYAKyQtl3QwjYuSo5P2GQUuKOtnA3fadilfW2apLAdWAHe1qPNxYImkE0pdZwD3VTWwL3vg7zr02OqdYs4++dSu6p26YO+tH+95zOHV5/c8JoB/8qN5iavDjpyXuJ3QqWngZUz7YuAOYBjYYHubpMuBrbZHgfXAdZLGgMdoJGTKfjcD22nczX6R7b2N9v1snaX8ncBnJe2jkdDfXtXGvkzgETG4Onkjj+3bgNsmlf1R0/ozwDnTHHsFcEU7dZbyW4FbZ9K+JPCI6CsDdCNmEnhE9JfhPMwqIqKeBulZKEngEdFXBih/J4FHRH9JAo+IqKm80CEioqZyETMioqYyhBIRUVOZhRIRUVMDlL+rE7iklYBtbylvjlgNfKfcDhoRsaCkB15IuozGq38WSfoijQePfwW4RNKryr3+ERELxgDl78rHyZ4NvBZ4HXAR8BbbHwLOBM6d7iBJI5K2Stq6bsM1nWprRESloWG1vdRd1RDKRHkE4h5J/2z7CQDbT5dHHk7pgLdc7NntTjU2IqJKhlB+6jlJi23vAV69v1DSEmDaBB4RMW8yD/wFr7P9LIDt5oR9ED99C0VExMKRHnjD/uQ9RfmPgPl5VUhERAsZQomIqKvhwXnVbxJ4RPSVPMwqIqKuMoQSEVFP6YFHRNRVeuARETWVHnhERD0ps1AiImoqQyj19smnds1L3L1/97l5iTtf9v71VfMSd/iNb+95zH07v93zmABDJ542L3HrTIPTAe/PBB4RAyw98IiIeso0woiIuhqgHvgAjRZFxCDQ8FDbS2Vd0mpJOySNSbpkiu2HSLqpbN8s6bimbZeW8h2SzqyqUw1XSLpf0n2S3lPVvvTAI6K/dGgIRdIw8DHgDGAc2CJp1Pb2pt0uBB63fbyktcCVwLnl/cFrgZOBY4AvSTqhHDNdnW8FjgVOtL1P0s9XtTE98IjoL1L7S2srgTHbO20/B2wE1kzaZw1wbVnfBKxS43m2a4CNtp+1/QAwVuprVed/Bi7f/+4F249UNTAJPCL6iqS2lwpLgeY5yeOlbMp9bE8Au4EjWhzbqs5/TaP3vlXS30haUdXAJPCI6C9DantpfgF7WUbmseWHAM/YPhX4C2BD1QEZA4+IvjKTW+kPeAH7z3qQxpj0fstK2VT7jEtaBCwBHq04drryceCWsn4rcHVV+9MDj4j+0rkx8C3ACknLJR1M46Lk6KR9Rvnp+4HPBu607VK+tsxSWQ6sAO6qqPP/Ar9a1v8jcH9VA9MDj4i+0qkbeWxPSLoYuAMYBjbY3ibpcmCr7VFgPXCdpDHgMRoJmbLfzcB2YAK4yPZegKnqLCH/BLhe0vuBJ4F3VLVxxglc0qdtnz/T4yIieqKDN/LYvg24bVLZHzWtPwOcM82xVwBXtFNnKf8x8MaZtK9lApc0+eeCgF+V9LIS8M0zCRYR0XW5lf4Fy2j8BLgKMI0Efirwv1odVK7kjgB86s8/ysjb3zrnhkZEtKON6YF9oyqBnwq8F/hD4Pdt3yPpadv/r9VBB1zZ3bPbnWhoRERb8kKHhnJH0Eckfab8fbjqmIiIeZUe+IFsjwPnSHoj8ER3mxQRMQdJ4FOz/Xng811qS0TE3A1lCCUiop7SA4+IqKkk8IiImhoenu8W9EwSeET0l/TAIyJqKgk8IqKmksAjImoq0wgjImoqCTxmY/iXJ7/vtEeeeWpewvqZn8xLXA46uOchh048recxY5YyhBIRUU9KDzwioqbSA4+IqKkk8IiImkoCj4ioqdxKHxFRU+mBR0TUVBJ4RERNZRphRERNpQceEVFTSeARETWVWShTk/QrwErgXttf6E6TIiLmYIB64C1H+yXd1bT+TuD/AC8FLpN0SZfbFhExc1L7S81VXa49qGl9BDjD9geBNwC/Nd1BkkYkbZW0dd2Ga+beyoiIdg0Ntb/UXNUQypCkw2kketn+IYDtpyRNTHeQ7XXAOgD27HaH2hoRUa0PetbtqvoKWgJ8E9gK/JykowEkvQQYnP9KEVEfQ8PtLxUkrZa0Q9LYVMPGkg6RdFPZvlnScU3bLi3lOySdOYM6/7ekJ9s51ZY9cNvHTbNpH/Dr7QSIiOipoc70LSUNAx8DzgDGgS2SRm1vb9rtQuBx28dLWgtcCZwr6SRgLXAycAzwJUknlGOmrVPSqcDh7bZxVoNAtvfYfmA2x0ZEdJWG2l9aWwmM2d5p+zlgIzD5tVtrgGvL+iZglSSV8o22ny25cqzUN22d5QvjT4E/aPdU6z+KHxHRbAazUJonXJRlpKmmpcCups/jpYyp9rE9AewGjmhxbKs6LwZGbf+g3VPNjTwR0V9mMLvkgAkX80jSMcA5wOkzOS4JPCL6S+dmoTwIHNv0eVkpm2qfcUmLaEz8eLTi2KnKXwUcD4w1RmBYLGnM9vGtGpghlIjoL52bhbIFWCFpuaSDaVyUHJ20zyhwQVk/G7jTtkv52jJLZTmwArhrujptf972v7J9XJk8sqcqeUN64BHRbzp0g47tCUkXA3cAw8AG29skXQ5stT0KrAeukzQGPEYjIVP2uxnYDkwAF9neCzBVnbNtoxpfFl2UG3m675mn5iWsn/nJvMTVkp+fh6D5sdoTi5fMefxj76f/R9s5Z/j8D9T6fpb0wCOivwzQl20SeET0lw7dyFMHSeD94EWHzkvYfRv+eF7iDr/7inmJGzXRxi3y/SIJPCL6S4ZQIiJqKkMoERE1NUCPk00Cj4j+kiGUiIiayhBKRERNZRZKRERNZQglIqKmMoQSEVFT6YFHRNRUphFGRNRUhx4nWwdJ4BHRXwZoFkrLrypJp0k6rKy/WNIHJf2VpCslLelNEyMiZmAGLzWuu6rfGhuAPWX9z2i87+3KUnb1dAc1v+l53YZrOtHOiIj2DA21v9Rc1RDKkO2Jsn6q7VPK+tcl3TPdQQe86Tlv5ImIXuqDnnW7qr6C7pX0trL+bUmnAkg6AXi+qy2LiJgNDbW/1FxVD/wdwJ9J+gDwI+DvJe0CdpVtERELywBdxGyZwG3vBt5aLmQuL/uP2364F42LiJix3Il5INtPAN/uclsiIuauD4ZG2pV54BHRXwboImYSeET0l/TAIyLqSemBR0TU1NDgpLXBOdOIGAyZhRIRUVMZA4+IqKkBGgMfnK+qiBgMHbyVXtJqSTskjUm6ZIrth0i6qWzfLOm4pm2XlvIdks6sqlPS9aX8XkkbJB1U2T67y8+aysOsIqJdi5fMufu87+4vtJ1zhk55w7TxJA0D9wNnAOPAFuA829ub9nk38O9sv0vSWuDXbZ8r6STgRmAlcAzwJeCEctiUdUr6NeBvyj43AF+1/YmW7W/3RCMiamF4uP2ltZXAmO2dtp8DNgJrJu2zBri2rG8CVqkxj3ENsNH2s7YfAMZKfdPWafs2F8BdwLKqBiaBR0R/mcEQSvO7C8oy0lTTUhoP7ttvvJQx1T7l0du7gSNaHFtZZxk6+R3g9qpTzUXMiOgvM7iIecC7CxaOj9MYPvla1Y5J4BHRXzo3jfBB4Nimz8tK2VT7jEtaROOtZY9WHDttnZIuA44EfredBmYIJSL6S+feibkFWCFpuaSDgbXA6KR9RoELyvrZwJ1lDHsUWFtmqSwHVtAY1562TknvAM6kcVFzXzunmh54RPSX4c6kNdsTki4G7gCGgQ22t0m6HNhqexRYD1wnaQx4jEZCpux3M7AdmAAusr0XYKo6S8hPAt+j8eIcgFtsX96qjZlGGBELRwemEXrHN9rOOXrFa2p910964BHRX3IrfURETQ3QrfRJ4BHRX9IDj4ioqQHqgbf8qpL0HknHttonImJB6dyt9Ate1W+NDwGbJX1N0rslHdmLRkVEzFoHn0a40FWdwU4adwp9CHg1sF3S7ZIukPTS6Q5qfr7Aug3XdK61ERFVOncjz4LXch64pLttn9L0+SDgLOA84PW2q3vkmQceEe3qxDzw729rfx74L5xc6yxedRHzgJOz/TyN2z5HJS3uWqsiImarD3rW7apK4OdOt8H2ng63JSJi7pLAG2zf36uGRER0RB9cnGxX5oFHRH8ZnA54EnhE9JvByeBJ4BHRXzIGHhFRU0ngERE1lYuYERF1lR54REQ9ZQglIqKmksAj2tDei7M7b4DGOGM2ksAjImpJ6YFHRNTUAP1CSwKPiP6SHnhERE0lgUdE1FUSeEREPaUHHhFRU4OTv5PAI6LPZBZKRERNZQglIqKuksAjIuopPfAGSQcDa4GHbH9J0m8CvwzcB6yz/XwP2hgR0b4BSuCyPf1G6XoaSX4x8GPgJcAtwKpy7AWVEfbsnj5A1FseZhWdtnjJ3LPvTHJOJ+LNJ9vTLsA/lL+LgIeB4fJZ+7dNc9wIsLUsI61iVMSf9bFzWQYp7iCd66DFHaRzHdSlqgd+L3AKcCjwfeAXbT8m6UXAt2z/m858jUwbf6vtU7sZY9DjDtK5DlrcQTrXQVV1EXM98B1gGPhD4DOSdgKvATZ2uW0REdFCywRu+yOSbirrD0n6NPB64C9s39WLBkZExNQqpxHafqhp/cfApq626EDrehhrUOMO0rkOWtxBOteB1HIMPCIiFq7Mx4qIqKkk8IiImlqwCVzSakk7JI1JuqRHMTdIeqRMn+wJScdK+oqk7ZK2SXpvj+K+SNJdkr5d4n6wF3FL7GFJ35L0172KWeJ+V9I/SrpH0tYexXyZpE2SviPpPkn/oQcxX1HOcf/yhKT3dTtuif3+8u/pXkk3linH0SULcgxc0jBwP3AGMA5sAc6zvb3LcV8HPAl82va/7WaspphHA0fbvlvSS4FvAm/pwbkKONT2k5IOAr4OvNf2N7oZt8T+L8CpwGG239TteE1xvwucavtHPYx5LfA121eVR1MsLpMBehV/GHgQOM3297ocaymNf0cn2X5a0s3Abbav6WbcQbZQe+ArgTHbO20/R2PO+ZpuB7X9VeCxbseZFPMHtu8u6z+h8ZyZpT2Ia9tPlo8HlaXr3+aSlgFvBK7qdqz5JmkJ8Doa91Ng+7leJu9iFfDP3U7eTRYBL5a0/xEcD1XsH3OwUBP4UmBX0+dxepDU5puk44BXAZt7FG9Y0j3AI8AXbfci7keBPwDm40EqBr4g6ZuSRnoQbznwQ+DqMmR0laRDexC32Vrgxl4Esv0g8D9p3LX9A2C37S/0IvagWqgJfOBIegnwWeB9tp/oRUzbe22/ElgGrJTU1WEjSW8CHrH9zW7GaeFXbJ8CnAVcVIbMumkRjUdRfML2q4CngJ5cz4EXnib6ZuAzPYp3OI1fysuBY4BDJf12L2IPqoWawB8Ejm36vKyU9aUyBv1Z4Hrbt/Q6fvlZ/xVgdZdDvRZ4cxmL3gj8J0l/2eWYLyg9RGw/AtxKY6ium8aB8aZfNptoJPReOQu42/bDPYr3euAB2z9041HTt9B4/HR0yUJN4FuAFZKWNz2TfHSe29QV5WLieuA+2x/uYdwjJb2srL+YxgXj73Qzpu1LbS+zfRyN/6d32u5JD03SoeUiMWUY4w1AV2cb2f4XYJekV5SiVUBXL05Pch49Gj4pvg+8RtLi8u96FY1rOtElC/KNPLYnJF0M3EHjQVobbG/rdlxJNwKnAy+XNA5cZnt9l8O+Fvgd4B/LeDTAf7N9W5fjHg1cW2YpDAE32+7ptL4eOwq4tZFXWATcYPv2HsT9PeD60hHZCbytBzH3f0mdAfxuL+IB2N4saRNwNzABfIvcVt9VC3IaYUREVFuoQygREVEhCTwioqaSwCMiaioJPCKippLAIyJqKgk8IqKmksAjImrq/wOHTyi72pxmggAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mean_preds = preds[:,0] # mean values predicted by a virtual ensemble\n",
    "\n",
    "data = preds[:,2] # estimated data uncertainty\n",
    "print(\"Average predicted data uncertainty:\")\n",
    "sns.heatmap(data.reshape([num_cat,num_cat]), cmap=\"Reds\", vmin = 0, vmax = 0.05)\n",
    "# plt.savefig(\"heart_data_vsgb.pdf\")\n",
    "plt.show()\n",
    "\n",
    "knowledge = preds[:,1] # knowledge uncertainty\n",
    "print(\"Knowledge uncertainty via virtual ensemble:\")\n",
    "sns.heatmap(knowledge.reshape([num_cat,num_cat]), cmap=\"Reds\")\n",
    "# plt.savefig(\"heart_knowledge_vsgb.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}