MachineLearningMetrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zvyedSW8W1qx"
   },
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "from sklearn.metrics import accuracy_score, roc_curve, roc_auc_score, f1_score, precision_score, confusion_matrix, cohen_kappa_score\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import Button, Layout, GridspecLayout, AppLayout\n",
    "\n",
    "plt.ioff()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mAqOhliYieK1"
   },
   "outputs": [],
   "source": [
    "def generate_data(difference, variance, prior, samples):\n",
    "  np.random.seed(42)\n",
    "  mu2 = 0.5-difference/2\n",
    "  mu1 = 0.5+difference/2\n",
    "  N1 = int(samples*prior)\n",
    "  N2 = samples - N1\n",
    "  d1 = np.clip(np.random.normal(loc=mu1, scale=variance, size=N1), 0, 1)\n",
    "  d2 = np.clip(np.random.normal(loc=mu2, scale=variance, size=N2), 0, 1)\n",
    "  jitter = np.random.uniform(low=-0.25, high=0.25, size=samples)\n",
    "  y_true = np.concatenate((np.zeros(N2), np.ones(N1)))\n",
    "  y_pred = np.concatenate((d2, d1))\n",
    "  y_true_jitter = y_true + jitter\n",
    "  return y_true, y_pred, y_true_jitter  \n",
    "\n",
    "def roc_and_threshold(y_true, y_pred, threshold):\n",
    "  fpr, tpr, thresholds = roc_curve(y_true, y_pred)\n",
    "  index = np.nonzero(thresholds < threshold)[0][0]\n",
    "  if not (np.abs(threshold - thresholds[index]) < 0.01):\n",
    "    op = {'x':-1, 'y':-1, 't':threshold}\n",
    "  else:  \n",
    "    op = {'x':fpr[index], 'y':tpr[index], 't':thresholds[index]}\n",
    "  roc = {'x':fpr, 'y':tpr}\n",
    "  return roc, op\n",
    "\n",
    "def calculate_metrics(y_true, y_pred, threshold):\n",
    "  auc = roc_auc_score(y_true, y_pred)\n",
    "  y_p_bin = y_pred > threshold\n",
    "  conf_mat = confusion_matrix(y_true, y_p_bin)\n",
    "  TN, FP, FN, TP = np.ravel(conf_mat)\n",
    "  sensitivity = TP/(TP+FN)\n",
    "  specificity = TN/(TN+FP)\n",
    "  kappa_score = cohen_kappa_score(y_true, y_p_bin)\n",
    "  ppv = precision_score(y_true, y_p_bin)\n",
    "  npv = TN/(TN+FN)\n",
    "  metrics = {'Accuracy':accuracy_score(y_true, y_p_bin), 'F1-score':f1_score(y_true, y_p_bin),\n",
    "             'Sensitivity':sensitivity, 'Specificity':specificity, 'PPV':ppv, 'NPV':npv, 'AUC':auc, 'Kappa':kappa_score, 'conf_mat':conf_mat}\n",
    "  roc, operating_point = roc_and_threshold(y_true, y_pred, threshold)\n",
    "  if operating_point['x'] == -1:\n",
    "    operating_point['x'] = 1-metrics['Specificity']\n",
    "    operating_point['y'] = metrics['Sensitivity']\n",
    "  return metrics, roc, operating_point\n",
    "\n",
    "def update_data_and_metrics(difference, variance, prior, samples, threshold):\n",
    "  y_true, y_pred, y_true_jitter = generate_data(difference, variance, prior, samples)\n",
    "  metrics, roc, operating_point = calculate_metrics(y_true, y_pred, threshold)\n",
    "  data = {'y_true':y_true, 'y_pred':y_pred, 'y_true_jitter':y_true_jitter}\n",
    "  return data, metrics, roc, operating_point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LU3-TWDprJ2t"
   },
   "outputs": [],
   "source": [
    "style = {'description_width': 'initial'}\n",
    "difference = widgets.FloatSlider(value=0.5,min=0.0,max=0.96,step=0.05,description='Class separation',\n",
    "    continuous_update=False, style=style\n",
    ")\n",
    "variance = widgets.FloatSlider(value=0.25, min=0.05, max=0.51, step=0.05, description='Within class spread',\n",
    "    continuous_update=False, style=style\n",
    ")\n",
    "threshold = widgets.FloatSlider(value=0.5, min=0.05, max=0.96, step=0.05, description='Classification threshold',\n",
    "    continuous_update=False, style=style\n",
    ")\n",
    "prior_prob = widgets.FloatSlider(value=0.5, min=0.05, max=0.96, step=0.05, description='Prior probability',\n",
    "    continuous_update=False, style=style\n",
    ")\n",
    "num_samples = widgets.IntSlider(value=1000, min=50, max=10000, step=50, description='Number of samples',\n",
    "    continuous_update=False, style=style\n",
    ")\n",
    "\n",
    "model_quality = widgets.Label(value='Estimated model quality: Good')\n",
    "\n",
    "all_widgets = [difference, variance, num_samples, prior_prob, threshold]\n",
    "\n",
    "def update(change):\n",
    "  data, metrics, roc, operating_point = update_data_and_metrics(difference.value, variance.value, prior_prob.value, num_samples.value, threshold.value)\n",
    "  #create_plot_objects(data, metrics, roc, operating_point)\n",
    "  update_plot_objects(data, metrics, roc, operating_point, axlist)\n",
    "\n",
    "def create_plot_objects_mpl(data, metrics, roc, operating_point):\n",
    "  plt.close('all')\n",
    "  #fig = plt.figure(figsize=(20,8)) \n",
    "  fig = plt.figure(figsize=(10,4))\n",
    "  ax = fig.add_subplot(131)\n",
    "\n",
    "  threshold = operating_point['t']   \n",
    "    \n",
    "  Nneg = np.sum(data['y_true']==0)\n",
    "  nx = data['y_true_jitter'][0:Nneg]\n",
    "  ny = data['y_pred'][0:Nneg]\n",
    "\n",
    "  nind_true = np.argwhere(ny<threshold)\n",
    "  nind_false = np.argwhere(ny>=threshold)\n",
    "  nx_true = nx[nind_true]\n",
    "  nx_false = nx[nind_false]\n",
    "  ny_true = ny[nind_true]\n",
    "  ny_false = ny[nind_false]\n",
    "\n",
    "  px = data['y_true_jitter'][Nneg:]\n",
    "  py = data['y_pred'][Nneg:]\n",
    "  pind_true = np.argwhere(py>=threshold)\n",
    "  pind_false = np.argwhere(py<threshold)  \n",
    "  px_true = px[pind_true]\n",
    "  px_false = px[pind_false]\n",
    "  py_true = py[pind_true]\n",
    "  py_false = py[pind_false]  \n",
    "\n",
    "  #plot1 = ax.plot(nx, ny, 'C0', marker='o', ls=\"\", markersize=1)\n",
    "  #plot11 = ax.plot(px,py, 'C1', marker='o', ls=\"\", markersize=1)\n",
    "\n",
    "  plot10 = ax.plot(nx_true, ny_true, '#1f78b4', marker='o', ls=\"\", markersize=1)\n",
    "  plot11 = ax.plot(nx_false, ny_false, '#a6cee3', marker='o', ls=\"\", markersize=1) \n",
    "  plot12 = ax.plot(px_true, py_true, '#ff7f00', marker='o', ls=\"\", markersize=1)\n",
    "  plot13 = ax.plot(px_false, py_false, '#fdbf6f', marker='o', ls=\"\", markersize=1)   \n",
    "\n",
    "  ax.set_xticks([0, 1])\n",
    "  ax.set_xlabel('True class label')\n",
    "  ax.set_ylabel('Model output probability')\n",
    "  plot2 = ax.plot([-0.25,1.25], [threshold, threshold], color='red')\n",
    "  ax.set_title('Model predictions')\n",
    "  plottxt = ax.text(x=0.5, y=operating_point['t']+0.02, s='Threshold', horizontalalignment='center', color='red')\n",
    "\n",
    "  #plist1 = {'swarm':[plot1[0],plot11[0]],'line':plot2[0], 'text':plottxt}\n",
    "  plist1 = {'swarm':[plot10[0],plot11[0],plot12[0],plot13[0]],'line':plot2[0], 'text':plottxt}\n",
    "\n",
    "  ax2 = fig.add_subplot(132)\n",
    "  plist2 = {}\n",
    "  for ii, metric in enumerate(metrics.keys()):\n",
    "    plist2['plot{}'.format(ii)] = {}\n",
    "    col = 'black'\n",
    "    fs = 10\n",
    "    if metric != 'conf_mat':\n",
    "      s = '{:.2f}'.format(metrics[metric])\n",
    "      sname = metric\n",
    "      xval = 0.1+0.4  \n",
    "      if metrics[metric] >= 0.85:\n",
    "        col = 'green'\n",
    "      elif ((metrics[metric] >= 0.65) and (metrics[metric] < 0.85)):\n",
    "        col = 'orange'\n",
    "      else:\n",
    "        col = 'red'   \n",
    "    else:\n",
    "      TN, FP, FN, TP = np.ravel(metrics[metric])\n",
    "      s = 'TN: {:6s}  FP: {:6s}\\nFN: {:6s}  TP: {:6s}'.format(str(TN), str(FP), str(FN), str(TP))\n",
    "      sname = ' '\n",
    "      xval = 0.1\n",
    "      fs = 12\n",
    "    plist2['plot{}'.format(ii)]['name'] = ax2.text(x=0.1, y=0.1+0.1*ii, s=sname, fontsize=fs)\n",
    "    plist2['plot{}'.format(ii)]['value'] = ax2.text(x=xval, y=0.1+0.1*ii, s=s, color=col, fontsize=fs)\n",
    "    \n",
    "  plt.axis('off')\n",
    "\n",
    "  ax3 = fig.add_subplot(133)\n",
    "  plot3 = ax3.plot(roc['x'], roc['y'])\n",
    "  plot4 = ax3.plot([0,1],[0,1],color='orange')\n",
    "  plot5 = ax3.plot([operating_point['x']], [operating_point['y']], markersize=8, color='red', marker=\"x\", ls='')\n",
    "  ax3.set_xlabel('1-Specificity')\n",
    "  ax3.set_ylabel('Sensitivity')\n",
    "  ax3.set_title('ROC curve. AUC = {:.3f}'.format(metrics['AUC']))\n",
    "  plist3 = {'roc':plot3[0], 'base':plot4[0], 'op':plot5[0]}\n",
    "  axlist = [ax, ax2, ax3]\n",
    "  plist  = [plist1, plist2, plist3]\n",
    "  return fig, axlist, plist\n",
    "\n",
    "def update_plot_objects_mpl(data, metrics, roc, operating_point, axlist, plist):\n",
    "  Nneg = np.sum(data['y_true']==0)\n",
    "  nx = data['y_true_jitter'][0:Nneg]\n",
    "  ny = data['y_pred'][0:Nneg]\n",
    "  px = data['y_true_jitter'][Nneg:]\n",
    "  py = data['y_pred'][Nneg:]  \n",
    "\n",
    "  threshold = operating_point['t']\n",
    "\n",
    "  nind_true = np.argwhere(ny<threshold)\n",
    "  nind_false = np.argwhere(ny>=threshold)\n",
    "  pind_true = np.argwhere(py>=threshold)\n",
    "  pind_false = np.argwhere(py<threshold) \n",
    "\n",
    "  nx_true = nx[nind_true]\n",
    "  nx_false = nx[nind_false]\n",
    "  ny_true = ny[nind_true]\n",
    "  ny_false = ny[nind_false]\n",
    "  px_true = px[pind_true]\n",
    "  px_false = px[pind_false]\n",
    "  py_true = py[pind_true]\n",
    "  py_false = py[pind_false] \n",
    "\n",
    "  nplot_true = plist[0]['swarm'][0]\n",
    "  nplot_true.set_data([nx_true, ny_true])\n",
    "  pplot_true = plist[0]['swarm'][2]\n",
    "  pplot_true.set_data([px_true,py_true])\n",
    "\n",
    "  nplot_false = plist[0]['swarm'][1]\n",
    "  nplot_false.set_data([nx_false, ny_false])\n",
    "  pplot_false = plist[0]['swarm'][3]\n",
    "  pplot_false.set_data([px_false,py_false])\n",
    "\n",
    "  plot2 = plist[0]['line']\n",
    "  plot2.set_data([[-0.25,1.25], [operating_point['t'], operating_point['t']]])\n",
    "  plottxt = plist[0]['text']\n",
    "  plottxt.set_position([0.5, operating_point['t']+0.02])\n",
    "  \n",
    "  for ii, metric in enumerate(metrics.keys()):\n",
    "    curr_plot = plist[1]['plot{}'.format(ii)]\n",
    "    col = 'black'\n",
    "    fs = 10\n",
    "    if metric != 'conf_mat':\n",
    "      s = '{:.2f}'.format(metrics[metric])\n",
    "      if metrics[metric] >= 0.85:\n",
    "        col = 'green'\n",
    "      elif ((metrics[metric] > 0.65) and (metrics[metric] < 0.85)):\n",
    "        col = 'orange'\n",
    "      else:\n",
    "        col = 'red'    \n",
    "    else:\n",
    "      TN, FP, FN, TP = np.ravel(metrics[metric])\n",
    "      s = 'TN: {:6s}  FP: {:6s}\\nFN: {:6s}  TP: {:6s}'.format(str(TN), str(FP), str(FN), str(TP))\n",
    "      fs = 12\n",
    "    curr_plot['value'].set_text(s)\n",
    "    curr_plot['value'].set_color(col)\n",
    "    curr_plot['value'].set_fontsize(fs)\n",
    "\n",
    "  plot3 = plist[2]['roc']\n",
    "  plot3.set_data([roc['x'], roc['y']])\n",
    "\n",
    "  plot5 = plist[2]['op']\n",
    "  plot5.set_data([operating_point['x'], operating_point['y']])\n",
    "  axlist[2].set_title('ROC curve. AUC = {:.3f}'.format(metrics['AUC']))\n",
    "  return axlist, plist\n",
    "\n",
    "\n",
    "def update_quality_label(labelwidget, auc):\n",
    "    if auc > 0.95:\n",
    "        qual = 'Very good'\n",
    "    elif (auc <= 0.95) and (auc > 0.9):\n",
    "        qual = 'Good'\n",
    "    elif (auc <= 0.9) and (auc > 0.75):\n",
    "        qual = 'Moderate'\n",
    "    elif (auc <= 0.75) and (auc > 0.65):\n",
    "        qual = 'Poor'\n",
    "    elif (auc <= 0.65):\n",
    "        qual = 'Very poor'\n",
    "    labelwidget.value = 'Estimated model quality: ' + qual\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 577,
     "referenced_widgets": [
      "0a03ecb6254f4e1e8543236f9b47e304",
      "96628d03bc8b46129e1dbf0c158ddd00",
      "2ea531573f354820ac62d93623ab02bb",
      "7f50e3db1a4a4d92bbdc94e239ae8ba8",
      "5c8ea29653f4498d9329aaa5e6b7945d",
      "43105045bdf84a81a2a06da326a07e0b",
      "1a3a5a6192c64431a4d958831a1574b6",
      "5642593bf6a24b01b01f2043ac14ace1",
      "8bb964b0ed0a48179f61cf96c250dbdd",
      "033bf2a594c344a78796eedb383d20a3",
      "8174fb92453142b882a9d26529526037",
      "b0545a86cf4f4a7fbfa2efcfd374a882",
      "a77aa2c3a3b347749ff808d4d91aa8af",
      "a31f131147674ed9910d69da46562efd",
      "20198a25377c4fcbbe7a83f14b7f6700",
      "8af8ddde3d0d4772b4710c8611d7c6d8",
      "c001ac5f6a32440a907c692dd37bb3e5"
     ]
    },
    "colab_type": "code",
    "id": "Ya3YfkQZtPq_",
    "outputId": "1ac5c128-15e1-43af-e1a4-aba18ce01707"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8489f1989884f4b909f2a17458e29bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "AppLayout(children=(GridspecLayout(children=(Label(value='$\\\\mathbf{Population \\\\; properties}$', layout=Layou…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "grid = GridspecLayout(4, 3)\n",
    "\n",
    "grid[0,0] = widgets.Label(value='$\\mathbf{Population \\; properties}$')\n",
    "grid[1,0] = num_samples\n",
    "grid[2,0] = prior_prob\n",
    "grid[0,1] = widgets.Label(value='$\\mathbf{Model \\; strength}$')\n",
    "grid[1,1] = difference\n",
    "grid[2,1] = variance\n",
    "grid[0,2] = widgets.Label(value='$\\mathbf{Post-processing}$')\n",
    "grid[1,2] = threshold\n",
    "grid[3,1] = model_quality\n",
    "\n",
    "data, metrics, roc, operating_point = update_data_and_metrics(difference.value, variance.value, prior_prob.value, num_samples.value, threshold.value)\n",
    "update_quality_label(model_quality, metrics['AUC'])\n",
    "fig, axlist, plist = create_plot_objects_mpl(data, metrics, roc, operating_point)\n",
    "\n",
    "def update_plots(figure, axes_list, plot_list):\n",
    "  def update(change):\n",
    "    data, metrics, roc, operating_point = update_data_and_metrics(difference.value, variance.value, prior_prob.value, num_samples.value, threshold.value)\n",
    "    #create_plot_objects(data, metrics, roc, operating_point)\n",
    "    update_plot_objects_mpl(data, metrics, roc, operating_point, axes_list, plot_list)\n",
    "    figure.canvas.draw_idle()\n",
    "    figure.canvas.flush_events()\n",
    "    update_quality_label(model_quality, metrics['AUC'])\n",
    "  return update  \n",
    "\n",
    "\n",
    "\n",
    "for widget in all_widgets:\n",
    "  widget.observe(update_plots(figure=fig, axes_list=axlist, plot_list=plist))  \n",
    "  \n",
    "#plt.show()\n",
    "AppLayout(header=grid, center=fig.canvas, pane_heights=[1,6,0])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QuZnENKDmsU5"
   },
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "MachineLearningMetrics.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "033bf2a594c344a78796eedb383d20a3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "SliderStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "SliderStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": "initial",
      "handle_color": null
     }
    },
    "0a03ecb6254f4e1e8543236f9b47e304": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "GridBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "GridBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "GridBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2ea531573f354820ac62d93623ab02bb",
       "IPY_MODEL_7f50e3db1a4a4d92bbdc94e239ae8ba8",
       "IPY_MODEL_5c8ea29653f4498d9329aaa5e6b7945d",
       "IPY_MODEL_43105045bdf84a81a2a06da326a07e0b",
       "IPY_MODEL_1a3a5a6192c64431a4d958831a1574b6"
      ],
      "layout": "IPY_MODEL_96628d03bc8b46129e1dbf0c158ddd00"
     }
    },
    "1a3a5a6192c64431a4d958831a1574b6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatSliderModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatSliderModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "FloatSliderView",
      "continuous_update": false,
      "description": "Classification threshold",
      "description_tooltip": null,
      "disabled": false,
      "layout": "IPY_MODEL_c001ac5f6a32440a907c692dd37bb3e5",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "readout": true,
      "readout_format": ".2f",
      "step": 0.1,
      "style": "IPY_MODEL_8af8ddde3d0d4772b4710c8611d7c6d8",
      "value": 0.9
     }
    },
    "20198a25377c4fcbbe7a83f14b7f6700": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": "widget004",
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2ea531573f354820ac62d93623ab02bb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "IntSliderModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "IntSliderModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "IntSliderView",
      "continuous_update": false,
      "description": "Number of samples",
      "description_tooltip": null,
      "disabled": false,
      "layout": "IPY_MODEL_8bb964b0ed0a48179f61cf96c250dbdd",
      "max": 10000,
      "min": 50,
      "orientation": "horizontal",
      "readout": true,
      "readout_format": "d",
      "step": 50,
      "style": "IPY_MODEL_5642593bf6a24b01b01f2043ac14ace1",
      "value": 1000
     }
    },
    "43105045bdf84a81a2a06da326a07e0b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatSliderModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatSliderModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "FloatSliderView",
      "continuous_update": false,
      "description": "Within class spread",
      "description_tooltip": null,
      "disabled": false,
      "layout": "IPY_MODEL_20198a25377c4fcbbe7a83f14b7f6700",
      "max": 0.95,
      "min": 0.05,
      "orientation": "horizontal",
      "readout": true,
      "readout_format": ".2f",
      "step": 0.05,
      "style": "IPY_MODEL_a31f131147674ed9910d69da46562efd",
      "value": 0.25
     }
    },
    "5642593bf6a24b01b01f2043ac14ace1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "SliderStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "SliderStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": "initial",
      "handle_color": null
     }
    },
    "5c8ea29653f4498d9329aaa5e6b7945d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatSliderModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatSliderModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "FloatSliderView",
      "continuous_update": false,
      "description": "Prior probability",
      "description_tooltip": null,
      "disabled": false,
      "layout": "IPY_MODEL_a77aa2c3a3b347749ff808d4d91aa8af",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "readout": true,
      "readout_format": ".2f",
      "step": 0.05,
      "style": "IPY_MODEL_b0545a86cf4f4a7fbfa2efcfd374a882",
      "value": 0.5
     }
    },
    "7f50e3db1a4a4d92bbdc94e239ae8ba8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatSliderModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatSliderModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "FloatSliderView",
      "continuous_update": false,
      "description": "Class separation",
      "description_tooltip": null,
      "disabled": false,
      "layout": "IPY_MODEL_8174fb92453142b882a9d26529526037",
      "max": 0.95,
      "min": 0,
      "orientation": "horizontal",
      "readout": true,
      "readout_format": ".2f",
      "step": 0.05,
      "style": "IPY_MODEL_033bf2a594c344a78796eedb383d20a3",
      "value": 0.5
     }
    },
    "8174fb92453142b882a9d26529526037": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": "widget002",
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8af8ddde3d0d4772b4710c8611d7c6d8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "SliderStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "SliderStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": "initial",
      "handle_color": null
     }
    },
    "8bb964b0ed0a48179f61cf96c250dbdd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": "widget001",
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "96628d03bc8b46129e1dbf0c158ddd00": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": "\"widget001 widget002 widget005\"\n\"widget003 widget004 .\"",
      "grid_template_columns": "repeat(3, 1fr)",
      "grid_template_rows": "repeat(2, 1fr)",
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a31f131147674ed9910d69da46562efd": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "SliderStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "SliderStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": "initial",
      "handle_color": null
     }
    },
    "a77aa2c3a3b347749ff808d4d91aa8af": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": "widget003",
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b0545a86cf4f4a7fbfa2efcfd374a882": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "SliderStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "SliderStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": "initial",
      "handle_color": null
     }
    },
    "c001ac5f6a32440a907c692dd37bb3e5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": "widget005",
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}