5-1-CTMC-Numerical.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };var element = document.getElementById(\"1001\");\n",
       "  if (element == null) {\n",
       "    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.3.4.min.js\"];\n",
       "  var css_urls = [];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {} // ensure no trailing comma for IE\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };var element = document.getElementById(\"1001\");\n  if (element == null) {\n    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n    return false;\n  }\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.3.4.min.js\"];\n  var css_urls = [];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import display, Math, clear_output\n",
    "\n",
    "import dtmc\n",
    "import ctmc\n",
    "import mcviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1002\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1002\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };var element = document.getElementById(\"1002\");\n",
       "  if (element == null) {\n",
       "    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1002' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.3.4.min.js\"];\n",
       "  var css_urls = [];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {} // ensure no trailing comma for IE\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1002\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1002\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };var element = document.getElementById(\"1002\");\n  if (element == null) {\n    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1002' but no matching script tag was found. \")\n    return false;\n  }\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.3.4.min.js\"];\n  var css_urls = [];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1002\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import bokeh.plotting as bplt\n",
    "from bokeh.models import Range1d, LabelSet, ColumnDataSource\n",
    "from bokeh.models import Arrow, TeeHead, VeeHead, NormalHead, OpenHead\n",
    "from bokeh.models import Whisker, Span\n",
    "from bokeh.models.glyphs import Step\n",
    "from bokeh.models.markers import Circle\n",
    "from bokeh.io import output_notebook, show, push_notebook\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Continuous-Time Markov Chains: Section 1 – Numerical Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Consider the rat-in-a-maze example from discrete-time Markov chains.  Previously we only counted “visits” to each room and ignored the amount of time spent in the room (this was the essence of why our model had discrete time).  We now want to incorporate the time spent in a room.\n",
    "\n",
    "We will again start with a numerical example and specify the dynamics.  We will use the familiar model of the rat in a maze, but add to the model the length of time spent on each visit to the room.  In particular, we will assume that the visits last an exponentially distributed amount of time, where the rate of the exponential distribution may depend on the room visited.  That is, any visit to room $i$ lasts for a time that follows an $\\text{Exponential}(v_i)$ distribution.\n",
    "\n",
    "**Remark:** Why an exponential distribution that depends on the state?  The key idea is to incorporate the time spent in a room in a way that preserves the Markov property in continuous time.  We want the stochastic process to be memoryless."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\displaystyle \\mathcal{S} = \\{1,2,3,4\\}$"
      ],
      "text/plain": [
       "<IPython.core.display.Math object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/latex": [
       "$\\displaystyle \\mathbf{P} = \\pmatrix{0& 1/2& 0& 1/2\\\\ 1/2& 0& 1/2& 0\\\\ 0& 1/2& 0& 1/2\\\\ 1/2& 0& 1/2& 0}\\qquad\\begin{matrix}v_1 = 1\\\\ v_2 = 2\\\\ v_3 = 2\\\\ v_4 = 1\\end{matrix}$"
      ],
      "text/plain": [
       "<IPython.core.display.Math object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: Markov chain Pages: 1 -->\n",
       "<svg width=\"182pt\" height=\"182pt\"\n",
       " viewBox=\"0.00 0.00 181.51 181.51\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 177.5099)\">\n",
       "<title>Markov chain</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-177.5099 177.5099,-177.5099 177.5099,4 -4,4\"/>\n",
       "<!-- 1 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>1</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"86.7549\" cy=\"-18\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"86.7549\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"18\" cy=\"-86.7549\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"18\" y=\"-83.0549\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M66.6772,-38.0778C57.6491,-47.1058 47.0199,-57.735 38.001,-66.754\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"69.2264,-40.4783 73.8227,-30.9323 64.2767,-35.5285 69.2264,-40.4783\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"35.4705,-64.3347 30.8743,-73.8806 40.4203,-69.2844 35.4705,-64.3347\"/>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"155.5099\" cy=\"-86.7549\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"155.5099\" y=\"-83.0549\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;4 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M106.8327,-38.0778C115.8608,-47.1058 126.4899,-57.735 135.5089,-66.754\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"109.2332,-35.5285 99.6872,-30.9323 104.2834,-40.4783 109.2332,-35.5285\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"133.0896,-69.2844 142.6356,-73.8806 138.0394,-64.3347 133.0896,-69.2844\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>3</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"86.7549\" cy=\"-155.5099\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"86.7549\" y=\"-151.8099\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3</text>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M38.0778,-106.8327C47.1058,-115.8608 57.735,-126.4899 66.754,-135.5089\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"40.4783,-104.2834 30.9323,-99.6872 35.5285,-109.2332 40.4783,-104.2834\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"64.3347,-138.0394 73.8806,-142.6356 69.2844,-133.0896 64.3347,-138.0394\"/>\n",
       "</g>\n",
       "<!-- 3&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>3&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M106.8327,-135.4321C115.8608,-126.404 126.4899,-115.7749 135.5089,-106.7559\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"104.2834,-133.0316 99.6872,-142.5776 109.2332,-137.9814 104.2834,-133.0316\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"138.0394,-109.1752 142.6356,-99.6292 133.0896,-104.2254 138.0394,-109.1752\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7fbfd8479f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "S = [1, 2, 3, 4]\n",
    "P = np.array([[0, 1/2, 0, 1/2], [1/2, 0, 1/2, 0], [0, 1/2, 0, 1/2], [1/2, 0, 1/2, 0]])\n",
    "v = [1, 2, 2, 1]\n",
    "\n",
    "display(Math(r'\\mathcal{S} = \\{' + ','.join(map(str,S)) + '\\}'))\n",
    "ctmc.disp_ctmc(P, S, v, frac=True)\n",
    "digraph = mcviz.dot_mc(P, S, bidir=True)\n",
    "svg_str = digraph.pipe().decode('utf-8')\n",
    "display(digraph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedded DTMC\n",
    "\n",
    "Note that the specification of the continuous-time Markov chain includes an “embedded” discrete-time Markov chain (given by $\\mathcal{S}$ and $\\mathbf{P}$).  The embedded DTMC still describes much of the dynamics; in particular, the sequence of visits.  The CTMC additionally describes the length of each visit."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 1.1\n",
    "\n",
    "Consider the continuous-time Markov chain specified above.\n",
    "\n",
    "Given $t = 1.3489$ and $X(t) = 3$:\n",
    "* What is the expected amount of time until the Markov chain leaves state $3$?  \n",
    "* What is the probability it next moves to state $2$?\n",
    "* What is the probability that it moves from state $3$ to state $4$ within the next 1.5 time units? (Assume it remains in state $3$ until it moves.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample Path Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def slider_anim(s):\n",
    "    for t in np.arange(s.min, s.max, s.step):\n",
    "        time.sleep(0.01)\n",
    "        s.value=t            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "steps = 50\n",
    "sample_path = ctmc.sim_path(P, v, [1, 0, 0, 0], paths=1, steps=steps, S=S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt = 0.05\n",
    "t_slider = widgets.FloatSlider(value=0, \n",
    "                               min=min(sample_path['start']),\n",
    "                               max=max(sample_path['end']), \n",
    "                               step=dt,\n",
    "                               description='t')\n",
    "diagram = widgets.Output()\n",
    "with diagram:\n",
    "    dtmc.draw(sample_path.loc[0,'X'], svg_str)\n",
    "    \n",
    "def diagram_draw(i, out):\n",
    "    with out:\n",
    "        clear_output(wait=True)\n",
    "        dtmc.draw(i, svg_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55a72b24178e4b9bb8741949b9a5a1bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatSlider(value=0.0, description='t', max=26.855860434442018, step=0.05)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"736516e4-7007-4a9a-996b-40dc648530dd\" data-root-id=\"1003\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"59147b2d-e6f5-44ca-b370-7b253e62f4a7\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1014\",\"type\":\"LinearAxis\"}],\"center\":[{\"id\":\"1018\",\"type\":\"Grid\"},{\"id\":\"1023\",\"type\":\"Grid\"},{\"id\":\"1049\",\"type\":\"Span\"}],\"left\":[{\"id\":\"1019\",\"type\":\"LinearAxis\"}],\"renderers\":[{\"id\":\"1033\",\"type\":\"GlyphRenderer\"},{\"id\":\"1038\",\"type\":\"GlyphRenderer\"},{\"id\":\"1043\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1004\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1026\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1048\",\"type\":\"Range1d\"},\"x_scale\":{\"id\":\"1010\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1047\",\"type\":\"Range1d\"},\"y_scale\":{\"id\":\"1012\",\"type\":\"LinearScale\"}},\"id\":\"1003\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1058\",\"type\":\"Selection\"},{\"attributes\":{\"data_source\":{\"id\":\"1040\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1041\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1042\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1044\",\"type\":\"CDSView\"}},\"id\":\"1043\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"data_source\":{\"id\":\"1030\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1031\",\"type\":\"Segment\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1032\",\"type\":\"Segment\"},\"selection_glyph\":null,\"view\":{\"id\":\"1034\",\"type\":\"CDSView\"}},\"id\":\"1033\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"text\":\"CTMC Sample Path\"},\"id\":\"1004\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1059\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":5},\"x0\":{\"field\":\"x0\"},\"x1\":{\"field\":\"x1\"},\"y0\":{\"field\":\"y0\"},\"y1\":{\"field\":\"y1\"}},\"id\":\"1032\",\"type\":\"Segment\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":8},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1037\",\"type\":\"Circle\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1024\",\"type\":\"PanTool\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1025\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"1053\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null,\"end\":5},\"id\":\"1047\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"1051\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"ticks\":[1,2,3,4]},\"id\":\"1045\",\"type\":\"FixedTicker\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":8},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1036\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1010\",\"type\":\"LinearScale\"},{\"attributes\":{\"source\":{\"id\":\"1040\",\"type\":\"ColumnDataSource\"}},\"id\":\"1044\",\"type\":\"CDSView\"},{\"attributes\":{\"active_drag\":{\"id\":\"1024\",\"type\":\"PanTool\"},\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":{\"id\":\"1025\",\"type\":\"WheelZoomTool\"},\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1024\",\"type\":\"PanTool\"},{\"id\":\"1025\",\"type\":\"WheelZoomTool\"}]},\"id\":\"1026\",\"type\":\"Toolbar\"},{\"attributes\":{\"line_color\":{\"value\":\"#1f77b4\"},\"line_width\":{\"value\":5},\"x0\":{\"field\":\"x0\"},\"x1\":{\"field\":\"x1\"},\"y0\":{\"field\":\"y0\"},\"y1\":{\"field\":\"y1\"}},\"id\":\"1031\",\"type\":\"Segment\"},{\"attributes\":{},\"id\":\"1054\",\"type\":\"Selection\"},{\"attributes\":{\"data_source\":{\"id\":\"1035\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1036\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1037\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1039\",\"type\":\"CDSView\"}},\"id\":\"1038\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":{\"__ndarray__\":\"AAAAAAAAAADITawLdAfkPzatin+4JOQ/HGg3Vzff5z95yncQmtHqP1UUM4Rpgu8/HgiIf6ATA0CTAdUsadUEQKAvVZm5xAZAgCG8on7TDUAA8IyxA7sOQF37yt391A5Aejl0xUyvEkBA4jPoHHEVQNV845OjghVAo/VRn4lOFkA8oXZWjoUdQDSzjeyZHR9AWAt/2HUII0CDiBQQ3e8kQMhseqQDLCVAfG/H/BgnJkAl+SVWNhIoQMr3QqK9RCtAsEtr5dVqK0C8sq942xQtQE8jGRRkYi1AuYXRHfmGLUA4uyBLLL8tQLz58yE0DS9AY4XxCUEZL0BaPq6PKGcvQCyk5q9RBzJAL/HJwasmM0D48uiZuUszQO+VARugfzNABVhFwhO8M0BKHsRCNnA0QDZVn7YIozRAf50NQ7IuNUDaMVsQ5Ew1QMggZxIzHzZAIH9g7P+RNkDdaIPB0FA4QKlHGn4pUjhAFtAE99Q9OUDtSCB+TLY5QCD9PAaYADpAk4H3owYJOkAytxeGciQ6QJe5SuiGoDpA\",\"dtype\":\"float64\",\"shape\":[51]},\"y\":[1,4,3,4,3,4,3,2,3,4,1,2,1,2,1,2,1,4,3,2,1,2,1,2,1,2,1,2,1,4,3,4,1,2,3,2,1,2,3,4,1,4,3,4,1,4,3,2,3,2,3]},\"selected\":{\"id\":\"1056\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1057\",\"type\":\"UnionRenderers\"}},\"id\":\"1035\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":8},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1042\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"white\"},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":8},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1041\",\"type\":\"Circle\"},{\"attributes\":{\"bounds\":[-0.5,27.355860434442018],\"callback\":null,\"end\":3.841315638650485,\"start\":-0.5},\"id\":\"1048\",\"type\":\"Range1d\"},{\"attributes\":{},\"id\":\"1012\",\"type\":\"LinearScale\"},{\"attributes\":{\"axis_label\":\"t\",\"formatter\":{\"id\":\"1053\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1015\",\"type\":\"BasicTicker\"}},\"id\":\"1014\",\"type\":\"LinearAxis\"},{\"attributes\":{\"callback\":null,\"data\":{\"x0\":{\"__ndarray__\":\"AAAAAAAAAADITawLdAfkPzatin+4JOQ/HGg3Vzff5z95yncQmtHqP1UUM4Rpgu8/HgiIf6ATA0CTAdUsadUEQKAvVZm5xAZAgCG8on7TDUAA8IyxA7sOQF37yt391A5Aejl0xUyvEkBA4jPoHHEVQNV845OjghVAo/VRn4lOFkA8oXZWjoUdQDSzjeyZHR9AWAt/2HUII0CDiBQQ3e8kQMhseqQDLCVAfG/H/BgnJkAl+SVWNhIoQMr3QqK9RCtAsEtr5dVqK0C8sq942xQtQE8jGRRkYi1AuYXRHfmGLUA4uyBLLL8tQLz58yE0DS9AY4XxCUEZL0BaPq6PKGcvQCyk5q9RBzJAL/HJwasmM0D48uiZuUszQO+VARugfzNABVhFwhO8M0BKHsRCNnA0QDZVn7YIozRAf50NQ7IuNUDaMVsQ5Ew1QMggZxIzHzZAIH9g7P+RNkDdaIPB0FA4QKlHGn4pUjhAFtAE99Q9OUDtSCB+TLY5QCD9PAaYADpAk4H3owYJOkAytxeGciQ6QJe5SuiGoDpA\",\"dtype\":\"float64\",\"shape\":[51]},\"x1\":{\"__ndarray__\":\"yE2sC3QH5D82rYp/uCTkPxxoN1c33+c/ecp3EJrR6j9VFDOEaYLvPx4IiH+gEwNAkwHVLGnVBECgL1WZucQGQIAhvKJ+0w1AAPCMsQO7DkBd+8rd/dQOQHo5dMVMrxJAQOIz6BxxFUDVfOOTo4IVQKP1UZ+JThZAPKF2Vo6FHUA0s43smR0fQFgLf9h1CCNAg4gUEN3vJEDIbHqkAywlQHxvx/wYJyZAJfklVjYSKEDK90KivUQrQLBLa+XVaitAvLKveNsULUBPIxkUZGItQLmF0R35hi1AOLsgSyy/LUC8+fMhNA0vQGOF8QlBGS9AWj6ujyhnL0AspOavUQcyQC/xycGrJjNA+PLomblLM0DvlQEboH8zQAVYRcITvDNASh7EQjZwNEA2VZ+2CKM0QH+dDUOyLjVA2jFbEORMNUDIIGcSMx82QCB/YOz/kTZA3WiDwdBQOECpRxp+KVI4QBbQBPfUPTlA7Uggfky2OUAg/TwGmAA6QJOB96MGCTpAMrcXhnIkOkCXuUrohqA6QGveX6sZ2zpA\",\"dtype\":\"float64\",\"shape\":[51]},\"y0\":[1,4,3,4,3,4,3,2,3,4,1,2,1,2,1,2,1,4,3,2,1,2,1,2,1,2,1,2,1,4,3,4,1,2,3,2,1,2,3,4,1,4,3,4,1,4,3,2,3,2,3],\"y1\":[1,4,3,4,3,4,3,2,3,4,1,2,1,2,1,2,1,4,3,2,1,2,1,2,1,2,1,2,1,4,3,4,1,2,3,2,1,2,3,4,1,4,3,4,1,4,3,2,3,2,3]},\"selected\":{\"id\":\"1054\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1055\",\"type\":\"UnionRenderers\"}},\"id\":\"1030\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"source\":{\"id\":\"1030\",\"type\":\"ColumnDataSource\"}},\"id\":\"1034\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"X(t)\",\"formatter\":{\"id\":\"1051\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1045\",\"type\":\"FixedTicker\"}},\"id\":\"1019\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"1020\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1055\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1015\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1057\",\"type\":\"UnionRenderers\"},{\"attributes\":{\"ticker\":{\"id\":\"1015\",\"type\":\"BasicTicker\"}},\"id\":\"1018\",\"type\":\"Grid\"},{\"attributes\":{\"dimension\":\"height\",\"line_color\":{\"value\":\"red\"},\"line_dash\":[6],\"line_width\":{\"value\":3},\"location\":0},\"id\":\"1049\",\"type\":\"Span\"},{\"attributes\":{\"source\":{\"id\":\"1035\",\"type\":\"ColumnDataSource\"}},\"id\":\"1039\",\"type\":\"CDSView\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":{\"__ndarray__\":\"yE2sC3QH5D82rYp/uCTkPxxoN1c33+c/ecp3EJrR6j9VFDOEaYLvPx4IiH+gEwNAkwHVLGnVBECgL1WZucQGQIAhvKJ+0w1AAPCMsQO7DkBd+8rd/dQOQHo5dMVMrxJAQOIz6BxxFUDVfOOTo4IVQKP1UZ+JThZAPKF2Vo6FHUA0s43smR0fQFgLf9h1CCNAg4gUEN3vJEDIbHqkAywlQHxvx/wYJyZAJfklVjYSKEDK90KivUQrQLBLa+XVaitAvLKveNsULUBPIxkUZGItQLmF0R35hi1AOLsgSyy/LUC8+fMhNA0vQGOF8QlBGS9AWj6ujyhnL0AspOavUQcyQC/xycGrJjNA+PLomblLM0DvlQEboH8zQAVYRcITvDNASh7EQjZwNEA2VZ+2CKM0QH+dDUOyLjVA2jFbEORMNUDIIGcSMx82QCB/YOz/kTZA3WiDwdBQOECpRxp+KVI4QBbQBPfUPTlA7Uggfky2OUAg/TwGmAA6QJOB96MGCTpAMrcXhnIkOkCXuUrohqA6QGveX6sZ2zpA\",\"dtype\":\"float64\",\"shape\":[51]},\"y\":[1,4,3,4,3,4,3,2,3,4,1,2,1,2,1,2,1,4,3,2,1,2,1,2,1,2,1,2,1,4,3,4,1,2,3,2,1,2,3,4,1,4,3,4,1,4,3,2,3,2,3]},\"selected\":{\"id\":\"1058\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1059\",\"type\":\"UnionRenderers\"}},\"id\":\"1040\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1056\",\"type\":\"Selection\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1020\",\"type\":\"BasicTicker\"}},\"id\":\"1023\",\"type\":\"Grid\"}],\"root_ids\":[\"1003\"]},\"title\":\"Bokeh Application\",\"version\":\"1.3.4\"}};\n",
       "  var render_items = [{\"docid\":\"59147b2d-e6f5-44ca-b370-7b253e62f4a7\",\"notebook_comms_target\":\"1060\",\"roots\":{\"1003\":\"736516e4-7007-4a9a-996b-40dc648530dd\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1003"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6762108039ca401a8a65f523dce48872",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output(outputs=({'output_type': 'display_data', 'data': {'text/plain': '<IPython.core.display.SVG object>', 'i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(t_slider)\n",
    "p, h, t_mark = ctmc.plot_path(sample_path, S)\n",
    "display(diagram)\n",
    "widgets.interactive_output(lambda t: ctmc.path_show_t(t, sample_path, p, h, t_mark), {'t': t_slider});\n",
    "widgets.interactive_output(lambda t: diagram_draw(sample_path.loc[(sample_path['start']<=t)&(sample_path['end']>t), 'X'].values[0], diagram), {'t': t_slider});"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Animate the sample path above\n",
    "slider_anim(t_slider)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample path of embedded DTMC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd2991be14fd4bdbb955d3acd3f65230",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntSlider(value=0, description='n', max=50)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"a2836a55-37c3-42a9-a6d0-b3df6634155e\" data-root-id=\"1121\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"f9dc4736-9831-4238-bb14-bc2470341bce\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1132\",\"type\":\"LinearAxis\"}],\"center\":[{\"id\":\"1136\",\"type\":\"Grid\"},{\"id\":\"1141\",\"type\":\"Grid\"},{\"id\":\"1156\",\"type\":\"Span\"}],\"left\":[{\"id\":\"1137\",\"type\":\"LinearAxis\"}],\"renderers\":[{\"id\":\"1150\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1122\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1144\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1155\",\"type\":\"Range1d\"},\"x_scale\":{\"id\":\"1128\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"1154\",\"type\":\"Range1d\"},\"y_scale\":{\"id\":\"1130\",\"type\":\"LinearScale\"}},\"id\":\"1121\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"1138\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"1172\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1171\",\"type\":\"Selection\"},{\"attributes\":{\"ticks\":[1,2,3,4]},\"id\":\"1152\",\"type\":\"FixedTicker\"},{\"attributes\":{\"axis_label\":\"X_n\",\"formatter\":{\"id\":\"1168\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1152\",\"type\":\"FixedTicker\"}},\"id\":\"1137\",\"type\":\"LinearAxis\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":8},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1149\",\"type\":\"Circle\"},{\"attributes\":{\"text\":\"Embedded DTMC Sample Path\"},\"id\":\"1122\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1168\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"dimension\":\"height\",\"line_color\":{\"value\":\"red\"},\"line_dash\":[6],\"line_width\":{\"value\":3},\"location\":0},\"id\":\"1156\",\"type\":\"Span\"},{\"attributes\":{},\"id\":\"1130\",\"type\":\"LinearScale\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1138\",\"type\":\"BasicTicker\"}},\"id\":\"1141\",\"type\":\"Grid\"},{\"attributes\":{\"axis_label\":\"n\",\"formatter\":{\"id\":\"1170\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1133\",\"type\":\"BasicTicker\"}},\"id\":\"1132\",\"type\":\"LinearAxis\"},{\"attributes\":{\"callback\":null,\"data\":{\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50],\"y\":[1,4,3,4,3,4,3,2,3,4,1,2,1,2,1,2,1,4,3,2,1,2,1,2,1,2,1,2,1,4,3,4,1,2,3,2,1,2,3,4,1,4,3,4,1,4,3,2,3,2,3]},\"selected\":{\"id\":\"1171\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1172\",\"type\":\"UnionRenderers\"}},\"id\":\"1147\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1170\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1133\",\"type\":\"BasicTicker\"},{\"attributes\":{\"source\":{\"id\":\"1147\",\"type\":\"ColumnDataSource\"}},\"id\":\"1151\",\"type\":\"CDSView\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"units\":\"screen\",\"value\":8},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"1148\",\"type\":\"Circle\"},{\"attributes\":{\"active_drag\":{\"id\":\"1142\",\"type\":\"PanTool\"},\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":{\"id\":\"1143\",\"type\":\"WheelZoomTool\"},\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1142\",\"type\":\"PanTool\"},{\"id\":\"1143\",\"type\":\"WheelZoomTool\"}]},\"id\":\"1144\",\"type\":\"Toolbar\"},{\"attributes\":{\"data_source\":{\"id\":\"1147\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1148\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"1149\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"1151\",\"type\":\"CDSView\"}},\"id\":\"1150\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1142\",\"type\":\"PanTool\"},{\"attributes\":{\"callback\":null,\"end\":5},\"id\":\"1154\",\"type\":\"Range1d\"},{\"attributes\":{\"bounds\":[-1,52],\"callback\":null,\"end\":10,\"start\":-1},\"id\":\"1155\",\"type\":\"Range1d\"},{\"attributes\":{\"ticker\":{\"id\":\"1133\",\"type\":\"BasicTicker\"}},\"id\":\"1136\",\"type\":\"Grid\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1143\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"1128\",\"type\":\"LinearScale\"}],\"root_ids\":[\"1121\"]},\"title\":\"Bokeh Application\",\"version\":\"1.3.4\"}};\n",
       "  var render_items = [{\"docid\":\"f9dc4736-9831-4238-bb14-bc2470341bce\",\"notebook_comms_target\":\"1173\",\"roots\":{\"1121\":\"a2836a55-37c3-42a9-a6d0-b3df6634155e\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1121"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a08ad4f6a26842fe9fb41b9322a9eadf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97477b851c48479ba7f9f6f0ba25ab39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_slider = widgets.IntSlider(value=0,\n",
    "                             min=0,\n",
    "                             max=steps,\n",
    "                             description='n')\n",
    "dtmc_diagram = widgets.Output()\n",
    "with dtmc_diagram:\n",
    "    dtmc.draw(sample_path.loc[0,'X'], svg_str)\n",
    "\n",
    "display(n_slider)\n",
    "dtmc_p, dtmc_h, n_mark = ctmc.plot_dtmc_path(sample_path, S)\n",
    "display(dtmc_diagram)\n",
    "widgets.interactive_output(lambda n: ctmc.dtmc_path_show_n(n, sample_path, dtmc_p, dtmc_h, n_mark), {'n': n_slider});\n",
    "widgets.interactive_output(lambda n: diagram_draw(sample_path.loc[n, 'X'], dtmc_diagram), {'n': n_slider})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}