2-4-DTMC-driver.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"1001\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null && id in Bokeh.index) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var id = msg.content.text.trim();\n",
       "            if (id in Bokeh.index) {\n",
       "              Bokeh.index[id].model.document.clear();\n",
       "              delete Bokeh.index[id];\n",
       "            }\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"1001\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error() {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < css_urls.length; i++) {\n",
       "      var url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error;\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };var element = document.getElementById(\"1001\");\n",
       "  if (element == null) {\n",
       "    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.3.4.min.js\"];\n",
       "  var css_urls = [];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {} // ensure no trailing comma for IE\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"1001\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }\n\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n  };var element = document.getElementById(\"1001\");\n  if (element == null) {\n    console.error(\"Bokeh: ERROR: autoload.js configured with elementid '1001' but no matching script tag was found. \")\n    return false;\n  }\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-1.3.4.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-1.3.4.min.js\"];\n  var css_urls = [];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"1001\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(css_urls, js_urls, function() {\n      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import display, Math\n",
    "\n",
    "import dtmc\n",
    "import mcviz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DTMC: Section 4 - The Ride-Share Driver\n",
    "\n",
    "We will consider a (slightly) different DTMC model and introduce the idea of unconditional probability distributions for $X_n$ and the stationary distribution."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Setting\n",
    "\n",
    "Suppose you are driving for a ride-share service (e.g., Uber, Lyft) in the Miami area.  Starting in your home neighborhood, you pick up a passenger and drive to her destination and drop her off.  Your next pick-up is in the same neighborhood as your previous drop-off, drive to that passenger's destination and drop him off, and repeat.\n",
    "\n",
    "What is the probability you end up back in Coral Gables after 30 passengers?\n",
    "\n",
    "**Assumptions**\n",
    "\n",
    "* Simplify the Miami area to four neighborhoods:\n",
    "    1. Coral Gables\n",
    "    2. Kendall\n",
    "    3. Downtown\n",
    "    4. Miami Beach\n",
    "\n",
    "\n",
    "* You always pick-up the next passenger in the same neighborhood as your previous drop-off.  (That is, you don't choose to go to a different neighborhood without a passenger.)\n",
    "\n",
    "* The destination of any passenger is random and\n",
    "    * the distribution is known\n",
    "    * it depends on the passenger's pick-up location\n",
    "    * but does not depend on any previous passenger's pick-up or drop-off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\displaystyle \\mathcal{S} = \\{1,2,3,4\\}$"
      ],
      "text/plain": [
       "<IPython.core.display.Math object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/latex": [
       "$\\displaystyle \\mathbf{P} = \\pmatrix{0.3& 0.2& 0.4& 0.1\\\\ 0.4& 0.1& 0.4& 0.1\\\\ 0.7& 0.2& 0.0& 0.1\\\\ 0.5& 0.3& 0.1& 0.1}$"
      ],
      "text/plain": [
       "<IPython.core.display.Math object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: Markov chain Pages: 1 -->\n",
       "<svg width=\"200pt\" height=\"182pt\"\n",
       " viewBox=\"0.00 0.00 199.51 181.51\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 177.5099)\">\n",
       "<title>Markov chain</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-177.5099 195.5099,-177.5099 195.5099,4 -4,4\"/>\n",
       "<!-- 1 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>1</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"86.7549\" cy=\"-18\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"86.7549\" y=\"-14.3\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">1</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;1 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>1&#45;&gt;1</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M103.7922,-24.3792C113.6128,-25.6811 122.7549,-23.5547 122.7549,-18 122.7549,-14.5283 119.1838,-12.3958 114.1079,-11.6025\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"113.786,-8.1029 103.7922,-11.6208 113.7985,-15.1029 113.786,-8.1029\"/>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>2</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"155.5099\" cy=\"-86.7549\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"155.5099\" y=\"-83.0549\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>1&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M106.8327,-38.0778C115.8608,-47.1058 126.4899,-57.735 135.5089,-66.754\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"109.2332,-35.5285 99.6872,-30.9323 104.2834,-40.4783 109.2332,-35.5285\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"133.0896,-69.2844 142.6356,-73.8806 138.0394,-64.3347 133.0896,-69.2844\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>3</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"86.7549\" cy=\"-155.5099\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"86.7549\" y=\"-151.8099\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">3</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>1&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M86.7549,-46.3411C86.7549,-69.9564 86.7549,-103.4331 86.7549,-127.0746\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"90.255,-46.2602 86.7549,-36.2602 83.255,-46.2603 90.255,-46.2602\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"83.255,-127.1741 86.7549,-137.1741 90.255,-127.1741 83.255,-127.1741\"/>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>4</title>\n",
       "<ellipse fill=\"none\" stroke=\"#000000\" cx=\"18\" cy=\"-86.7549\" rx=\"18\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"18\" y=\"-83.0549\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">4</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;4 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>1&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M66.6772,-38.0778C57.6491,-47.1058 47.0199,-57.735 38.001,-66.754\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"69.2264,-40.4783 73.8227,-30.9323 64.2767,-35.5285 69.2264,-40.4783\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"35.4705,-64.3347 30.8743,-73.8806 40.4203,-69.2844 35.4705,-64.3347\"/>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;2 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>2&#45;&gt;2</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M172.5472,-93.1341C182.3678,-94.436 191.5099,-92.3096 191.5099,-86.7549 191.5099,-83.2833 187.9387,-81.1507 182.8628,-80.3574\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"182.5409,-76.8579 172.5472,-80.3757 182.5534,-83.8579 182.5409,-76.8579\"/>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;3 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>2&#45;&gt;3</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M135.4321,-106.8327C126.404,-115.8608 115.7749,-126.4899 106.7559,-135.5089\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"137.9814,-109.2332 142.5776,-99.6872 133.0316,-104.2834 137.9814,-109.2332\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"104.2254,-133.0896 99.6292,-142.6356 109.1752,-138.0394 104.2254,-133.0896\"/>\n",
       "</g>\n",
       "<!-- 2&#45;&gt;4 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>2&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M127.1688,-86.7549C103.5535,-86.7549 70.0768,-86.7549 46.4353,-86.7549\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"127.2497,-90.255 137.2496,-86.7549 127.2496,-83.255 127.2497,-90.255\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"46.3358,-83.255 36.3358,-86.7549 46.3357,-90.255 46.3358,-83.255\"/>\n",
       "</g>\n",
       "<!-- 3&#45;&gt;4 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>3&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M66.6772,-135.4321C57.6491,-126.404 47.0199,-115.7749 38.001,-106.7559\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"64.2767,-137.9814 73.8227,-142.5776 69.2264,-133.0316 64.2767,-137.9814\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"40.4203,-104.2254 30.8743,-99.6292 35.4705,-109.1752 40.4203,-104.2254\"/>\n",
       "</g>\n",
       "<!-- 4&#45;&gt;4 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>4&#45;&gt;4</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M35.0373,-93.1341C44.8579,-94.436 54,-92.3096 54,-86.7549 54,-83.2833 50.4289,-81.1507 45.3529,-80.3574\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"45.031,-76.8579 35.0373,-80.3757 45.0435,-83.8579 45.031,-76.8579\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7ff3f0373110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "S = [1, 2, 3, 4]\n",
    "P = np.array([[0.3, 0.2, 0.4, 0.1],\n",
    "              [0.4, 0.1, 0.4, 0.1],\n",
    "              [0.7, 0.2, 0, 0.1],\n",
    "              [0.5, 0.3, 0.1, 0.1]\n",
    "             ])\n",
    "names = ['Gables', 'Kendall', 'Downtown', 'Beach']\n",
    "dtmc.disp_dtmc(P, S)\n",
    "display(mcviz.dot_mc(P, S, bidir=True, self_edge=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 4.1\n",
    "\n",
    "Using the transition probability matrix $\\mathbf{P}$ above and given that you start (pick up your first passenger) in Coral Gables, what is the probability that:\n",
    "* you drop off your first passenger in Coral Gables?  Kendall?  Downtown?  Miami Beach?\n",
    "* you drop off your tenth passenger in Downtown?\n",
    "* you end up back in Coral Gables after 30 passengers?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Distribution and Unconditional Probabilities\n",
    "\n",
    "So far, we've only looked at conditional probabilities that were elements of the $m$-step transition probability matrix, such as:\n",
    "\n",
    "$$ p_{13}^{(10)} = \\mathsf{P}(X_{10} = 3 \\,|\\, X_0 = 1) $$\n",
    "\n",
    "However, if we assume a probability distribution for $X_0$ (a distribution over $\\mathcal{S}$) we can calculate unconditional probability distribution of every $X_n$, $n \\geq 1$.\n",
    "\n",
    "### Distribution Over $\\mathcal{S}$\n",
    "\n",
    "Remember that each step in the stochastic process is a random variable $X_n$ so it has a probability distribution over the state space $\\mathcal{S}$.  Since $\\mathcal{S}$ is discrete, this will be a discrete distribution.  Instead of a probability mass function, we will write the distribution of $X_n$ as a *vector* $\\mathbf{d}_n$ where the $i$th element of the vector is $\\mathsf{P}(X_n = i)$.  (You can think of this as a probability mass vector instead of a probability mass function.)\n",
    "\n",
    "$$ \\mathbf{d}_n = \\begin{pmatrix} \\mathsf{P}(X_n = 1) & \\mathsf{P}(X_n = 2) & \\mathsf{P}(X_n = 3) & \\mathsf{P}(X_n = 4) \\end{pmatrix} $$\n",
    "\n",
    "For example, the probability distribution for $X_0$ for the state space $\\mathcal{S} = \\{1, 2, 3, 4\\}$ may be $\\mathsf{P}(X_0 = i) = 1/4$ for all $i \\in \\mathcal{S}$ (so $X_0$ is uniformly distributed over $\\mathcal{S}$) in which case the vector is\n",
    "\n",
    "$$ \\mathbf{d}_0 = \\begin{pmatrix} 1/4 & 1/4 & 1/4 & 1/4 \\end{pmatrix} $$\n",
    "\n",
    "Another distribution for $X_0$ could be\n",
    "\n",
    "$$ \\mathbf{d}_0 = \\begin{pmatrix} 1 & 0 & 0 & 0 \\end{pmatrix} $$\n",
    "\n",
    "which corresponds to starting in state $1$ with probability $1$.\n",
    "\n",
    "Any vector that represents a distribution must have non-negative elements that sum to 1.\n",
    "\n",
    "### Unconditional Probabilities\n",
    "\n",
    "To get unconditional probabilities, we simply condition on $X_0$:\n",
    "\n",
    "$$\\mathsf{P}(X_1 = j) = \\sum_{i \\in \\mathcal{S}} \\mathsf{P}(X_1 = j \\,|\\, X_0 = i) \\mathsf{P}(X_0 = i)$$\n",
    "\n",
    "or in matrix and vector notation\n",
    "\n",
    "$$\\mathbf{d}_1 = \\mathbf{d}_0 \\mathbf{P}.$$\n",
    "\n",
    "In general\n",
    "\n",
    "$$\\mathbf{d}_n = \\mathbf{d}_{n-1} \\mathbf{P} = \\mathbf{d}_0 \\mathbf{P}^n = \\mathbf{d}_0 \\mathbf{P}^{(n)}.$$\n",
    "\n",
    "We can think of $\\mathbf{d}_n$ as the (probabilistic) state of the system at time $n$ and the transition probability matrix $\\mathbf{P}$ as “propogating” the system one step forward in time (and $\\mathbf{d}_0$ is the initial condition).  It is in this way that $\\mathbf{P}$ defines the dynamics of the Markov chain.  (In the same way that a dynamics of a projectile can be defined by a differential equation or a quantum mechanical wavefunction can be defined by the Schrödinger equation.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 4.2\n",
    "\n",
    "For $\\mathbf{d}_0 = \\begin{pmatrix} 1 & 0 & 0 & 0 \\end{pmatrix}$ calculate $\\mathbf{d}_5$, $\\mathbf{d}_{10}$, and $\\mathbf{d}_{20}$ and interpret each distribution.\n",
    "\n",
    "Please don't do this by hand.  You may use any matrix multiplication software you like (e.g., R, MATLAB, Python, Mathematica, a fancy calculator).  \n",
    "\n",
    "If you want to use Python (and Numpy), the relevant functions are [`dot`](https://docs.scipy.org/doc/numpy/reference/generated/numpy.dot.html) and [`matrix_power`](https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.matrix_power.html) (and remember we imported the `numpy` package as `np`, so you would write `np.dot` and `np.linalg.matrix_power` to use these functions)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 4.3\n",
    "\n",
    "Pick any (valid) distribution for $\\mathbf{d}_0$ and calculate $\\mathbf{d}_{20}$.  What do you notice?  Try changing $\\mathbf{d}_0$ and calculate $\\mathbf{d}_{20}$ again.\n",
    "\n",
    "Calculate $\\mathbf{P}^{(20)}$.  How does that explain your result?  What does this tell you about $\\mathbf{P}^{(m)}$ for $m \\geq 20$?\n",
    "\n",
    "What does this mean in the setting of this problem?  Is that surprising?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 4.4\n",
    "\n",
    "Find a distribution (vector) $\\pi$ that satisfies\n",
    "\n",
    "$$ \\pi = \\pi \\mathbf{P} $$\n",
    "\n",
    "$\\pi$ is known as the “stationary distribution.”  Why do you think it has this name?\n",
    "\n",
    "*Hint:* Look at your results for Exercise 4.3.\n",
    "\n",
    "**Note:** In this course, we will never use $\\pi$ to denote the mathematical constant.  Instead, $\\pi$ will almost always denote the stationary distribution of a Markov chain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.475, 0.2  , 0.225, 0.1  ])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pi = [0.25, 0.25, 0.25, 0.25] # Input your answer here\n",
    "np.dot(pi, P) # Check that it satisifes pi = pi P"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization\n",
    "\n",
    "The following codeblock plots the distribution $d_n$ for various $n$, so you can see how the probability distribution propogates over time.  Be sure to specify a valid initial distribution $d_0$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/latex": [
       "$\\displaystyle d_0 = \\pmatrix{1& 0& 0& 0}$"
      ],
      "text/plain": [
       "<IPython.core.display.Math object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0bb80b0d16245dd930e277411774bcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=0, description='n', max=50), Output()), _dom_classes=('widget-interact',…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "  <div class=\"bk-root\" id=\"07f0cddf-9776-4339-9823-5bb0adc5d26c\" data-root-id=\"1002\"></div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"a54ec473-cbdd-4c92-903a-c6febf58d7d0\":{\"roots\":{\"references\":[{\"attributes\":{\"below\":[{\"id\":\"1011\",\"type\":\"CategoricalAxis\"}],\"center\":[{\"id\":\"1014\",\"type\":\"Grid\"},{\"id\":\"1019\",\"type\":\"Grid\"},{\"id\":\"1035\",\"type\":\"LabelSet\"}],\"left\":[{\"id\":\"1015\",\"type\":\"LinearAxis\"}],\"renderers\":[{\"id\":\"1037\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"1040\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1026\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"1003\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"1007\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"1039\",\"type\":\"Range1d\"},\"y_scale\":{\"id\":\"1009\",\"type\":\"LinearScale\"}},\"id\":\"1002\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"source\":{\"id\":\"1033\",\"type\":\"ColumnDataSource\"},\"text\":{\"field\":\"top\"},\"text_align\":\"center\",\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"top\"}},\"id\":\"1035\",\"type\":\"LabelSet\"},{\"attributes\":{\"callback\":null,\"factors\":[\"Gables\",\"Kendall\",\"Downtown\",\"Beach\"]},\"id\":\"1003\",\"type\":\"FactorRange\"},{\"attributes\":{\"fill_color\":{\"value\":\"blue\"},\"top\":{\"field\":\"top\"},\"width\":{\"value\":0.8},\"x\":{\"field\":\"x\"}},\"id\":\"1034\",\"type\":\"VBar\"},{\"attributes\":{},\"id\":\"1007\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"1020\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"1046\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1016\",\"type\":\"BasicTicker\"},{\"attributes\":{\"ticker\":{\"id\":\"1012\",\"type\":\"CategoricalTicker\"}},\"id\":\"1014\",\"type\":\"Grid\"},{\"attributes\":{\"dimension\":1,\"ticker\":{\"id\":\"1016\",\"type\":\"BasicTicker\"}},\"id\":\"1019\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null},\"id\":\"1039\",\"type\":\"Range1d\"},{\"attributes\":{\"source\":{\"id\":\"1033\",\"type\":\"ColumnDataSource\"}},\"id\":\"1038\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1012\",\"type\":\"CategoricalTicker\"},{\"attributes\":{},\"id\":\"1023\",\"type\":\"SaveTool\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_multi\":null,\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"1020\",\"type\":\"PanTool\"},{\"id\":\"1021\",\"type\":\"WheelZoomTool\"},{\"id\":\"1022\",\"type\":\"BoxZoomTool\"},{\"id\":\"1023\",\"type\":\"SaveTool\"},{\"id\":\"1024\",\"type\":\"ResetTool\"},{\"id\":\"1025\",\"type\":\"HelpTool\"}]},\"id\":\"1026\",\"type\":\"Toolbar\"},{\"attributes\":{\"text\":\"\"},\"id\":\"1040\",\"type\":\"Title\"},{\"attributes\":{\"overlay\":{\"id\":\"1048\",\"type\":\"BoxAnnotation\"}},\"id\":\"1022\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"1045\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"axis_label\":\"d_n\",\"formatter\":{\"id\":\"1043\",\"type\":\"BasicTickFormatter\"},\"ticker\":{\"id\":\"1016\",\"type\":\"BasicTicker\"}},\"id\":\"1015\",\"type\":\"LinearAxis\"},{\"attributes\":{\"callback\":null,\"data\":{\"top\":{\"__ndarray__\":\"AAAAAAAA8D8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"dtype\":\"float64\",\"shape\":[4]},\"x\":[\"Gables\",\"Kendall\",\"Downtown\",\"Beach\"]},\"selected\":{\"id\":\"1047\",\"type\":\"Selection\"},\"selection_policy\":{\"id\":\"1046\",\"type\":\"UnionRenderers\"}},\"id\":\"1033\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"1025\",\"type\":\"HelpTool\"},{\"attributes\":{},\"id\":\"1024\",\"type\":\"ResetTool\"},{\"attributes\":{\"data_source\":{\"id\":\"1033\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"1034\",\"type\":\"VBar\"},\"hover_glyph\":null,\"muted_glyph\":null,\"view\":{\"id\":\"1038\",\"type\":\"CDSView\"}},\"id\":\"1037\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"formatter\":{\"id\":\"1045\",\"type\":\"CategoricalTickFormatter\"},\"ticker\":{\"id\":\"1012\",\"type\":\"CategoricalTicker\"}},\"id\":\"1011\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"1048\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"1009\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"1043\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"1047\",\"type\":\"Selection\"}],\"root_ids\":[\"1002\"]},\"title\":\"Bokeh Application\",\"version\":\"1.3.4\"}};\n",
       "  var render_items = [{\"docid\":\"a54ec473-cbdd-4c92-903a-c6febf58d7d0\",\"notebook_comms_target\":\"1049\",\"roots\":{\"1002\":\"07f0cddf-9776-4339-9823-5bb0adc5d26c\"}}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "1002"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "d_0 = [1, 0, 0, 0]\n",
    "dtmc.dist_interact(P, names, d_0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}