day 69 - rmsprop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from bokeh.plotting import figure, show, output_notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def gradient_descent(F, dF, x, steps=100, lr=0.001):\n",
    "    loss = []\n",
    "    \n",
    "    for _ in range(steps):\n",
    "        dx = dF(x)\n",
    "        x -= lr * dx\n",
    "        loss.append(F(x))\n",
    "\n",
    "    return x, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rmsprop(F, dF, x, steps=100, lr=0.001, decay=.9, eps=1e-8):\n",
    "    loss = []\n",
    "    dx_mean_sqr = np.zeros(x.shape, dtype=float)\n",
    "\n",
    "    for _ in range(steps):\n",
    "        dx = dF(x)\n",
    "        dx_mean_sqr = decay * dx_mean_sqr + (1 - decay) * dx ** 2\n",
    "        x -= lr * dx / (np.sqrt(dx_mean_sqr) + eps)\n",
    "        loss.append(F(x))\n",
    "    \n",
    "    return x, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def rmsprop_momentum(F, dF, x, steps=100, lr=0.001, decay=.9, eps=1e-8, mu=.9):\n",
    "    loss = []\n",
    "    dx_mean_sqr = np.zeros(x.shape, dtype=float)\n",
    "    momentum = np.zeros(x.shape, dtype=float)\n",
    "\n",
    "    for _ in range(steps):\n",
    "        dx = dF(x)\n",
    "        dx_mean_sqr = decay * dx_mean_sqr + (1 - decay) * dx ** 2\n",
    "        momentum = mu * momentum + lr * dx / (np.sqrt(dx_mean_sqr) + eps)\n",
    "        x -= momentum\n",
    "        loss.append(F(x))\n",
    "\n",
    "    return x, loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def F(x):\n",
    "    residual = A @ x - np.eye(len(A), dtype=float)\n",
    "    return np.sum(residual ** 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dF(x):\n",
    "    return 2 * A.T @ (A @ x - np.eye(len(A), dtype=float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "A = np.array([\n",
    "    [2, 5, 1, 4, 6],\n",
    "    [3, 5, 0, 0, 0],\n",
    "    [1, 1, 0, 3, 8],\n",
    "    [6, 6, 2, 2, 1],\n",
    "    [8, 3, 5, 1, 4],\n",
    "], dtype=float)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.79, -0.01,  0.18,  0.19, -0.08],\n",
       "        [-0.01,  0.8 ,  0.  ,  0.2 , -0.07],\n",
       "        [ 0.18,  0.  ,  0.85, -0.15,  0.07],\n",
       "        [ 0.19,  0.2 , -0.15,  0.66,  0.13],\n",
       "        [-0.08, -0.07,  0.07,  0.13,  0.95]]), 0.54691984767143453)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, loss1 = gradient_descent(F, dF, A * 0, steps=300)\n",
    "(A @ X).round(2), loss1[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.84, -0.05,  0.1 ,  0.1 , -0.06],\n",
       "        [-0.04,  0.82,  0.03,  0.19, -0.03],\n",
       "        [ 0.12,  0.03,  0.9 , -0.08,  0.03],\n",
       "        [ 0.15,  0.2 , -0.12,  0.75,  0.06],\n",
       "        [-0.08, -0.09,  0.04,  0.1 ,  0.97]]), 0.32396954419819657)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, loss2 = rmsprop(F, dF, A * 0, steps=300)\n",
    "(A @ X).round(2), loss2[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.99,  0.01,  0.  , -0.01,  0.  ],\n",
       "        [-0.  ,  1.  ,  0.  , -0.  ,  0.  ],\n",
       "        [-0.  ,  0.01,  1.  , -0.01,  0.  ],\n",
       "        [-0.01,  0.01,  0.  ,  0.99,  0.  ],\n",
       "        [-0.01,  0.01,  0.  , -0.01,  1.  ]]), 0.00062303887772378397)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, loss3 = rmsprop_momentum(F, dF, A * 0, steps=300)\n",
    "(A @ X).round(2), loss3[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"http://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"d8b01c18-bc74-42bd-8485-9e68ed5f812a\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(global) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    window._bokeh_onload_callbacks = [];\n",
       "    window._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "\n",
       "  \n",
       "  if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    window._bokeh_timeout = Date.now() + 5000;\n",
       "    window._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    if (window.Bokeh !== undefined) {\n",
       "      var el = document.getElementById(\"d8b01c18-bc74-42bd-8485-9e68ed5f812a\");\n",
       "      el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "    } else if (Date.now() < window._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function run_callbacks() {\n",
       "    window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    delete window._bokeh_onload_callbacks\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    window._bokeh_onload_callbacks.push(callback);\n",
       "    if (window._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    window._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        window._bokeh_is_loading--;\n",
       "        if (window._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"d8b01c18-bc74-42bd-8485-9e68ed5f812a\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid 'd8b01c18-bc74-42bd-8485-9e68ed5f812a' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.5.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.5.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "      document.getElementById(\"d8b01c18-bc74-42bd-8485-9e68ed5f812a\").textContent = \"BokehJS is loading...\";\n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.5.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.5.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.5.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.5.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i](window.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < window._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!window._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      window._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"d8b01c18-bc74-42bd-8485-9e68ed5f812a\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (window._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(this));"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <div class=\"bk-plotdiv\" id=\"845ef23a-3d24-4d6c-a171-8af1f2db5ae0\"></div>\n",
       "    </div>\n",
       "<script type=\"text/javascript\">\n",
       "  \n",
       "  (function(global) {\n",
       "    function now() {\n",
       "      return new Date();\n",
       "    }\n",
       "  \n",
       "    var force = false;\n",
       "  \n",
       "    if (typeof (window._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "      window._bokeh_onload_callbacks = [];\n",
       "      window._bokeh_is_loading = undefined;\n",
       "    }\n",
       "  \n",
       "  \n",
       "    \n",
       "    if (typeof (window._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "      window._bokeh_timeout = Date.now() + 0;\n",
       "      window._bokeh_failed_load = false;\n",
       "    }\n",
       "  \n",
       "    var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "       \"<div style='background-color: #fdd'>\\n\"+\n",
       "       \"<p>\\n\"+\n",
       "       \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "       \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "       \"</p>\\n\"+\n",
       "       \"<ul>\\n\"+\n",
       "       \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "       \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "       \"</ul>\\n\"+\n",
       "       \"<code>\\n\"+\n",
       "       \"from bokeh.resources import INLINE\\n\"+\n",
       "       \"output_notebook(resources=INLINE)\\n\"+\n",
       "       \"</code>\\n\"+\n",
       "       \"</div>\"}};\n",
       "  \n",
       "    function display_loaded() {\n",
       "      if (window.Bokeh !== undefined) {\n",
       "        var el = document.getElementById(\"845ef23a-3d24-4d6c-a171-8af1f2db5ae0\");\n",
       "        el.textContent = \"BokehJS \" + Bokeh.version + \" successfully loaded.\";\n",
       "      } else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(display_loaded, 100)\n",
       "      }\n",
       "    }\n",
       "  \n",
       "    function run_callbacks() {\n",
       "      window._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "      delete window._bokeh_onload_callbacks\n",
       "      console.info(\"Bokeh: all callbacks have finished\");\n",
       "    }\n",
       "  \n",
       "    function load_libs(js_urls, callback) {\n",
       "      window._bokeh_onload_callbacks.push(callback);\n",
       "      if (window._bokeh_is_loading > 0) {\n",
       "        console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "        return null;\n",
       "      }\n",
       "      if (js_urls == null || js_urls.length === 0) {\n",
       "        run_callbacks();\n",
       "        return null;\n",
       "      }\n",
       "      console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "      window._bokeh_is_loading = js_urls.length;\n",
       "      for (var i = 0; i < js_urls.length; i++) {\n",
       "        var url = js_urls[i];\n",
       "        var s = document.createElement('script');\n",
       "        s.src = url;\n",
       "        s.async = false;\n",
       "        s.onreadystatechange = s.onload = function() {\n",
       "          window._bokeh_is_loading--;\n",
       "          if (window._bokeh_is_loading === 0) {\n",
       "            console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "            run_callbacks()\n",
       "          }\n",
       "        };\n",
       "        s.onerror = function() {\n",
       "          console.warn(\"failed to load library \" + url);\n",
       "        };\n",
       "        console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "        document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "      }\n",
       "    };var element = document.getElementById(\"845ef23a-3d24-4d6c-a171-8af1f2db5ae0\");\n",
       "    if (element == null) {\n",
       "      console.log(\"Bokeh: ERROR: autoload.js configured with elementid '845ef23a-3d24-4d6c-a171-8af1f2db5ae0' but no matching script tag was found. \")\n",
       "      return false;\n",
       "    }\n",
       "  \n",
       "    var js_urls = [];\n",
       "  \n",
       "    var inline_js = [\n",
       "      function(Bokeh) {\n",
       "        (function() {\n",
       "          var fn = function() {\n",
       "            var docs_json = {\"ea5831f7-2cea-46a4-a8a9-b4a1eb57f3e9\":{\"roots\":{\"references\":[{\"attributes\":{\"data_source\":{\"id\":\"8713bfa4-212b-4f08-8981-9387d6323826\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"f70bb995-7def-4f9b-af16-20e8bed7092e\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"2b8c2914-ddea-4f70-bb1f-43be4d792a3d\",\"type\":\"Line\"},\"selection_glyph\":null},\"id\":\"34a3d93d-3b46-42ea-8e1b-4a3ba4cc4674\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"label\":{\"value\":\"gd\"},\"renderers\":[{\"id\":\"b0178f1d-e21a-4874-b87d-388c9d245602\",\"type\":\"GlyphRenderer\"}]},\"id\":\"67b88d11-548f-4717-8a94-4ffead5fb532\",\"type\":\"LegendItem\"},{\"attributes\":{\"line_color\":{\"value\":\"steelblue\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"5635e601-36a9-4dc7-9812-5420912b5a0b\",\"type\":\"Line\"},{\"attributes\":{\"below\":[{\"id\":\"8e2a3323-2142-491a-9ac6-640962d4db6e\",\"type\":\"LinearAxis\"}],\"left\":[{\"id\":\"ca22340b-f87c-4f94-bbb7-d54a976f5016\",\"type\":\"LinearAxis\"}],\"renderers\":[{\"id\":\"8e2a3323-2142-491a-9ac6-640962d4db6e\",\"type\":\"LinearAxis\"},{\"id\":\"e3631b5a-f06d-44d3-b1b3-ebda516d1ec2\",\"type\":\"Grid\"},{\"id\":\"ca22340b-f87c-4f94-bbb7-d54a976f5016\",\"type\":\"LinearAxis\"},{\"id\":\"9bd75ad7-31e9-473a-a75e-4a38055af5ca\",\"type\":\"Grid\"},{\"id\":\"836983e3-aa62-49b5-8cef-9d5528035ecb\",\"type\":\"BoxAnnotation\"},{\"id\":\"9330c8e4-a316-47a4-8118-4ae1c6bba80e\",\"type\":\"Legend\"},{\"id\":\"b0178f1d-e21a-4874-b87d-388c9d245602\",\"type\":\"GlyphRenderer\"},{\"id\":\"3b8662a0-3dc5-4701-bbf6-8c2314c6b54e\",\"type\":\"GlyphRenderer\"},{\"id\":\"34a3d93d-3b46-42ea-8e1b-4a3ba4cc4674\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"c7898c91-8397-455c-92eb-6b115984fb73\",\"type\":\"Title\"},\"tool_events\":{\"id\":\"818798dc-836f-4a97-ae3e-2dcc184ad680\",\"type\":\"ToolEvents\"},\"toolbar\":{\"id\":\"e7a324ab-3e93-4d41-a6a8-ed55930d15c6\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"0687db4b-b14e-472d-a74c-e0c5961f3336\",\"type\":\"DataRange1d\"},\"y_range\":{\"id\":\"3c525c59-f688-4111-bec4-9aa98fdb4713\",\"type\":\"DataRange1d\"}},\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"771e6cf1-0c23-4cd8-9621-93df07563b27\",\"type\":\"SaveTool\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"c7898c91-8397-455c-92eb-6b115984fb73\",\"type\":\"Title\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"ab1314a3-a0f7-42b0-a675-34754be6f5c6\",\"type\":\"Line\"},{\"attributes\":{\"data_source\":{\"id\":\"36a204bb-64e2-4e94-92ed-a9241d3e1fde\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"537a63fb-2a68-41d4-bc5a-85a59c3a605b\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"c10aa718-aad2-4e0d-86b9-e79865141b67\",\"type\":\"Line\"},\"selection_glyph\":null},\"id\":\"3b8662a0-3dc5-4701-bbf6-8c2314c6b54e\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"label\":{\"value\":\"rmsprop\"},\"renderers\":[{\"id\":\"3b8662a0-3dc5-4701-bbf6-8c2314c6b54e\",\"type\":\"GlyphRenderer\"}]},\"id\":\"1da51afc-06cf-4dbe-974e-1127f6db7c69\",\"type\":\"LegendItem\"},{\"attributes\":{\"formatter\":{\"id\":\"96522778-bfaa-4c65-b89e-b2221939c6b4\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"2f694159-4422-488b-b341-c0001a3fd564\",\"type\":\"BasicTicker\"}},\"id\":\"8e2a3323-2142-491a-9ac6-640962d4db6e\",\"type\":\"LinearAxis\"},{\"attributes\":{\"overlay\":{\"id\":\"836983e3-aa62-49b5-8cef-9d5528035ecb\",\"type\":\"BoxAnnotation\"},\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"b03cd9e6-d94b-46ee-8e77-92ad2e14d377\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"33e4abb9-9039-4f30-b1a4-baaef9cc4dea\",\"type\":\"HelpTool\"},{\"attributes\":{},\"id\":\"bbc0027b-91c8-4192-a54d-35794499031e\",\"type\":\"BasicTicker\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"836983e3-aa62-49b5-8cef-9d5528035ecb\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"formatter\":{\"id\":\"abe2b5de-955a-4a63-846e-0f98b593dfb4\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"bbc0027b-91c8-4192-a54d-35794499031e\",\"type\":\"BasicTicker\"}},\"id\":\"ca22340b-f87c-4f94-bbb7-d54a976f5016\",\"type\":\"LinearAxis\"},{\"attributes\":{\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"82082733-4016-4247-92d2-f5e276530cd6\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"c3b17656-f5b3-4e9d-98ed-bca985f41548\",\"type\":\"PanTool\"},{\"attributes\":{\"line_color\":{\"value\":\"red\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"f70bb995-7def-4f9b-af16-20e8bed7092e\",\"type\":\"Line\"},{\"attributes\":{\"data_source\":{\"id\":\"39449c29-761d-45ed-b5e3-15794cce98f3\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"5635e601-36a9-4dc7-9812-5420912b5a0b\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"ab1314a3-a0f7-42b0-a675-34754be6f5c6\",\"type\":\"Line\"},\"selection_glyph\":null},\"id\":\"b0178f1d-e21a-4874-b87d-388c9d245602\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"line_color\":{\"value\":\"green\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"537a63fb-2a68-41d4-bc5a-85a59c3a605b\",\"type\":\"Line\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"2b8c2914-ddea-4f70-bb1f-43be4d792a3d\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"abe2b5de-955a-4a63-846e-0f98b593dfb4\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"x\",\"y\"],\"data\":{\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299],\"y\":[4.565469241724881,4.323801879880114,4.148470106090875,4.0068268940452185,3.8855671122141455,3.7780434872702715,3.6805677227501548,3.5909250966159196,3.507652552362475,3.4297146088849133,3.356334518182458,3.286911849135425,3.2209697863913442,3.158124715041163,3.098062466838738,3.040523437099828,2.9852892465876333,2.9321749886997366,2.8810208107281183,2.8316888002427216,2.7840564337991234,2.7380182591125424,2.693478113138192,2.650367212146636,2.6086320257149502,2.568233517618328,2.528847053897311,2.4906271874322155,2.4535184777812034,2.41750003030704,2.3824915835395486,2.348463155892808,2.3153649363790154,2.2831692562678123,2.2518351309694977,2.221337175606867,2.1916378058019106,2.1627163846343236,2.1345363176013854,2.1070882422804256,2.080323043696816,2.0542394220769014,2.0287480693975706,2.0038861435616346,1.9795986098351293,1.9559084302780658,1.9327701607128283,1.910185212993782,1.8881211674650873,1.8665763196550582,1.8455249533356646,1.8249666007550682,1.8048769090685628,1.7852594712401206,1.766085103320428,1.747363859807558,1.7290533495114722,1.7111724013777432,1.6936738059283292,1.6765826422984818,1.6598584688207783,1.6435181845364675,1.627527196994159,1.6118946662674565,1.5965901340422117,1.5816205414247264,1.566956620930255,1.5526070717865557,1.5385402555518362,1.5247695381561894,1.511257436229724,1.4980236682333832,1.485025497566413,1.4722885182231793,1.4597701074468277,1.4474976060062117,1.4354317188374368,1.4235977218283447,1.411959521467919,1.400540120975661,1.3893051945831842,1.3782768932227845,1.3674209290217951,1.3567598775973346,1.3462584284882564,1.3359421886884844,1.3258072798921134,1.3164073327910426,1.3066980174726033,1.2968172782249405,1.2870433130234673,1.2775941007087308,1.2682953270545607,1.259156038720152,1.2501159572492562,1.2412038955594553,1.2323772250698497,1.2236665244222484,1.215031379261902,1.2065052816270478,1.198047631244951,1.18970051171214,1.1814248624966646,1.1732895233518057,1.165228035857324,1.1573045801155843,1.1494676725454416,1.1426302845616596,1.1345969180857889,1.126303197581412,1.1183061620483676,1.110612302201569,1.1030243744227959,1.0955572214601355,1.0881320722367165,1.0808044939978705,1.0735102690664042,1.0663192547472569,1.059160469543639,1.0521221330138677,1.0451069567130735,1.0382632390090447,1.0318518173562357,1.0262635888103255,1.019134436548148,1.012095440735005,1.0052525193100739,0.9986560459198536,0.9921276488981197,0.985787563170377,0.9804332751594235,0.9741220971004396,0.9673465492753387,0.9610121034818088,0.954815253853237,0.9487563180669281,0.9426828419438603,0.936719434095843,0.9307229734816318,0.9248224685893709,0.9188760826907302,0.9130326130849155,0.9071859436180216,0.9015280471202823,0.8958304593255146,0.890237271689793,0.8845163752824983,0.8789710184403117,0.8734918401079879,0.8681620947303623,0.8626908644433323,0.857217280608497,0.851728389304382,0.8463736558628395,0.8410132668501036,0.8357572305681654,0.8304725414512809,0.8252847785047366,0.8200738250249088,0.8149772807921467,0.8098685288582922,0.8048941145899589,0.7998977005948685,0.7950281210189478,0.7901040243175589,0.785264709558943,0.7803504088376024,0.7755166322889927,0.7706453000154668,0.7658755216485165,0.761083205155992,0.7563882678366288,0.7516691233738058,0.747044195485501,0.7423971351455141,0.7378487485315179,0.7332809952301986,0.7288156895825292,0.7243235208334575,0.7199245910156447,0.7154801160185156,0.7111157834661662,0.7067021539525288,0.7023711012894613,0.6980029194258358,0.693721708026735,0.6894116080897591,0.6851871025009708,0.6809369680038321,0.6767701253315657,0.672579288667111,0.6684698209583225,0.6643354844425189,0.6602788804041325,0.6561931425837441,0.6521806708241085,0.6481351376647495,0.6441615675683865,0.6401546212813138,0.6362208127533647,0.6322555280051805,0.6283638707488531,0.6244431984919441,0.6205952050723637,0.6167203365353496,0.6129161178086354,0.6090860089869887,0.6053237401374574,0.6015351182608291,0.5978115894060582,0.5940606494262423,0.5903735180099243,0.586658347012274,0.5830073139651432,0.5793279857299316,0.57571342202045,0.5720704336722536,0.5684921945030073,0.5648857353513675,0.5613432528195483,0.5577730350614892,0.5542653976296932,0.550730379016666,0.5472562917743012,0.543754892047193,0.5403131882000047,0.536844146704632,0.5334344132112541,0.5299972998449181,0.5266197189431626,0.5232145765705141,0.519869236570369,0.5164960151030924,0.513182534654085,0.5098408862708605,0.5065585087869107,0.5032478319232205,0.49999567209267726,0.496715229588211,0.4934925271773061,0.4902416580907824,0.4870480011735774,0.48382633020389687,0.48066169027927125,0.4774691273040806,0.47433363872127177,0.4711701739205071,0.4680638415625548,0.464929339031817,0.4618518868681946,0.4587460138416181,0.4556969281577594,0.452619220118336,0.44959792261669246,0.44654792273863037,0.4435539657563575,0.44053135568757085,0.4375645301104202,0.4345691738457871,0.4316294752512966,0.4286613556181375,0.42574884787170275,0.42280794893941226,0.4199226187707526,0.41700883031160285,0.4141505100867477,0.4112636021691919,0.408431989745545,0.40557166299486336,0.4027664173958238,0.39993239392548985,0.39715324595796025,0.39434534309586405,0.39159215493140825,0.38881029879080403,0.38608304531588394,0.38332722453013557,0.38062592019977537,0.3778961154225431,0.3752207353463923,0.3725168662932793,0.3698673032114682,0.36718921880896466,0.36456529299506885,0.3619128049078409,0.35931431490722504,0.3566872509843722,0.3541140317814676,0.351512276132076,0.3489642305111978,0.34638772824936,0.3438648174118426,0.3413135461598354,0.33881575147916954,0.33628968073509624,0.33381696253887677,0.33131602527175835,0.32886830143274526,0.32639239160323363,0.32396954419819657]}},\"id\":\"36a204bb-64e2-4e94-92ed-a9241d3e1fde\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"bbc0027b-91c8-4192-a54d-35794499031e\",\"type\":\"BasicTicker\"}},\"id\":\"9bd75ad7-31e9-473a-a75e-4a38055af5ca\",\"type\":\"Grid\"},{\"attributes\":{\"items\":[{\"id\":\"67b88d11-548f-4717-8a94-4ffead5fb532\",\"type\":\"LegendItem\"},{\"id\":\"1da51afc-06cf-4dbe-974e-1127f6db7c69\",\"type\":\"LegendItem\"},{\"id\":\"daba0ffa-12cb-4cd0-b5fe-7eaae1c008fa\",\"type\":\"LegendItem\"}],\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"9330c8e4-a316-47a4-8118-4ae1c6bba80e\",\"type\":\"Legend\"},{\"attributes\":{},\"id\":\"2f694159-4422-488b-b341-c0001a3fd564\",\"type\":\"BasicTicker\"},{\"attributes\":{\"label\":{\"value\":\"rmsprop+momentum\"},\"renderers\":[{\"id\":\"34a3d93d-3b46-42ea-8e1b-4a3ba4cc4674\",\"type\":\"GlyphRenderer\"}]},\"id\":\"daba0ffa-12cb-4cd0-b5fe-7eaae1c008fa\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"96522778-bfaa-4c65-b89e-b2221939c6b4\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"216064c0-a1b9-4475-b272-d44089f79428\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"818798dc-836f-4a97-ae3e-2dcc184ad680\",\"type\":\"ToolEvents\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"c3b17656-f5b3-4e9d-98ed-bca985f41548\",\"type\":\"PanTool\"},{\"id\":\"82082733-4016-4247-92d2-f5e276530cd6\",\"type\":\"WheelZoomTool\"},{\"id\":\"b03cd9e6-d94b-46ee-8e77-92ad2e14d377\",\"type\":\"BoxZoomTool\"},{\"id\":\"771e6cf1-0c23-4cd8-9621-93df07563b27\",\"type\":\"SaveTool\"},{\"id\":\"216064c0-a1b9-4475-b272-d44089f79428\",\"type\":\"ResetTool\"},{\"id\":\"33e4abb9-9039-4f30-b1a4-baaef9cc4dea\",\"type\":\"HelpTool\"}]},\"id\":\"e7a324ab-3e93-4d41-a6a8-ed55930d15c6\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null},\"id\":\"0687db4b-b14e-472d-a74c-e0c5961f3336\",\"type\":\"DataRange1d\"},{\"attributes\":{\"plot\":{\"id\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"2f694159-4422-488b-b341-c0001a3fd564\",\"type\":\"BasicTicker\"}},\"id\":\"e3631b5a-f06d-44d3-b1b3-ebda516d1ec2\",\"type\":\"Grid\"},{\"attributes\":{\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"x\":{\"field\":\"x\"},\"y\":{\"field\":\"y\"}},\"id\":\"c10aa718-aad2-4e0d-86b9-e79865141b67\",\"type\":\"Line\"},{\"attributes\":{\"callback\":null},\"id\":\"3c525c59-f688-4111-bec4-9aa98fdb4713\",\"type\":\"DataRange1d\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"x\",\"y\"],\"data\":{\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299],\"y\":[3.7956839999999996,3.3351323370399997,3.0643780878696907,2.8663133160863796,2.709871095515805,2.5826105191316024,2.4773070682731353,2.3889584519645815,2.313888970826335,2.2493408708869658,2.193222631075687,2.1439339380207363,2.1002393319532247,2.0611759221091708,2.0259856576660495,1.9940654502036648,1.9649303154016904,1.9381860283229049,1.9135087435782494,1.8906297253114754,1.869323835481977,1.8494007947084814,1.830698495786946,1.8130778433214139,1.7964187335832964,1.7806168911613118,1.765581353642041,1.7512324500605299,1.737500158690286,1.7243227589011683,1.7116457132050729,1.6994207313416192,1.6876049798607315,1.6761604092496425,1.6650531770377992,1.6542531500836166,1.6437334728309676,1.6334701910334037,1.6234419225085712,1.6136295680709003,1.604016057019455,1.5945861225194244,1.5853261029759214,1.5762237661061558,1.5672681529065884,1.5584494391120662,1.5497588120741723,1.5411883612610682,1.5327309808122989,1.524380282778009,1.51613051983941,1.507976516451257,1.49991360747161,1.491937583452362,1.4840446418585302,1.4762313435671348,1.4684945740692898,1.4608315088633097,1.4532395825833397,1.4457164614581801,1.4382600187394319,1.430868312777562,1.423539567459519,1.4162721547526929,1.409064579127732,1.4019154636573845,1.394823537610517,1.3877876253800256,1.3808066366007865,1.373879557329344,1.3670054421708842,1.3601834072514054,1.3534126239439923,1.3466923132679574,1.3400217408883457,1.3334002126511295,1.326827070596404,1.3203016893980843,1.313823473184185,1.3073918526966923,1.3010062827544704,1.2946662399865674,1.2883712208068192,1.2821207396037775,1.2759143271227817,1.2697515290195058,1.2636319045665216,1.2575550254964227,1.251520474966822,1.2455278466341069,1.2395767438242717,1.2336667787903832,1.22779757204737,1.2219687517758344,1.2161799532874678,1.21043081854546,1.2047209957339962,1.1990501388715804,1.1934179074634856,1.1878239661891328,1.1822679846206667,1.1767496369693822,1.1712686018570229,1.1658245621093029,1.1604172045692664,1.1550462199283804,1.1497113025734622,1.1444121504477653,1.1391484649247128,1.1339199506929396,1.1287263156514444,1.1235672708137827,1.1184425302203498,1.1133518108578948,1.1082948325855186,1.103271318066463,1.0982809927050965,1.0933235845885543,1.088398824432546,1.083506445530908,1.0786461837085068,1.0738177772771624,1.069020966994273,1.0642554960238773,1.0595211098999058,1.0548175564914042,1.0501445859695342,1.0455019507761765,1.0408894055939857,1.036306707317749,1.0317536150269404,1.0272298899593375,1.0227352954856253,1.018269597084881,1.0138325623208735,1.009423960819094,1.005043564244467,1.0006911462796784,0.9963664826040691,0.9920693508730544,0.9877995306980222,0.9835568036266792,0.9793409531238093,0.9751517645524157,0.9709890251552203,0.9668525240364982,0.9627420521442244,0.9586574022525159,0.9545983689443495,0.9505647485945423,0.9465563393529791,0.9425729411280767,0.9386143555704712,0.9346803860569196,0.9307708376744075,0.9268855172044502,0.9230242331075869,0.9191867955080546,0.9153730161786364,0.9115827085256845,0.9078156875743034,0.9040717699536961,0.9003507738826656,0.8966525191552679,0.892976827126613,0.8893235206988114,0.8856924243070619,0.8820833639058777,0.8784961669554482,0.8749306624081359,0.8713866806951015,0.8678640537130614,0.8643626148111683,0.860882198778019,0.8574226418287837,0.8539837815924582,0.8505654570992325,0.8471675087679806,0.8437897783938644,0.8404321091360527,0.8370943455055545,0.8337763333531637,0.830477919857514,0.8271989535132449,0.8239392841192746,0.8206987627671796,0.8174772418296827,0.8142745749492426,0.8110906170267491,0.8079252242103204,0.8047782538842033,0.8016495646577704,0.7985390163546221,0.7954464700017817,0.7923717878189925,0.7893148332081104,0.7862754707425889,0.7832535661570642,0.7802489863370313,0.7772615993086137,0.7742912742284239,0.7713378813735181,0.7684012921314382,0.7654813789903466,0.7625780155292469,0.7596910764082941,0.756820437359191,0.7539659751756728,0.7511275677040752,0.7483050938339877,0.7454984334889917,0.742707467617482,0.7399320781835685,0.7371721481580626,0.734427561509543,0.7316982031955012,0.7289839591535677,0.7262847162928174,0.7236003624851507,0.7209307865567539,0.718275878279637,0.7156355283632443,0.7130096284461438,0.7103980710877893,0.7078007497603566,0.7052175588406526,0.702648393602099,0.7000931502067851,0.6975517256975948,0.6950240179904005,0.6925099258663315,0.6900093489641086,0.6875221877724473,0.6850483436225335,0.6825877186805589,0.6801402159403316,0.677705739215948,0.6752841931345309,0.6728754831290348,0.6704795154311137,0.668096197064054,0.6657254358357698,0.6633671403318627,0.6610212199087407,0.6586875846868037,0.6563661455436848,0.654056814107555,0.6517595027504894,0.6494741245818887,0.647200593441963,0.6449388238952735,0.6426887312243307,0.6404502314232511,0.6382232411914713,0.6360076779275166,0.6338034597228279,0.6316105053556408,0.6294287342849231,0.6272580666443635,0.6250984232364163,0.6229497255263988,0.6208118956366406,0.6186848563406873,0.6165685310575546,0.6144628438460339,0.6123677193990505,0.6102830830380694,0.6082088607075555,0.6061449789694778,0.6040913649978698,0.6020479465734306,0.6000146520781824,0.5979914104901694,0.5959781513782082,0.593974804896683,0.5919813017803889,0.5899975733394199,0.5880235514541045,0.5860591685699851,0.5841043576928421,0.5821590523837652,0.5802231867542655,0.5782966954614338,0.5763795137031397,0.5744715772132774,0.5725728222570501,0.5706831856263002,0.5688026046348773,0.5669310171140528,0.5650683614079699,0.5632145763691381,0.5613696013539675,0.5595333762183426,0.5577058413132348,0.5558869374803551,0.5540766060478468,0.5522747888260137,0.5504814281030905,0.5486964666410477,0.5469198476714345]}},\"id\":\"39449c29-761d-45ed-b5e3-15794cce98f3\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"x\",\"y\"],\"data\":{\"x\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299],\"y\":[4.565469241724881,4.065146843674193,3.6605839598954377,3.360858945723255,3.1201276889967575,2.8947645951798453,2.6620481676746177,2.422033912677105,2.1958499594332834,2.0134539025741214,1.8932364523413447,1.8288053311177916,1.7928477097980038,1.7546749043760097,1.6971267024286143,1.621242883700381,1.5391644326331073,1.4632984562539546,1.399163760778968,1.3447274480335414,1.2946905942464915,1.2458141639134017,1.1993269912247504,1.1589634809138398,1.126696166432354,1.100089976625315,1.0737550094329786,1.0434260141725593,1.0088105397947997,0.9727358627319959,0.9378531640630988,0.9046741564004784,0.8727159271746445,0.8424125184147276,0.8149282173239707,0.7904481252822826,0.7676744361820874,0.7451173772674231,0.722270992276013,0.6994766297814247,0.6771368673172335,0.6553689086712531,0.634218427536359,0.6137804403360896,0.5938907500466518,0.5741190434511433,0.5542281131764274,0.5342865118173057,0.5145182439606839,0.49534166440074806,0.47699508922626216,0.4592645665108345,0.44192984458479045,0.4248279552120467,0.4077264468070027,0.3906836743396402,0.3740532610229837,0.3578958866340634,0.34201427123301587,0.3263840849017585,0.31089153881607273,0.2955869614418341,0.28074492887024793,0.26633437463342313,0.2521154424435941,0.23798147093458377,0.22412295131988252,0.2106713654119869,0.1975326264721127,0.18469581026984175,0.17225032878784866,0.16024659709683775,0.14864169620495174,0.13739356906820138,0.1265359615935662,0.11616819362214169,0.10635125282252961,0.09707358165622801,0.08840026235303262,0.08030510003617154,0.07230809511259417,0.06452723402170506,0.05773619091942324,0.05169509277622113,0.04559892440318405,0.03982074077022412,0.03490771341219817,0.030424767374333955,0.026163571341435387,0.02241511416353747,0.019254136955185765,0.016022728410888185,0.013042482390140515,0.010949782880876932,0.009237510178764743,0.007548431073190039,0.005665493325830326,0.005128146674780678,0.004079439895442336,0.0025180214740987694,0.002601428324479665,0.0018992059182509047,0.0011247392937110267,0.001520984453305028,0.0010305064855337263,0.0008988054596974042,0.0010683474634957087,0.0006833242687774212,0.000903901465775922,0.001226233488417055,0.0008615816729085915,0.0008014879348200774,0.0011856170308851715,0.001185286129836536,0.0010924776637997902,0.001147018607299385,0.0011900588788315808,0.0011468152839186495,0.00107203230165152,0.001180950009161944,0.00119294087928901,0.0011696242172181215,0.0011435916374599937,0.0010118902304042487,0.0009710320477265593,0.0008635572172928983,0.0009522685458335002,0.001128167277692783,0.0009820071403222814,0.0007307921181271805,0.0005697905695776661,0.0007690288878856677,0.0009295071325641044,0.0007389014293178342,0.0005152056465537374,0.00046737734750491633,0.0007201841091033676,0.0007442609442350721,0.0005560508557849649,0.0004243256990137782,0.00047892709561762626,0.0007063699506439827,0.0006286621869337159,0.0004886377971260747,0.00040516751059296256,0.0005223433518932218,0.0007102056332045545,0.0005880231063242991,0.00047026953888781116,0.0004010915899586505,0.0005727663554657142,0.0007205459015790323,0.0005731613509514312,0.00045401389332459455,0.00040637304972943636,0.0006254613591195482,0.0007144291222850921,0.000551388265485834,0.00043086673861371155,0.0004268352121463925,0.0006721697730716534,0.0006851818250035437,0.0005212609375076845,0.0004082802407798422,0.00046293140167462095,0.0007036451147994853,0.00064327954484141,0.0004920661486150981,0.0003928887649205042,0.0005100731634797224,0.0007174352882314891,0.0006015993050187335,0.0004665942650782362,0.000388106858415699,0.000562893717840917,0.000715080098704537,0.000565945215393317,0.00044282322598464166,0.0003963400535802581,0.0006160982503485905,0.0006987727614829999,0.0005359758109895936,0.0004201426280542121,0.00041974542298554575,0.0006632930549443641,0.0006710680747809664,0.0005099942091617937,0.00040083291959348486,0.0004580160323325527,0.0006978505784478352,0.0006360720628287505,0.0004865304375091157,0.0003884214659029207,0.0005075753985054355,0.0007151729672287286,0.0005992565780319331,0.00046413101541121354,0.0003859941959371146,0.0005627699660270472,0.0007144693169009922,0.0005650275657019744,0.0004416187027491329,0.00039615651004458685,0.0006172116138466054,0.0006980545067442213,0.000535114768453152,0.00041945943951948955,0.0004206933156707147,0.000664433351351135,0.0006698142758020312,0.0005090861414611547,0.00040038660873028414,0.00045934773606002037,0.000698432473538446,0.0006346796087179623,0.00048567304750963023,0.00038809586603732457,0.0005089837895073812,0.0007152061044460164,0.000598022436691527,0.00046331299715392487,0.0003859097356914193,0.0005641886781638367,0.0007140820635240445,0.0005640051536170344,0.0004408183316986143,0.0003964924920927003,0.0006185750184496549,0.0006973155351887615,0.0005342483082924175,0.0004187576476922385,0.0004215337212489511,0.0006655641304205227,0.000668824411764959,0.0005083374275955324,0.0003998846864540187,0.00046062943400951436,0.0006991458874034819,0.0006335853564046329,0.0004850038085354014,0.00038787444654057797,0.000510532094978121,0.0007154110715416497,0.0005969750626812167,0.0004626642035551412,0.00038602830439164824,0.0005658064449323906,0.0007137984791317602,0.0005630812871509056,0.0004401647977562628,0.0003970074655560406,0.0006200679257715323,0.0006966349313900297,0.0005334490258129367,0.000418146340370496,0.0004224722921705038,0.0006667467925847103,0.0006678722389900122,0.000507631123707063,0.00039941951293254657,0.00046194024558403633,0.0006998682612001782,0.0006325178430343515,0.00048434828826628487,0.0003876615700934818,0.0005120838987533631,0.00071560752302904,0.000595939697494508,0.00046201607892215884,0.0003861535111092974,0.0005674218742914448,0.0007135027923539773,0.0005621611974677625,0.00043951105509403037,0.0003975365800162865,0.0006215564309737992,0.0006959431414782113,0.0005326529025070293,0.00041754048887057736,0.00042342693394586126,0.0006679212113282334,0.0006669153558348563,0.0005069287912166882,0.00039896382179808543,0.0004632640709003647,0.0007005786928340813,0.0006314518867372714,0.00048369628852144104,0.0003874596034542217,0.0005136422765020016,0.0007157912893691279,0.0005949091590930954,0.0004613692363208717,0.0003862907295975906,0.0005690373610770121,0.0007131952749244931,0.0005612460030412062,0.0004388581862023839,0.00039807834081178814,0.000623038877723784]}},\"id\":\"8713bfa4-212b-4f08-8981-9387d6323826\",\"type\":\"ColumnDataSource\"}],\"root_ids\":[\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.5\"}};\n",
       "            var render_items = [{\"docid\":\"ea5831f7-2cea-46a4-a8a9-b4a1eb57f3e9\",\"elementid\":\"845ef23a-3d24-4d6c-a171-8af1f2db5ae0\",\"modelid\":\"ad8f06e8-e58e-462d-bf8b-54a431c8d480\"}];\n",
       "            \n",
       "            Bokeh.embed.embed_items(docs_json, render_items);\n",
       "          };\n",
       "          if (document.readyState != \"loading\") fn();\n",
       "          else document.addEventListener(\"DOMContentLoaded\", fn);\n",
       "        })();\n",
       "      },\n",
       "      function(Bokeh) {\n",
       "      }\n",
       "    ];\n",
       "  \n",
       "    function run_inline_js() {\n",
       "      \n",
       "      if ((window.Bokeh !== undefined) || (force === true)) {\n",
       "        for (var i = 0; i < inline_js.length; i++) {\n",
       "          inline_js[i](window.Bokeh);\n",
       "        }if (force === true) {\n",
       "          display_loaded();\n",
       "        }} else if (Date.now() < window._bokeh_timeout) {\n",
       "        setTimeout(run_inline_js, 100);\n",
       "      } else if (!window._bokeh_failed_load) {\n",
       "        console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "        window._bokeh_failed_load = true;\n",
       "      } else if (force !== true) {\n",
       "        var cell = $(document.getElementById(\"845ef23a-3d24-4d6c-a171-8af1f2db5ae0\")).parents('.cell').data().cell;\n",
       "        cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "      }\n",
       "  \n",
       "    }\n",
       "  \n",
       "    if (window._bokeh_is_loading === 0) {\n",
       "      console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "      run_inline_js();\n",
       "    } else {\n",
       "      load_libs(js_urls, function() {\n",
       "        console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "        run_inline_js();\n",
       "      });\n",
       "    }\n",
       "  }(this));\n",
       "</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "output_notebook()\n",
    "\n",
    "plot = figure()\n",
    "plot.line(x=range(len(loss1)), y=loss1, color='steelblue', legend='gd')\n",
    "plot.line(x=range(len(loss2)), y=loss2, color='green', legend='rmsprop')\n",
    "plot.line(x=range(len(loss3)), y=loss3, color='red', legend='rmsprop+momentum')\n",
    "\n",
    "show(plot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}