notebooks/usage_%dipush_%dipull.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ddip User Guide -- `%dipush` and `%dipull`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Passing objects between the local notebook and the DDP process group\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for connection file: ~/.ipython/profile_default/security/ipcontroller-ippdpp_c-client.json\n"
     ]
    }
   ],
   "source": [
    "%load_ext Ddip\n",
    "%makedip -r -g all --verbose False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process [8418] variable is :{1: 'foo', 2: 'bar'}\n"
     ]
    }
   ],
   "source": [
    "# Everything here is run in local notebook namespace.\n",
    "\n",
    "import os\n",
    "def var_printer(f): print(f\"Process [{os.getpid()}] variable is :{f}\")\n",
    "\n",
    "foobar = { 1 : \"foo\", 2 : \"bar\" }\n",
    "var_printer(foobar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `foobar` and `var_printer` are not visible in the DDP remote processes, unless we `%dipush` them over:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "RemoteError",
     "evalue": "NameError(name 'foobar' is not defined)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)\u001b[0;32m<ipython-input-1-bf03192dbf9f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m",
      "\u001b[1;32m      2\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mvar_printer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Process [{os.getpid()}] variable is :{f}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
      "\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m",
      "\u001b[0;32m----> 4\u001b[0;31m \u001b[0mvar_printer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfoobar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
      "\u001b[0m\u001b[0;31mNameError\u001b[0m: name 'foobar' is not defined"
     ]
    }
   ],
   "source": [
    "%%dip\n",
    "# With %%dip at the top of a cell, everything here is executed in parallel in the DDP process group\n",
    "\n",
    "import os\n",
    "def var_printer(f): print(f\"Process [{os.getpid()}] variable is :{f}\")\n",
    "\n",
    "var_printer(foobar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From local to DDP:  `%dipush`"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "%dipush push_vars [push_vars ...]\n",
    "\n",
    "positional arguments:\n",
    "  push_vars  Push a list of variables from local ipython/notebook namespace to\n",
    "             the DDP group processes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "%dipush foobar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process [8472] variable is :{1: 'foo', 2: 'bar'}\n"
     ]
    }
   ],
   "source": [
    "%%dip\n",
    "var_printer(foobar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let's create something remotely.  They are not visible in the local notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mOut[0:3]: \u001b[0m8472"
      ]
     },
     "metadata": {
      "after": [],
      "completed": "2020-01-31T15:47:31.419048",
      "data": {},
      "engine_id": 0,
      "engine_uuid": "5683b316-2643205dc2d402aa23f176e0",
      "error": null,
      "execute_input": "remote_pid = os.getpid()\nremote_pid\n",
      "execute_result": {
       "data": {
        "text/plain": "8472"
       },
       "execution_count": 3,
       "metadata": {}
      },
      "follow": [],
      "msg_id": "069921d2-f2ec34c8d23d3bd054b7f024",
      "outputs": [],
      "received": "2020-01-31T15:47:31.422011",
      "started": "2020-01-31T15:47:31.409160",
      "status": "ok",
      "stderr": "",
      "stdout": "",
      "submitted": "2020-01-31T15:47:31.404803"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mOut[1:3]: \u001b[0m8476"
      ]
     },
     "metadata": {
      "after": [],
      "completed": "2020-01-31T15:47:31.413251",
      "data": {},
      "engine_id": 1,
      "engine_uuid": "c6a8c8ec-69e0d656d4a8f0251f652d55",
      "error": null,
      "execute_input": "remote_pid = os.getpid()\nremote_pid\n",
      "execute_result": {
       "data": {
        "text/plain": "8476"
       },
       "execution_count": 3,
       "metadata": {}
      },
      "follow": [],
      "msg_id": "47df7bba-05ca6c3e16f805c804b82828",
      "outputs": [],
      "received": "2020-01-31T15:47:31.417074",
      "started": "2020-01-31T15:47:31.408364",
      "status": "ok",
      "stderr": "",
      "stdout": "",
      "submitted": "2020-01-31T15:47:31.405585"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mOut[2:3]: \u001b[0m8478"
      ]
     },
     "metadata": {
      "after": [],
      "completed": "2020-01-31T15:47:31.412442",
      "data": {},
      "engine_id": 2,
      "engine_uuid": "13a9d8af-ef88c8cb9b066c59e8d7f3a0",
      "error": null,
      "execute_input": "remote_pid = os.getpid()\nremote_pid\n",
      "execute_result": {
       "data": {
        "text/plain": "8478"
       },
       "execution_count": 3,
       "metadata": {}
      },
      "follow": [],
      "msg_id": "775446d4-43fcb4d0e3e658fb5bc029c0",
      "outputs": [],
      "received": "2020-01-31T15:47:31.415272",
      "started": "2020-01-31T15:47:31.408400",
      "status": "ok",
      "stderr": "",
      "stdout": "",
      "submitted": "2020-01-31T15:47:31.406104"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%dip --see all\n",
    "# --see all is a flag to show outputs from all DDP processes, not only the rank-0 process.\n",
    "remote_pid = os.getpid()\n",
    "remote_pid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'remote_pid' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-40447ba35d88>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# back in local notebook cell without the %%dip at the top\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mremote_pid\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'remote_pid' is not defined"
     ]
    }
   ],
   "source": [
    "# back in local notebook cell without the %%dip at the top\n",
    "remote_pid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### From remote DDP processes to local: `%dipull`\n",
    "\n",
    "#### `%dipull` by default pulls in the object from RANK 0 process"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "%dipull [-r RANK] pull_vars [pull_vars ...]\n",
    "\n",
    "positional arguments:\n",
    "  pull_vars  Pull a list of variables from the DDP group processes to local\n",
    "             ipython/notebook namespace.\n",
    "\n",
    "optional arguments:\n",
    "  -r RANK    Rank of the process to fetch the variables from. Default to RANK\n",
    "             0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process [8418]: remote_pid is 8472\n"
     ]
    }
   ],
   "source": [
    "%dipull remote_pid\n",
    "print(f\"Process [{os.getpid()}]: remote_pid is {remote_pid}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process [8418]: remote_pid is 8476\n"
     ]
    }
   ],
   "source": [
    "%dipull -r 1 remote_pid\n",
    "print(f\"Process [{os.getpid()}]: remote_pid is {remote_pid}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### It may seem tedious to create and remember what to push from one namespace to another.\n",
    "\n",
    "Alternative, the cell magic `%%dip everywhere` dictates the same code to be run in all namespaces, thus creating/manipulating objects of the same name (but different instances underneath).  **To learn more about `%%dip everywhere`**, try [this notebook](usage_%%dip_%autodip.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Different pid: 16359', 2, ['a', 'list'], {'and': 'a dict'}]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mOut[0:10]: \u001b[0m['Different pid: 16413', 2, ['a', 'list'], {'and': 'a dict'}]"
      ]
     },
     "metadata": {
      "after": [],
      "completed": "2020-01-30T14:07:10.050553",
      "data": {},
      "engine_id": 0,
      "engine_uuid": "7e6e1b1c-ab2e043949bd99f394e75361",
      "error": null,
      "execute_input": "x = [ f\"Different pid: {os.getpid()}\", 2, [\"a\", \"list\"], {\"and\": \"a dict\"}]\nx\n",
      "execute_result": {
       "data": {
        "text/plain": "['Different pid: 16413', 2, ['a', 'list'], {'and': 'a dict'}]"
       },
       "execution_count": 10,
       "metadata": {}
      },
      "follow": [],
      "msg_id": "4f76a80a-a39bbe8d98cb64ba4dbfadcc",
      "outputs": [],
      "received": "2020-01-30T14:07:10.054017",
      "started": "2020-01-30T14:07:10.046435",
      "status": "ok",
      "stderr": "",
      "stdout": "",
      "submitted": "2020-01-30T14:07:10.044061"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mOut[1:10]: \u001b[0m['Different pid: 16417', 2, ['a', 'list'], {'and': 'a dict'}]"
      ]
     },
     "metadata": {
      "after": [],
      "completed": "2020-01-30T14:07:10.050823",
      "data": {},
      "engine_id": 1,
      "engine_uuid": "d9de24e6-9f1e6e481f2b28e15644aef1",
      "error": null,
      "execute_input": "x = [ f\"Different pid: {os.getpid()}\", 2, [\"a\", \"list\"], {\"and\": \"a dict\"}]\nx\n",
      "execute_result": {
       "data": {
        "text/plain": "['Different pid: 16417', 2, ['a', 'list'], {'and': 'a dict'}]"
       },
       "execution_count": 10,
       "metadata": {}
      },
      "follow": [],
      "msg_id": "afff29c5-4886e4547d2d7f145dd83f19",
      "outputs": [],
      "received": "2020-01-30T14:07:10.056088",
      "started": "2020-01-30T14:07:10.047070",
      "status": "ok",
      "stderr": "",
      "stdout": "",
      "submitted": "2020-01-30T14:07:10.044464"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mOut[2:10]: \u001b[0m['Different pid: 16418', 2, ['a', 'list'], {'and': 'a dict'}]"
      ]
     },
     "metadata": {
      "after": [],
      "completed": "2020-01-30T14:07:10.050754",
      "data": {},
      "engine_id": 2,
      "engine_uuid": "340967fd-621f88c681790b1755564eae",
      "error": null,
      "execute_input": "x = [ f\"Different pid: {os.getpid()}\", 2, [\"a\", \"list\"], {\"and\": \"a dict\"}]\nx\n",
      "execute_result": {
       "data": {
        "text/plain": "['Different pid: 16418', 2, ['a', 'list'], {'and': 'a dict'}]"
       },
       "execution_count": 10,
       "metadata": {}
      },
      "follow": [],
      "msg_id": "2f3c616d-4aa8afbd1054cac55922d0f2",
      "outputs": [],
      "received": "2020-01-30T14:07:10.055056",
      "started": "2020-01-30T14:07:10.046646",
      "status": "ok",
      "stderr": "",
      "stdout": "",
      "submitted": "2020-01-30T14:07:10.044632"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%dip everywhere --see all\n",
    "x = [ f\"Different pid: {os.getpid()}\", 2, [\"a\", \"list\"], {\"and\": \"a dict\"}]\n",
    "x"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}