NotebookConstruction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nbformat as nbf\n",
    "import glob\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_logs = glob.glob(\"../logs/*.json\")\n",
    "\n",
    "# formats all logs properly \n",
    "# key = session ID\n",
    "# value = json w/ same format as what was originally held in nb metadata\n",
    "formatted_logs = {}\n",
    "\n",
    "for log in all_logs:\n",
    "    f = open(log, 'r')\n",
    "    \n",
    "    # merges body sent from various HTTP Posts\n",
    "    entries = f.read().split('}{')\n",
    "    for i in range(len(entries)):\n",
    "        entries[i] = '{' + entries[i] + '}'\n",
    "    entries[0] = entries[0][1:]\n",
    "    entries[-1] = entries[-1][:-1]\n",
    "    formatted_log = json.loads(entries[0])\n",
    "    for i in range(1,len(entries)):\n",
    "        j = json.loads(entries[i])\n",
    "        history = j['history']\n",
    "        formatted_log['history'].extend(history)\n",
    "    \n",
    "    # get time user started ipynb\n",
    "    formatted_log['startTime'] = formatted_log['history'][0]['time']\n",
    "    formatted_log['endTime'] = formatted_log['history'][len(formatted_log['history'])-1]['time']\n",
    "    \n",
    "    # separates user hash from file name (sessionID)\n",
    "    names = log.split('_')\n",
    "    formatted_log['userHash'] = names[0][5:]\n",
    "    \n",
    "    # excluding Jerry and Doris and potentially others\n",
    "    exclude_list = ['bca87887a1cc89312f7d073fd007ea68', '1a735d0ee6a6f9d7fdab573b50851da7']\n",
    "    if names[0][5:] not in exclude_list:\n",
    "        formatted_logs[names[1][:-5]] = formatted_log\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nbName</th>\n",
       "      <th>history</th>\n",
       "      <th>startTime</th>\n",
       "      <th>endTime</th>\n",
       "      <th>userHash</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6b6nhxtxh</th>\n",
       "      <td>cars_demo.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-12-...</td>\n",
       "      <td>2020-12-27T01:20:56.927Z</td>\n",
       "      <td>2020-12-27T01:47:49.232Z</td>\n",
       "      <td>gs/003724cc5e41a981d3fad0ca3f21888d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>czxswqqjy</th>\n",
       "      <td>1-specifying-intent.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-12-...</td>\n",
       "      <td>2020-12-27T02:41:41.804Z</td>\n",
       "      <td>2020-12-27T03:01:14.187Z</td>\n",
       "      <td>gs/003724cc5e41a981d3fad0ca3f21888d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8pzgwyy22</th>\n",
       "      <td>1-specifying-intent.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-12-...</td>\n",
       "      <td>2020-12-27T03:02:30.953Z</td>\n",
       "      <td>2020-12-27T03:12:26.956Z</td>\n",
       "      <td>gs/003724cc5e41a981d3fad0ca3f21888d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ug7hr39o8</th>\n",
       "      <td>cars_demo.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-11-...</td>\n",
       "      <td>2020-11-30T05:36:43.016Z</td>\n",
       "      <td>2020-11-30T06:11:08.975Z</td>\n",
       "      <td>gs/003a65e7008e9f7db64a6559970f962c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97vczg8cn</th>\n",
       "      <td>cars_demo.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-12-...</td>\n",
       "      <td>2020-12-03T02:55:46.498Z</td>\n",
       "      <td>2020-12-03T03:12:19.732Z</td>\n",
       "      <td>gs/006b045214badd8a59f7f11c1eae488c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ak5h0c64n</th>\n",
       "      <td>4-Data-Playground.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-11-...</td>\n",
       "      <td>2020-11-04T22:20:55.715Z</td>\n",
       "      <td>2020-11-04T22:53:09.488Z</td>\n",
       "      <td>gs/fc836b6c537cc69b6eb6ab9cb7cb6b05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1ukitdr9</th>\n",
       "      <td>1-Specify-Intent.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-11-...</td>\n",
       "      <td>2020-11-04T21:34:36.099Z</td>\n",
       "      <td>2020-11-04T21:47:49.607Z</td>\n",
       "      <td>gs/fc836b6c537cc69b6eb6ab9cb7cb6b05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>o995ki32j</th>\n",
       "      <td>cars_demo.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-12-...</td>\n",
       "      <td>2020-12-01T06:17:35.472Z</td>\n",
       "      <td>2020-12-01T06:34:09.014Z</td>\n",
       "      <td>gs/fe50a4ea1e132d4d505b7941f1f48ef0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jiklvm3yu</th>\n",
       "      <td>cars_demo.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-12-...</td>\n",
       "      <td>2020-12-25T16:37:15.015Z</td>\n",
       "      <td>2020-12-25T17:06:12.739Z</td>\n",
       "      <td>gs/feba0e7c024be16c21b46a6257a10367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ru4jx8w5v</th>\n",
       "      <td>cars_demo.ipynb</td>\n",
       "      <td>[{'type': 'executeCodeCell', 'time': '2020-11-...</td>\n",
       "      <td>2020-11-30T00:37:42.834Z</td>\n",
       "      <td>2020-11-30T00:39:07.991Z</td>\n",
       "      <td>gs/fff0c89988ecc9587435c7f48dc586ee</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>495 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              nbName  \\\n",
       "6b6nhxtxh            cars_demo.ipynb   \n",
       "czxswqqjy  1-specifying-intent.ipynb   \n",
       "8pzgwyy22  1-specifying-intent.ipynb   \n",
       "ug7hr39o8            cars_demo.ipynb   \n",
       "97vczg8cn            cars_demo.ipynb   \n",
       "...                              ...   \n",
       "ak5h0c64n    4-Data-Playground.ipynb   \n",
       "f1ukitdr9     1-Specify-Intent.ipynb   \n",
       "o995ki32j            cars_demo.ipynb   \n",
       "jiklvm3yu            cars_demo.ipynb   \n",
       "ru4jx8w5v            cars_demo.ipynb   \n",
       "\n",
       "                                                     history  \\\n",
       "6b6nhxtxh  [{'type': 'executeCodeCell', 'time': '2020-12-...   \n",
       "czxswqqjy  [{'type': 'executeCodeCell', 'time': '2020-12-...   \n",
       "8pzgwyy22  [{'type': 'executeCodeCell', 'time': '2020-12-...   \n",
       "ug7hr39o8  [{'type': 'executeCodeCell', 'time': '2020-11-...   \n",
       "97vczg8cn  [{'type': 'executeCodeCell', 'time': '2020-12-...   \n",
       "...                                                      ...   \n",
       "ak5h0c64n  [{'type': 'executeCodeCell', 'time': '2020-11-...   \n",
       "f1ukitdr9  [{'type': 'executeCodeCell', 'time': '2020-11-...   \n",
       "o995ki32j  [{'type': 'executeCodeCell', 'time': '2020-12-...   \n",
       "jiklvm3yu  [{'type': 'executeCodeCell', 'time': '2020-12-...   \n",
       "ru4jx8w5v  [{'type': 'executeCodeCell', 'time': '2020-11-...   \n",
       "\n",
       "                          startTime                   endTime  \\\n",
       "6b6nhxtxh  2020-12-27T01:20:56.927Z  2020-12-27T01:47:49.232Z   \n",
       "czxswqqjy  2020-12-27T02:41:41.804Z  2020-12-27T03:01:14.187Z   \n",
       "8pzgwyy22  2020-12-27T03:02:30.953Z  2020-12-27T03:12:26.956Z   \n",
       "ug7hr39o8  2020-11-30T05:36:43.016Z  2020-11-30T06:11:08.975Z   \n",
       "97vczg8cn  2020-12-03T02:55:46.498Z  2020-12-03T03:12:19.732Z   \n",
       "...                             ...                       ...   \n",
       "ak5h0c64n  2020-11-04T22:20:55.715Z  2020-11-04T22:53:09.488Z   \n",
       "f1ukitdr9  2020-11-04T21:34:36.099Z  2020-11-04T21:47:49.607Z   \n",
       "o995ki32j  2020-12-01T06:17:35.472Z  2020-12-01T06:34:09.014Z   \n",
       "jiklvm3yu  2020-12-25T16:37:15.015Z  2020-12-25T17:06:12.739Z   \n",
       "ru4jx8w5v  2020-11-30T00:37:42.834Z  2020-11-30T00:39:07.991Z   \n",
       "\n",
       "                                      userHash  \n",
       "6b6nhxtxh  gs/003724cc5e41a981d3fad0ca3f21888d  \n",
       "czxswqqjy  gs/003724cc5e41a981d3fad0ca3f21888d  \n",
       "8pzgwyy22  gs/003724cc5e41a981d3fad0ca3f21888d  \n",
       "ug7hr39o8  gs/003a65e7008e9f7db64a6559970f962c  \n",
       "97vczg8cn  gs/006b045214badd8a59f7f11c1eae488c  \n",
       "...                                        ...  \n",
       "ak5h0c64n  gs/fc836b6c537cc69b6eb6ab9cb7cb6b05  \n",
       "f1ukitdr9  gs/fc836b6c537cc69b6eb6ab9cb7cb6b05  \n",
       "o995ki32j  gs/fe50a4ea1e132d4d505b7941f1f48ef0  \n",
       "jiklvm3yu  gs/feba0e7c024be16c21b46a6257a10367  \n",
       "ru4jx8w5v  gs/fff0c89988ecc9587435c7f48dc586ee  \n",
       "\n",
       "[495 rows x 5 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(formatted_logs).transpose().sort_values('userHash')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df[\"nbName\"]=='4-Data-Playground.ipynb'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"length\"] = df.history.apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[(df[\"nbName\"]=='4-Data-Playground.ipynb') &(df[\"length\"]>3)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for entry in df.values:\n",
    "    nb = nbf.v4.new_notebook()\n",
    "    code = \"import lux\\nimport pandas as pd\"\n",
    "    nb['cells'] = [nbf.v4.new_code_cell(code)]\n",
    "    nb['kernelspec'] = {\"display_name\": \"Python 3\", \"language\": \"python\",\"name\": \"python3\"}\n",
    "    for log_entry in entry[1]:\n",
    "        if log_entry['type'] == 'executeCodeCell':\n",
    "            nb['cells'].append(nbf.v4.new_code_cell(log_entry['code']))\n",
    "    name = entry[4][3:] + entry[0]\n",
    "    nbf.write(nb, '../constructed_nbs/' + name+\".ipynb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}