data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "get all launches, which contains \"corona\" or \"covid\" in `spec`, from [Binder Gallery API](https://notebooks.gesis.org/gallery/api/v1.0/) and write them into a csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import csv\n",
    "import pandas as pd\n",
    "from datetime import datetime, date, timedelta\n",
    "from time import sleep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select a date which is early enough to get all launches of covid repos\n",
    "from_dt = datetime(2019,12,1)\n",
    "# from_dt = datetime(2020,6,5)\n",
    "to_dt = datetime(2020,9,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# launches = []\n",
    "\n",
    "# # from_dt and to_dt: Date and time in ISO 8601 format in UTC, e.g. 2019-05-31T16:17:56.946703\n",
    "# api_query_url = f'https://notebooks.gesis.org/gallery/api/v1.0/launches/{from_dt.isoformat()}/{to_dt.isoformat()}'\n",
    "\n",
    "# # because of pagination the api gives 500 results per page\n",
    "# # so for analysis you have to take data in all pages\n",
    "# next_page = 1\n",
    "# while next_page is not None:\n",
    "#     r = requests.get(api_query_url, params={'page': str(next_page)})\n",
    "#     # ex: https://notebooks.gesis.org/gallery/api/v1.0/launches/2019-12-01T00:00:00/2020-06-06T00:00:00?page=1\n",
    "#     response = r.json()\n",
    "\n",
    "#     if r.status_code == 429:\n",
    "#         # check the limit for # queries per second/minute\n",
    "#         #print(f'429: {response[\"message\"]}')\n",
    "#         sleep(1)\n",
    "#         continue\n",
    "\n",
    "#     assert r.status_code == 200\n",
    "        \n",
    "#     for launch in response['launches']:\n",
    "#         # get only covid related repos\n",
    "#         if \"covid\" in launch[\"spec\"] or \"corona\" in launch[\"spec\"]:\n",
    "#             launches.append(launch)\n",
    "#     # print(launch[\"timestamp\"], next_page, len(launches))\n",
    "#     next_page = response['next_page']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "launches = []\n",
    "archive_date = from_dt\n",
    "delta = timedelta(days=1)\n",
    "while archive_date <= to_dt:\n",
    "    archive_url = f\"https://archive.analytics.mybinder.org/events-{archive_date.date()}.jsonl\"\n",
    "    df = pd.read_json(archive_url, lines=True)\n",
    "    if 'ref' not in df.columns:\n",
    "        df['ref'] = \"\"\n",
    "    df[\"timestamp\"] = df[\"timestamp\"].dt.strftime(\"%Y-%m-%dT%H:%M:%S\")\n",
    "    for launch in df.to_dict(orient=\"records\"):\n",
    "        # get only covid related repos\n",
    "        if \"covid\" in launch[\"spec\"] or \"corona\" in launch[\"spec\"]:\n",
    "            launches.append(launch)\n",
    "    # print(archive_url, len(launches))\n",
    "    archive_date += delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(launches))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write launches into csv file\n",
    "header = launches[0].keys()\n",
    "filename = f\"covid_binder_launches_{from_dt.date()}_{to_dt.date()}.csv\".replace(\"-\", \"_\")\n",
    "with open(filename, \"w\") as output_file:\n",
    "    dict_writer = csv.DictWriter(output_file, header)\n",
    "    dict_writer.writeheader()\n",
    "    dict_writer.writerows(launches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}