get_all_agencies_by_function.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Harvest agencies associated with *all* functions\n",
    "\n",
    "This notebook loops through the list of functions that were [extracted from the RecordSearch interface](harvesting_functions_from_recordsearch.ipynb) and saves basic details of the agencies responsible for each function. To keep down the file size and avoid too much duplication it doesn't include the full range of relationships that an agency might have. If you want the full agency data, use [this notebook](get_agencies_associated_with_function.ipynb) to harvest agencies associated with an indivividual function or hierarchy.\n",
    "\n",
    "The JSON data file created has the following structure:\n",
    "\n",
    "``` json\n",
    "[\n",
    "    {\n",
    "        'term': FUNCTION NAME\n",
    "        'agencies': [\n",
    "            'agency_id': AGENCY IDENTIFIER,\n",
    "            'title': AGENCY NAME,\n",
    "            'dates': {\n",
    "                'date_str': AGENCY LIFE DATES AS A STRING,\n",
    "                'start_date': AGENCY START DATE (YYYY-MM-DD),\n",
    "                'end_date': AGENCY END DATE (YYYY-MM-DD),\n",
    "            },\n",
    "            'agency_status': TYPE/LEVEL OF AGENCY,\n",
    "            'location': AGENCY LOCATION,\n",
    "            'function_start_date': DATE AGENCY STARTED BEING RESPONSIBLE FOR THIS FUNCTION (YYYY-MM-DD),\n",
    "            'function_end_date': DATE AGENCY STOPPED BEING RESPONSIBLE FOR THIS FUNCTION (YYYY-MM-DD),\n",
    "        ]\n",
    "    }\n",
    "]\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up the harvesting code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "\n",
    "from IPython.display import FileLink, clear_output, display\n",
    "from recordsearch_data_scraper.scrapers import RSAgencySearch\n",
    "from tinydb import Query, TinyDB\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def harvest_agencies(function):\n",
    "    agencies = []\n",
    "    search = RSAgencySearch(function=function, record_detail=\"full\")\n",
    "    with tqdm(total=search.total_results) as pbar:\n",
    "        more = True\n",
    "        while more:\n",
    "            data = search.get_results()\n",
    "            if data[\"results\"]:\n",
    "                agencies += data[\"results\"]\n",
    "                pbar.update(len(data[\"results\"]))\n",
    "                time.sleep(0.5)\n",
    "            else:\n",
    "                more = False\n",
    "    return agencies\n",
    "\n",
    "\n",
    "def get_children(function):\n",
    "    \"\"\"\n",
    "    Gets child terms of a given function.\n",
    "    \"\"\"\n",
    "    f_list = []\n",
    "    if \"narrower\" in function:\n",
    "        for subf in function[\"narrower\"]:\n",
    "            f_list.append(subf[\"term\"])\n",
    "            f_list += get_children(subf)\n",
    "    return f_list\n",
    "\n",
    "\n",
    "def load_functions():\n",
    "    \"\"\"\n",
    "    Loads a pre-harvested JSON file containing functions data.\n",
    "    Returns a flat list of functions.\n",
    "    \"\"\"\n",
    "    functions_list = []\n",
    "    with open(\"data/functions.json\", \"r\") as json_file:\n",
    "        functions = json.load(json_file)\n",
    "    for function in functions:\n",
    "        functions_list.append(function[\"term\"])\n",
    "        functions_list += get_children(function)\n",
    "    # Get rid of duplicates\n",
    "    functions_list = set(functions_list)\n",
    "    # Sort terms\n",
    "    functions_list = sorted(functions_list)\n",
    "    return functions_list\n",
    "\n",
    "\n",
    "def get_function_dates(function, agency):\n",
    "    \"\"\"\n",
    "    Get the dates an agency was responsible for a given function.\n",
    "    \"\"\"\n",
    "    dates = {}\n",
    "    # Loop through the functions associated with an agency\n",
    "    for f in agency[\"functions\"]:\n",
    "        # Find the current function\n",
    "        if f[\"identifier\"].lower() == function:\n",
    "            # Get the dates this agency was responsible for the current function\n",
    "            dates[\"function_start_date\"] = f[\"start_date\"]\n",
    "            dates[\"function_end_date\"] = f[\"end_date\"]\n",
    "            break\n",
    "    return dates\n",
    "\n",
    "\n",
    "def get_all_agencies():\n",
    "    \"\"\"\n",
    "    Sends function terms off to the harvester to get related agencies.\n",
    "    \"\"\"\n",
    "    clear_output()\n",
    "    Record = Query()\n",
    "    # Get a list of functions\n",
    "    functions = load_functions()\n",
    "    db = TinyDB(\"data/db_agencies_by_function\")\n",
    "    # Loop through the list of functions\n",
    "    for function in functions:\n",
    "        clear_output()\n",
    "        print('\\nHarvesting \"{}\"'.format(function))\n",
    "        # Fire up the harvester for this function\n",
    "        results = harvest_agencies(function)\n",
    "        agencies = []\n",
    "        # Create a subset of the agency data to limit the filesize\n",
    "        for a in results:\n",
    "            # Keep the fields we want\n",
    "            agency = {\n",
    "                k: a[k]\n",
    "                for k in [\n",
    "                    \"identifier\",\n",
    "                    \"title\",\n",
    "                    \"start_date\",\n",
    "                    \"end_date\",\n",
    "                    \"agency_status\",\n",
    "                    \"location\",\n",
    "                ]\n",
    "            }\n",
    "            # Add extra fields to show when the agency was responsible for this function\n",
    "            agency.update(get_function_dates(function, a))\n",
    "            agencies.append(agency)\n",
    "        db.upsert({\"term\": function, \"agencies\": agencies}, Record.term == function)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start the harvest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "nbval-skip"
    ]
   },
   "outputs": [],
   "source": [
    "get_all_agencies()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the results for download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_json():\n",
    "    db = TinyDB(\"data/db_agencies_by_function\")\n",
    "    functions = db.all()\n",
    "    filename = \"data/agencies_by_function.json\"\n",
    "    with open(filename, \"w\") as json_file:\n",
    "        json.dump(functions, json_file, indent=4)\n",
    "        display(FileLink(filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": [
     "nbval-skip"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='data/agencies_by_function.json' target='_blank'>data/agencies_by_function.json</a><br>"
      ],
      "text/plain": [
       "/home/tim/Workspace/mycode/glam-workbench/recordsearch/notebooks/data/agencies_by_function.json"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "save_json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "\n",
    "Created by [Tim Sherratt](https://timsherratt.org/) as part of the [GLAM Workbench](https://glam-workbench.github.io/)."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}