newspaper_harvester_app.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "# Trove Newspaper & Gazette Harvester\n",
    "\n",
    "Download large quantities of digitised newspaper and gazette articles from [Trove](https://trove.nla.gov.au/newspaper/) with this simplified, web-based interface to the [Trove Harvester](https://github.com/wragge/troveharvester) command-line tool."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This notebook is designed to run in Voila as an app (with the code hidden).\n",
    "# To launch this notebook in Voila, just select 'View > Open with Voila in New Browser Tab'\n",
    "# Your browser might ask for permission to open the new tab as a popup."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "from pathlib import Path\n",
    "\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import HTML, display\n",
    "from requests.exceptions import HTTPError\n",
    "from trove_newspaper_harvester.core import (\n",
    "    Harvester,\n",
    "    NoQueryError,\n",
    "    get_harvest,\n",
    "    prepare_query,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "# Load environment variables if available\n",
    "%load_ext dotenv\n",
    "%dotenv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def start_harvest(b):\n",
    "    out.clear_output()\n",
    "    query_params = prepare_query(query=query_url.value)\n",
    "    try:\n",
    "        harvester = Harvester(\n",
    "            query_params=query_params,\n",
    "            key=api_key.value,\n",
    "            text=text.value,\n",
    "            pdf=pdf.value,\n",
    "            image=image.value,\n",
    "        )\n",
    "    except HTTPError as e:\n",
    "        if e.response.status_code == 403:\n",
    "            with out:\n",
    "                print(\"The request could not be authorised, check your API key.\")\n",
    "        else:\n",
    "            raise\n",
    "    except NoQueryError:\n",
    "        with out:\n",
    "            print(\"No query parameters found, check your query url.\")\n",
    "    else:\n",
    "        with out:\n",
    "            harvester.harvest()\n",
    "            if harvester.harvested > 0:\n",
    "                harvest = get_harvest()\n",
    "                harvester.save_csv()\n",
    "                Path(harvest, \"results.ndjson\").unlink()\n",
    "                harvester.remove_ndjson_from_crate()\n",
    "                shutil.make_archive(harvest, \"zip\", harvest)\n",
    "                display(\n",
    "                    HTML(\n",
    "                        f'<b>Download results</b>: <a href=\"{str(harvest)}.zip\" download>{str(harvest)}.zip</a>'\n",
    "                    )\n",
    "                )\n",
    "            else:\n",
    "                print(\"No results were harvested, check your query url.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All the UI stuff\n",
    "\n",
    "# Somewhere to put the results\n",
    "out = widgets.Output(layout=widgets.Layout(padding=\"40px\"))\n",
    "\n",
    "api_key = widgets.Text(\n",
    "    placeholder=\"Enter your Trove API key\", description=\"API key:\", disabled=False\n",
    ")\n",
    "\n",
    "query_url = widgets.Text(\n",
    "    placeholder=\"Enter the url of your search\",\n",
    "    description=\"Query url:\",\n",
    "    disabled=False,\n",
    "    layout=widgets.Layout(width=\"100%\"),\n",
    ")\n",
    "\n",
    "text = widgets.Checkbox(value=False, description=\"Save full text\", disabled=False)\n",
    "\n",
    "pdf = widgets.Checkbox(value=False, description=\"Save PDFs\", disabled=False)\n",
    "\n",
    "image = widgets.Checkbox(\n",
    "    value=False, description=\"Save articles as images\", disabled=False\n",
    ")\n",
    "\n",
    "start_button = widgets.Button(\n",
    "    description=\"Start harvest\",\n",
    "    disabled=False,\n",
    "    button_style=\"primary\",  # 'success', 'info', 'warning', 'danger' or ''\n",
    "    tooltip=\"Start harvest\",\n",
    "    icon=\"\",\n",
    ")\n",
    "\n",
    "start_button.on_click(start_harvest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "## Enter your Trove API key\n",
    "\n",
    "The harvester gets its data from the Trove API. To use the API [you need a key](https://trove.nla.gov.au/about/create-something/using-api) — the process is quick, painless, and free. Once you have a key, paste it in below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "display(api_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Enter your search query\n",
    "\n",
    "Use the [Trove web interface](https://trove.nla.gov.au/newspaper/) to construct your search. Remember that the harvester will get **all** of the matched results, not just the first 2,000 you see in the web interface. Once you're happy with your search, just copy the url and paste it below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(query_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "source": [
    "## Set harvest options\n",
    "\n",
    "By default the harvester only saves the metadata (date, page, title, newspaper etc) from the search results. If you want to save the full text content of each article, just check the 'Save full text' box. You can also save JPEG and PDF copies of every article by checking the 'Save articles as images' or 'Save PDFs' boxes, but note that this will slow down your harvest and generate large download files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(text)\n",
    "display(pdf)\n",
    "display(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(start_button)\n",
    "display(out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once your harvest is complete a link will appear to download the results as a single, zipped file. See the GLAM Workbench for more information about the [contents and format of the results folder](https://glam-workbench.net/trove-harvester/#your-harvested-data)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TESTING\n",
    "if os.getenv(\"GW_STATUS\") == \"dev\" and os.getenv(\"TROVE_API_KEY\"):\n",
    "    api_key.value = os.getenv(\"TROVE_API_KEY\")\n",
    "    query_url.value = \"https://trove.nla.gov.au/search/category/newspapers?keyword=wragge%201902&l-artType=newspapers&l-state=Queensland&l-title=840\"\n",
    "    start_button.click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "\n",
    "Created by [Tim Sherratt](https://timsherratt.org) ([@wragge](https://twitter.com/wragge)) for the [GLAM Workbench](https://github.com/glam-workbench/).  \n",
    "Support this project by [becoming a GitHub sponsor](https://github.com/sponsors/wragge?o=esb).\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0569896b7c354143962d45a97845a3d4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "0a78eeff0d6241e29f2fba98566492c4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_c480d7915c7c4da7a43594985f65c743",
       "max": 272,
       "style": "IPY_MODEL_8db3d211343f481ab54a4a9bb330d18c",
       "value": 272
      }
     },
     "15066b1037ee4db1919b938563bf4305": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "29eb3e0a61f642f7a68aba150f384210": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "39eec5d605c04756832d36dc61d7d810": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "56a0329997854750a35e9e9860422787": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ButtonStyleModel",
      "state": {}
     },
     "682a826ebe374a918870016dcd669d4d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_6bd6f5ca0376488da68d830aed48a18a",
        "IPY_MODEL_0a78eeff0d6241e29f2fba98566492c4",
        "IPY_MODEL_be0f90c5df9942758c8116a82f46ff80"
       ],
       "layout": "IPY_MODEL_0569896b7c354143962d45a97845a3d4"
      }
     },
     "6bd6f5ca0376488da68d830aed48a18a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_e0cba7bb37994cf9b59da82c73b5bcba",
       "style": "IPY_MODEL_b66bdf06718d48ceb9163c4dfbf317ef",
       "value": "100%"
      }
     },
     "72935870e9794cd796ef6b95b63c55f2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "76cc414602f14519afa8a3cfd4dd3652": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "CheckboxModel",
      "state": {
       "description": "Save PDFs",
       "disabled": false,
       "layout": "IPY_MODEL_7faa7e404c874c30ad64efc66215f7e2",
       "style": "IPY_MODEL_e7e37e6cdbe34a18994b82a64480f7e4",
       "value": false
      }
     },
     "7b32cae3153b4a118523745a72911e9f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "7faa7e404c874c30ad64efc66215f7e2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "8411ec2bd251455989882753a3a380f4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ButtonModel",
      "state": {
       "button_style": "primary",
       "description": "Start harvest",
       "layout": "IPY_MODEL_29eb3e0a61f642f7a68aba150f384210",
       "style": "IPY_MODEL_56a0329997854750a35e9e9860422787",
       "tooltip": "Start harvest"
      }
     },
     "87a5205519bf49deb02ec146eed6276c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "padding": "40px"
      }
     },
     "87e34ab14c3140e583ab25fee9b52682": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "8db3d211343f481ab54a4a9bb330d18c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "91d5951bcfb142f4a2beb29e883251aa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "width": "100%"
      }
     },
     "93af297c863a4bb29c1593bafd02742b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "TextModel",
      "state": {
       "description": "API key:",
       "layout": "IPY_MODEL_72935870e9794cd796ef6b95b63c55f2",
       "placeholder": "Enter your Trove API key",
       "style": "IPY_MODEL_fc49951118cf4aadbff2c0aedab5a1b6",
       "value": "gq29l1g1h75pimh4"
      }
     },
     "966d6e3d45664a78a31b057b7bf6726e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "CheckboxModel",
      "state": {
       "description": "Save articles as images",
       "disabled": false,
       "layout": "IPY_MODEL_9ffb5cd929e94c6fa4898a21cbf92589",
       "style": "IPY_MODEL_87e34ab14c3140e583ab25fee9b52682",
       "value": false
      }
     },
     "9ffb5cd929e94c6fa4898a21cbf92589": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "a73cb313c35748f88f7a013d6cc59063": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "CheckboxModel",
      "state": {
       "description": "Save full text",
       "disabled": false,
       "layout": "IPY_MODEL_ad7834c1ce934362bd3da86727282b54",
       "style": "IPY_MODEL_7b32cae3153b4a118523745a72911e9f",
       "value": false
      }
     },
     "ad7834c1ce934362bd3da86727282b54": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "b4a2f923c96f45468dac29e6c8ab2da4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "b66bdf06718d48ceb9163c4dfbf317ef": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "b91d850c44c2458e92f51bb8fb752785": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "TextModel",
      "state": {
       "description": "Query url:",
       "layout": "IPY_MODEL_91d5951bcfb142f4a2beb29e883251aa",
       "placeholder": "Enter the url of your search",
       "style": "IPY_MODEL_15066b1037ee4db1919b938563bf4305"
      }
     },
     "be0f90c5df9942758c8116a82f46ff80": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_b4a2f923c96f45468dac29e6c8ab2da4",
       "style": "IPY_MODEL_39eec5d605c04756832d36dc61d7d810",
       "value": " 272/272 [00:04&lt;00:00, 58.41article/s]"
      }
     },
     "c480d7915c7c4da7a43594985f65c743": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "ca0d628e49d94533b92f1ff3e6435a76": {
      "model_module": "@jupyter-widgets/output",
      "model_module_version": "1.0.0",
      "model_name": "OutputModel",
      "state": {
       "layout": "IPY_MODEL_87a5205519bf49deb02ec146eed6276c"
      }
     },
     "e0cba7bb37994cf9b59da82c73b5bcba": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "e7e37e6cdbe34a18994b82a64480f7e4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "fc49951118cf4aadbff2c0aedab5a1b6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}