catalog-check.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check an opendap catalog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "import xml.etree.ElementTree\n",
    "import lxml.etree\n",
    "import datetime\n",
    "import requests\n",
    "import tqdm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "catalog = pathlib.Path('/Users/baart_f/src/thredds-docker/catalogList.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = lxml.etree.parse(str(catalog))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81\n"
     ]
    }
   ],
   "source": [
    "refs = tree.findall('.//{http://www.unidata.ucar.edu/namespaces/thredds/InvCatalog/v1.0}catalogRef')\n",
    "print(len(refs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41779241b519456f887d4da38c370ab7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=81), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for i, ref in enumerate(tqdm.tqdm_notebook(refs)):\n",
    "    url = ref.attrib['{http://www.w3.org/1999/xlink}href']    \n",
    "    msg = ''\n",
    "    parent = ref.getparent()\n",
    "    \n",
    "    try:\n",
    "        resp = requests.get(url, timeout=5)\n",
    "        if resp.status_code != 200:\n",
    "            msg = \"result not ok (status %s)\" % (resp.status_code, )        \n",
    "    except requests.exceptions.Timeout as e: \n",
    "        msg = \"result not ok (timeout)\"\n",
    "    except requests.exceptions.ConnectionError as e:\n",
    "        msg = \"result not ok (connection error)\"\n",
    "        \n",
    "    if (msg):\n",
    "        comment = lxml.etree.Comment(\n",
    "            'Url checked: not ok: %s (%s)' % (msg, datetime.datetime.now(), )\n",
    "        )\n",
    "        ref.addprevious(comment)\n",
    "        ref_string = lxml.etree.tostring(ref)\n",
    "        comment = lxml.etree.Comment(ref_string)        \n",
    "        # replace by comment\n",
    "        parent.replace(ref, comment)\n",
    "    else:\n",
    "        comment = lxml.etree.Comment('Url checked: ok (%s)' % (datetime.datetime.now(), ))\n",
    "        ref.addprevious(comment)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree.write('catalog.xml',  xml_declaration=True, pretty_print=True, encoding='UTF-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "!open .\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<lxml.etree._ElementTree at 0x10465f588>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}