Authors-via-SRU.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ca17b7c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests \n",
    "from bs4 import BeautifulSoup\n",
    "from lxml import etree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "99b90f5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "names = pd.read_csv(\"authornames-csv.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "33ce0307",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number</th>\n",
       "      <th>Name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>A., Sonay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>Aarestrup, Emil</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>Abarbanell, Stephan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Abbott, David</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>Äbdülrähmanlı, Näriman</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4956</th>\n",
       "      <td>4959</td>\n",
       "      <td>Гейне, Генріх</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4957</th>\n",
       "      <td>4960</td>\n",
       "      <td>Гете, Иоганн Вольфганг</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4958</th>\n",
       "      <td>4961</td>\n",
       "      <td>Гримм, Братья</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4959</th>\n",
       "      <td>4962</td>\n",
       "      <td>Захер-Мазох, Леопольд фон</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4960</th>\n",
       "      <td>4963</td>\n",
       "      <td>Майринк, Густав</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4961 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Number                       Name\n",
       "0          3                  A., Sonay\n",
       "1          4            Aarestrup, Emil\n",
       "2          5        Abarbanell, Stephan\n",
       "3          6              Abbott, David\n",
       "4          7     Äbdülrähmanlı, Näriman\n",
       "...      ...                        ...\n",
       "4956    4959              Гейне, Генріх\n",
       "4957    4960     Гете, Иоганн Вольфганг\n",
       "4958    4961              Гримм, Братья\n",
       "4959    4962  Захер-Мазох, Леопольд фон\n",
       "4960    4963            Майринк, Густав\n",
       "\n",
       "[4961 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "8f6f3f09",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Names to list: \n",
    "new = names[\"Name\"].to_list()\n",
    "shortlist = new[0:250]\n",
    "supershort = new[0:25]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "e449bb0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None]\n"
     ]
    }
   ],
   "source": [
    "def solve_problem(entry): \n",
    "    if entry.endswith('.'):\n",
    "        entry = entry.replace(entry, \"testestestestestestsetes\")\n",
    "    else:\n",
    "        entry = entry\n",
    "\n",
    "new = [solve_problem(x) for x in supershort]\n",
    "print(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "76c3b178",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A., Sonay OR Aarestrup, Emil OR Abarbanell, Stephan OR Abbott, David OR Äbdülrähmanlı, Näriman OR Abedi, Isabel OR Abel, Winnie OR Abele, Jan OR Abell, Arthur M. OR Abeln, Reinhard OR Ablaß, Werner E. OR Abraham, Filo M. OR Abreu, Carlos OR Acero, Irene OR Achermann, Franz Heinrich OR Achleitner, Friedrich OR Achternbusch, Herbert OR Ackermann, Erich OR Ackermann, Rolf OR Acklin, Jürg OR Acosta, Luis A. OR Adam, Gabi OR Adams Richards, David OR Adams, Lilly OR Addams, Peter\n",
      "A., Sonay OR Aarestrup, Emil OR Abarbanell, Stephan OR Abbott, David OR Äbdülrähmanlı, Näriman OR Abedi, Isabel OR Abel, Winnie OR Abele, Jan OR Abell, Arthur M. OR Abeln, Reinhard OR Ablaß, Werner E. OR Abraham, Filo M. OR Abreu, Carlos OR Acero, Irene OR Achermann, Franz Heinrich OR Achleitner, Friedrich OR Achternbusch, Herbert OR Ackermann, Erich OR Ackermann, Rolf OR Acklin, Jürg OR Acosta, Luis A. OR Adam, Gabi OR Adams Richards, David OR Adams, Lilly OR Addams, Peter\n"
     ]
    }
   ],
   "source": [
    "#Umwandeln Liste in Text für Anfrage: \n",
    "\n",
    "string = \"\"\n",
    "first = True\n",
    "\n",
    "for item in supershort: \n",
    "    if first:\n",
    "        #string = string + \"(\" + item + \")\"\n",
    "        string = string + item \n",
    "        first = False\n",
    "    else: \n",
    "        #string = string + \" OR (\" + item + \")\"\n",
    "        string = string + \" OR \" + item\n",
    "    \n",
    "print(string)\n",
    "\n",
    "searchtext = string\n",
    "print(searchtext)\n",
    "\n",
    "\n",
    "#per=((A., Sonay) OR (Abarbanell, Stephan) OR (Abbott, David) OR (Abedi, Isabel) OR (Abel, Winnie)) and bbg=Tp*\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "ac0d9621",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://services.dnb.de/sru/authorities?version=1.1&operation=searchRetrieve&query=PER%3DAbarbanell%2C+Stephan%2520and+BBG%3DTp%2A&recordSchema=MARC21-xml\n",
      "<?xml version=\"1.0\" encoding=\"UTF-8\"?><html><body><searchretrieveresponse xmlns=\"http://www.loc.gov/zing/srw/\"><version>1.1</version><diagnostics><diag:diagnostic xmlns:diag=\"http://www.loc.gov/zing/srw/diagnostic/\"><diag:uri>info:srw/diagnostic/1/16</diag:uri><diag:details>Unsupported index</diag:details><diag:message>Abarbanell, Stephan%20and BBG</diag:message></diag:diagnostic></diagnostics></searchretrieveresponse></body></html>\n"
     ]
    }
   ],
   "source": [
    "#Send query to SRU: \n",
    "base_url = \"https://services.dnb.de/sru/authorities\" #for GND\n",
    "searchtext = \"PER=\"+\"Abarbanell, Stephan\"+\"%20\"+\"and BBG=Tp*\"\n",
    "\n",
    "parameter = {'version' : '1.1' , 'operation' : 'searchRetrieve' , 'query' : searchtext, 'recordSchema' : \"MARC21-xml\"}\n",
    "\n",
    "r1 = requests.get(base_url, params = parameter)\n",
    "print(r1.url)\n",
    "\n",
    "response = BeautifulSoup(r1.content)\n",
    "records = response.find_all('record')\n",
    "\n",
    "print(response)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "fb39dfd7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "608c5f2c5c604e78825950fd35d4f9eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/25 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Funktion für SRU-Schnittstelle: \n",
    "worklist = []\n",
    "from tqdm.notebook import tqdm\n",
    "from time import sleep\n",
    "\n",
    "i=0\n",
    "base_url = \"https://services.dnb.de/sru/authorities\" #for GND\n",
    "    \n",
    "for i in tqdm(range(0,len(supershort))):\n",
    "    #sleep(3)\n",
    "    \n",
    "    searchtext = \"PER=\"+supershort[i]+\"%20\"+\"and bbg=Tp*\"\n",
    "    parameter = {'version' : '1.1' , 'operation' : 'searchRetrieve' , 'query' : searchtext, 'recordSchema' : \"MARC21-xml\"}\n",
    "\n",
    "    \n",
    "    r1 = requests.get(base_url, params = parameter)\n",
    "    #print(r.url)\n",
    "\n",
    "    #Weiter mit Beautiful Soup\n",
    "    response = BeautifulSoup(r1.content)\n",
    "    records = response.find_all('record')\n",
    "\n",
    "    \n",
    "    query = supershort[i]\n",
    "    if records:\n",
    "        found = \"yes\"\n",
    "    else:\n",
    "        found = \"no\"\n",
    "        \n",
    "    result = {\"query\":query, \"found\":found} \n",
    "    worklist.append(result)\n",
    "    \n",
    "    \n",
    "    #output = [parse_record(record) for record in records]\n",
    "    #df = pd.DataFrame(output)\n",
    "    #worklist.append(df)\n",
    "\n",
    "    i+=1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "f025d68b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>found</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A., Sonay</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Aarestrup, Emil</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Abarbanell, Stephan</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Abbott, David</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Äbdülrähmanlı, Näriman</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Abedi, Isabel</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Abel, Winnie</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Abele, Jan</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Abell, Arthur M.</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Abeln, Reinhard</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Ablaß, Werner E.</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Abraham, Filo M.</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Abreu, Carlos</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Acero, Irene</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Achermann, Franz Heinrich</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Achleitner, Friedrich</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Achternbusch, Herbert</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Ackermann, Erich</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Ackermann, Rolf</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Acklin, Jürg</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Acosta, Luis A.</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Adam, Gabi</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Adams Richards, David</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Adams, Lilly</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Addams, Peter</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        query found\n",
       "0                   A., Sonay    no\n",
       "1             Aarestrup, Emil    no\n",
       "2         Abarbanell, Stephan    no\n",
       "3               Abbott, David    no\n",
       "4      Äbdülrähmanlı, Näriman    no\n",
       "5               Abedi, Isabel    no\n",
       "6                Abel, Winnie    no\n",
       "7                  Abele, Jan    no\n",
       "8            Abell, Arthur M.   yes\n",
       "9             Abeln, Reinhard    no\n",
       "10           Ablaß, Werner E.   yes\n",
       "11           Abraham, Filo M.   yes\n",
       "12              Abreu, Carlos    no\n",
       "13               Acero, Irene    no\n",
       "14  Achermann, Franz Heinrich    no\n",
       "15      Achleitner, Friedrich    no\n",
       "16      Achternbusch, Herbert    no\n",
       "17           Ackermann, Erich    no\n",
       "18            Ackermann, Rolf    no\n",
       "19               Acklin, Jürg    no\n",
       "20            Acosta, Luis A.   yes\n",
       "21                 Adam, Gabi    no\n",
       "22      Adams Richards, David    no\n",
       "23               Adams, Lilly    no\n",
       "24              Addams, Peter    no"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(worklist)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce8a5150",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}