Hands-on 4 - Data combineren.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hands-on 4 - Data combineren\n",
    "Nu brengen we onze data samen\n",
    "Let op: we willen weten hoe goed het is gelukt!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "def load_speaker_data(filename):\n",
    "    \"\"\"Leest de spreker data in van de file\"\"\"\n",
    "    df = pd.read_csv(filename)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import urllib\n",
    "def get_gender(name):\n",
    "    \"\"\"Gegeven een naam, haalt de geslacht van die persoon op uit Wikidata.\n",
    "    Geeft een lege waarde terug als geen gender is gevonden\"\"\"\n",
    "    gender = \"Unknown\"\n",
    "    query = \"SELECT DISTINCT ?genderLabel WHERE {?person wdt:P31 wd:Q5 . ?person ?label \\\"%s\\\" .?person wdt:P21 ?gender .SERVICE wikibase:label {  bd:serviceParam wikibase:language \\\"nl\\\".}}\"%name\n",
    "    headers = {\"Accept\":\"application/json\",\n",
    "               \"User-Agent\":\"wikidataDataAlsKansDemoBot/0.0 (http://www.beeldengeluid.nl)\"\n",
    "              }\n",
    "    response = requests.get(\"https://query.wikidata.org/sparql?query=\" + urllib.parse.quote(query), headers=headers)\n",
    "    \n",
    "    if response.status_code == 200:\n",
    "        genders = [binding[\"genderLabel\"][\"value\"] for binding in response.json()[\"results\"][\"bindings\"] if \"genderLabel\" in binding]\n",
    "        if genders:\n",
    "            gender = genders[0]\n",
    "    return gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "def add_gender_for_all_speakers(spreker_data):\n",
    "    gender_column = []\n",
    "    for person_name in spreker_data['Naam']:\n",
    "        print(person_name)\n",
    "        gender_column.append(get_gender(person_name))        \n",
    "        time.sleep(1)\n",
    "        \n",
    "    spreker_data['Geslacht'] = gender_column\n",
    "    \n",
    "    return spreker_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "def add_gender_for_all_speakers_efficiently(spreker_data):\n",
    "    gender_column = []\n",
    "    gender_lookup = {}\n",
    "    for person_name in spreker_data['Naam']:\n",
    "        print(person_name)\n",
    "        if person_name in gender_lookup:  \n",
    "            print(person_name) \n",
    "            gender = gender_lookup[person_name]\n",
    "        else:\n",
    "            gender = get_gender(person_name)\n",
    "            gender_lookup[person_name] = gender   \n",
    "            time.sleep(1) \n",
    "              \n",
    "        gender_column.append(gender) \n",
    "        \n",
    "    spreker_data['Geslacht'] = gender_column\n",
    "    \n",
    "    print(f\"Missing genders for {gender_column.count('Unknown')} of {len(gender_column)} persons\")\n",
    "    \n",
    "    return spreker_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spreker data combineren met geslacht"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spreker_data = load_speaker_data(\"speaker_times.csv\")\n",
    "print(spreker_data)\n",
    "spreker_geslacht_data = add_gender_for_all_speakers(spreker_data)\n",
    "print(spreker_geslacht_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "jupyter",
   "language": "python",
   "name": "jupyter"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}