data_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting data from the court cases scraped using scrapingcases.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import regex as re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some helper functions that find the case number and the date of the ruling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monat(month):\n",
    "    monate = ['Januar', 'Februar', 'März', 'April', 'Mai', 'Juni', 'Juli', 'August', 'September', 'Oktober', 'November', 'Dezember']\n",
    "    monthint = monate.index(month) + 1        \n",
    "    return monthint\n",
    "\n",
    "def mese(month):\n",
    "    monate = ['gennaio', 'febbraio', 'marzo', 'aprile', 'maggio', 'giugno', 'luglio', 'agosto', 'settembre', 'ottobre', 'novembre', 'dicembre']\n",
    "    monthint = monate.index(month.lower()) + 1        \n",
    "    return monthint\n",
    "    \n",
    "def mois(month):\n",
    "    monate = ['janvi', 'févri', 'mars', 'avril', 'mai', 'juin', 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre']\n",
    "    monthint = monate.index(month.lower()) + 1        \n",
    "    return monthint\n",
    "\n",
    "def find_case_number(ruling):\n",
    "    case_number = re.findall('[D-E]{1}-[0-9]{1,4}/[0-9]{4}', ruling)\n",
    "    return case_number\n",
    "\n",
    "\n",
    "def to_datetime(head):\n",
    "    if 'Urteil' in head:\n",
    "        dateline = re.findall('Urteil vom\\s{1,2}[0-9]{1,2}\\.\\s{0,1}\\w+ [0-9]{4}', head)\n",
    "        dateline = dateline[0]\n",
    "\n",
    "        day = re.findall('[0-9]{1,2}', dateline)\n",
    "        day = day[0]\n",
    "        month = re.findall('[A-S]{1}[a-zä]{1,20}', dateline)\n",
    "        month = monat(month[0])\n",
    "        year = re.findall('[0-9]{4}', dateline)\n",
    "        year = year[0]\n",
    "        \n",
    "        date = pd.to_datetime(str(day) + '/' + str(month) + '/' + str(year), format='%d/%m/%Y')\n",
    "        return date\n",
    "    \n",
    "    if 'Arrêt' in head:\n",
    "        dateline = re.findall('Arrêt du\\s{1,2}[0-9]{1,2}\\s{0,2}\\w+ [0-9]{4}', head.replace('er', '')) #.replace('nd', ''))\n",
    "        dateline = dateline[0]\n",
    "        \n",
    "        day = re.findall('[0-9]{1,2}', dateline)\n",
    "        day = day[0]\n",
    "        month = re.findall('[a-zéôû]{1,20}', dateline)\n",
    "        month = mois(month[3])\n",
    "        year = re.findall('[0-9]{4}', dateline)\n",
    "        year = year[0]\n",
    "\n",
    "        date = pd.to_datetime(str(day) + '/' + str(month) + '/' + str(year), format='%d/%m/%Y')\n",
    "        return date\n",
    "    \n",
    "    if 'Sentenza' in head:\n",
    "        dateline = re.findall('Sentenza del\\s{1,2}[0-9]{1,2}\\s{0,2}\\w+ [0-9]{4}', head.replace('°', '').replace(\"l'\", \" \")) #.replace('nd', ''))\n",
    "        no_del = re.findall('Sentenza\\s{1,2}[0-9]{1,2}\\s{0,2}\\w+ [0-9]{4}', head.replace('°', '').replace(\"l'\", \" \")) #.replace('nd', ''))\n",
    "        if len(no_del) > 0:\n",
    "            dateline = no_del[0].replace('Sentenza', 'Sentenza del')\n",
    "        \n",
    "        else:\n",
    "            dateline = dateline[0]\n",
    "        \n",
    "        day = re.findall('[0-9]{1,2}', dateline)\n",
    "        day = day[0]\n",
    "        month = re.findall('[a-zéôû]{1,20}', dateline)\n",
    "        month = mese(month[2])\n",
    "        year = re.findall('[0-9]{4}', dateline)\n",
    "        year = year[0]\n",
    "\n",
    "        date = pd.to_datetime(str(day) + '/' + str(month) + '/' + str(year), format='%d/%m/%Y')\n",
    "        return date"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make list of nationalities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pycountry'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-cf4232019476>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mgettext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mpycountry\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mcountries_german\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcountry\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpycountry\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcountries\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pycountry'"
     ]
    }
   ],
   "source": [
    "import gettext\n",
    "import pycountry\n",
    "\n",
    "countries_german = []\n",
    "for country in pycountry.countries:\n",
    "    german = gettext.translation('iso3166', pycountry.LOCALES_DIR, languages=['de'])\n",
    "    german.install()\n",
    "    countries_german.append(_(country.name))\n",
    "\n",
    "countries_french = []\n",
    "for country in pycountry.countries:\n",
    "    french = gettext.translation('iso3166', pycountry.LOCALES_DIR, languages=['fr'])\n",
    "    french.install()\n",
    "    countries_french.append(_(country.name))\n",
    "\n",
    "countries_italian = []\n",
    "for country in pycountry.countries:\n",
    "    italian = gettext.translation('iso3166', pycountry.LOCALES_DIR, languages=['it'])\n",
    "    italian.install()\n",
    "    countries_italian.append(_(country.name))\n",
    "\n",
    "nationalities = []\n",
    "for i in range(len(countries_german)):\n",
    "    nationalities.append([countries_german[i], countries_french[i], countries_italian[i]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search for nationalities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adjust list manually to match the court's practice.\n",
    "nationalities = [['unbekannt', 'nationalité indéterminée'], ['staatenlos', 'ohne nationalität', 'ohne Staats', 'sans nationalité'], ['Kosovo'],['Aruba'], ['Afghanistan'], ['Angola', 'Angola', 'Angola'], ['Anguilla'], ['Albanien', 'Albanie', 'Albania'], ['Andorra', 'Andorre'], ['Vereinigte Arabische Emirate', 'Émirats arabes unis', 'Emirats arabes unis', 'Emirati arabi uniti'], ['Argentinien', 'Argentine', 'Argentina'], ['Armenien', 'Arménie', 'Armenia'], ['Amerikanisch-Samoa', 'Samoa américaines', 'Samoa americane'], ['Antarktis', 'Antarctique', 'Antartide'], ['Französische Süd- und Antarktisgebiete', 'Terres australes françaises', 'Territori meridionali francesi'], ['Antigua und Barbuda', 'Antigua-et-Barbuda', 'Antigua e Barbuda'], ['Australien', 'Australie', 'Australia'], ['Österreich', 'Autriche', 'Austria'], ['Aserbaidschan', 'Azerbaïdjan', 'Azerbaigian'], ['Burundi', 'Burundi', 'Burundi'], ['Belgien', 'Belgique', 'Belgio'], ['Benin', 'Bénin', 'Benin'], ['Bonaire, Sint Eustatius und Saba', 'Bonaire, Saint-Eustache et Saba', 'Bonaire, Saint Eustatius e Saba'], ['Burkina Faso', 'Burkina Faso', 'Burkina-Faso'], ['Bangladesch', 'Bangladesh', 'Bangladesh'], ['Bulgarien', 'Bulgarie', 'Bulgaria'], ['Bahrain', 'Bahreïn', 'Bahrein'], ['Bahamas', 'Bahamas', 'Bahamas'], ['Bosnien und Herzegowina', 'Bosnie-Herzégovine', 'Bosnia-Erzegovina'], ['Saint-Barthélemy', 'Saint-Barthélemy', 'Saint-Barths'], ['Belarus', 'Weißrussland', 'Bélarus', 'Bielorussia'], ['Belize', 'Belize', 'Belize'], ['Bermuda', 'Bermudes', 'Bermuda'], ['Bolivien', 'Bolivie', 'Bolivia'], ['Brasilien', 'Brésil', 'Brasile'], ['Barbados', 'Barbade', 'Barbados'], ['Brunei Darussalam', 'Brunéi Darussalam', 'Brunei'], ['Bhutan', 'Bhoutan', 'Bhutan'], ['Bouvet-Insel', 'Bouvet, Île', 'Isola di Bouvet'], ['Botsuana', 'Botswana'], ['Zentralafrikanische Republik', 'Centrafricaine, République', 'Repubblica Centrafricana'], ['Kanada', 'Canada'], ['Kokos-(Keeling-)Inseln', 'Cocos (Keeling), Îles', 'Isole Cocos (Keeling)'], ['Chile', 'Chili', 'Cile'], ['China', 'Chine', 'Cina'], [\"Côte d'Ivoire\", \"Côte d'Ivoire\", \"Costa d'Avorio\"], ['Kamerun', 'Cameroun', 'Camerun'], ['Kongo (beide)', 'Kongo', 'Congo', 'Congo'], ['Cookinseln', 'Cook, Îles', 'Isole Cook'], ['Kolumbien', 'Colombie', 'Colombia'], ['Komoren', 'Comores', 'Comore'], ['Cabo Verde', 'Cabo Verde', 'Cabo Verde'], ['Costa Rica', 'Costa Rica', 'Costa Rica'], ['Kuba', 'Cuba', 'Cuba'], ['Curaçao', 'Curaçao', 'Curaçao'], ['Weihnachtsinseln', 'Christmas, Île', 'Isola Christmas'], ['Cayman-Inseln', 'Caïman, Îles', 'Isole Cayman'], ['Zypern', 'Chypre', 'Cipro'], ['Tschechische Republik','Czechia', 'Czechia', 'Czechia'], ['Deutschland', 'Allemagne', 'Germania'], ['Dschibuti', 'Djibouti', 'Gibuti'], ['Dominica', 'Dominique', 'Dominica'], ['Dänemark', 'Danemark', 'Danimarca'], ['Dominikanische Republik', 'Dominicaine, République', 'Repubblica Dominicana'], ['Algerien', 'Algérie', 'Algeria'], ['Ecuador', 'Équateur', 'Equateur', 'Ecuador'], ['Ägypten', 'Aegypten', 'Égypte', 'Egypte', 'Egitto'], ['Eritrea', 'Érythrée', 'Erythrée', 'Eritrea'], ['Westsahara', 'Sahara Occidental', 'Sahara occidentale'], ['Spanien', 'Espagne', 'Spagna'], ['Estland', 'Estonie', 'Estonia'], ['Äthiopien', 'Éthiopie', 'Ethiopie', 'Etiopia'], ['Finnland', 'Finlande', 'Finlandia'], ['Fidschi', 'Fidji', 'Fiji'], ['Falklandinseln (Malwinen)', 'Falkland, Îles (Malvinas)', 'Isole Falkland (Malvine)'], ['Frankreich', 'France', 'Francia'], ['Färöer-Inseln', 'Féroé, Îles', 'Isole Fær Øer'], ['Mikronesien', 'Micronésie', 'Micronesia'], ['Gabun', 'Gabon', 'Gabon'], ['Vereinigtes Königreich', 'Royaume-Uni', 'Regno Unito'], ['Georgien', 'Géorgie', 'Georgia'], ['Guernsey', 'Guernesey', 'Guernsey'], ['Ghana', 'Ghana', 'Ghana'], ['Gibraltar', 'Gibraltar', 'Gibilterra'], ['Guinea', 'Guinée', 'Guinea'], ['Guadeloupe', 'Guadeloupe', 'Guadalupa'], ['Gambia', 'Gambie', 'Gambia'], ['Guinea-Bissau', 'Guinée-Bissau', 'Guinea-Bissau'], ['Äquatorialguinea', 'Guinée Équatoriale', 'Guinée Equatoriale', 'Guinea equatoriale'], ['Griechenland', 'Grèce', 'Grecia'], ['Grenada', 'Grenade', 'Grenada'], ['Grönland', 'Groënland', 'Groenlandia'], ['Guatemala', 'Guatemala', 'Guatemala'], ['Französisch-Guyana', 'Guyane française', 'Guyana francese'], ['Guam', 'Guam', 'Guam'], ['Guyana', 'Guyana', 'Guyana'], ['Hongkong', 'Hong-Kong', 'Hong Kong'], ['Heard und McDonaldinseln', 'Heard, Île et McDonald, Îles', 'Isole Heard e McDonald'], ['Honduras', 'Honduras', 'Honduras'], ['Kroatien', 'Croatie', 'Croazia'], ['Haiti', 'Haïti', 'Haiti'], ['Ungarn', 'Hongrie', 'Ungheria'], ['Indonesien', 'Indonésie', 'Indonesia'], ['Insel Man', 'Île de Man', 'Isola di Man'], ['Indien', 'Inde', 'India'], ['Britisches Territorium im Indischen Ozean', \"Océan Indien, Territoire britannique de l'\", \"Territorio britannico dell'Oceano Indiano\"], ['Irland', 'Irlande', 'Irlanda'], ['Iran', 'Iran', 'Iran'], ['Irak', 'Irak', 'Iraq'], ['Island', 'Islande', 'Islanda'], ['Israel', 'Israël', 'Israele'], ['Italien', 'Italie', 'Italia'], ['Jamaika', 'Jamaïque', 'Giamaica'], ['Jersey', 'Jersey', 'Jersey'], ['Jordanien', 'Jordanie', 'Giordania'], ['Japan', 'Japon', 'Giappone'], ['Kasachstan', 'Kazakhstan', 'Kazakistan'], ['Kenia', 'Kenya', 'Kenya'], ['Kirgisistan', 'Kirghizistan', 'Kirghizistan'], ['Kambodscha', 'Cambodge', 'Cambogia'], ['Kiribati', 'Kiribati', 'Kiribati'], ['St. Kitts und Nevis', 'Saint-Kitts-et-Nevis', 'Saint Kitts e Nevis'], ['Korea', 'Corée', 'Corea del Sud'], ['Kuwait', 'Koweït', 'Kuwait'], ['Laos', 'Lao', 'Laos'], ['Libanon', 'Liban', 'Libano'], ['Liberia', 'Libéria', 'Liberia'], ['Libyen', 'Libye', 'Libia'], ['St. Lucia', 'Sainte-Lucie', 'Saint Lucia'], ['Liechtenstein', 'Liechtenstein', 'Liechtenstein'], ['Sri Lanka', 'Sri Lanka', 'Sri Lanka'], ['Lesotho', 'Lesotho', 'Lesotho'], ['Litauen', 'Lituanie', 'Lituania'], ['Luxemburg', 'Luxembourg', 'Lussemburgo'], ['Lettland', 'Lettonie', 'Lettonia'], ['Macao', 'Macau', 'Macao'], ['Saint Martin (Französischer Teil)', 'Saint-Martin (partie française)', 'Saint-Martin (Francia)'], ['Marokko', 'Maroc', 'Marocco'], ['Monaco', 'Monaco', 'Monaco'], ['Moldawien', 'Moldova', 'Moldavia'], ['Madagaskar', 'Madagascar', 'Madagascar'], ['Malediven', 'Maldives', 'Maldive'], ['Mexiko', 'Mexique', 'Messico'], ['Marshallinseln', 'Îles Marshall', 'Isole Marshall'], ['Mazedonien', 'Macédoine', 'Macedonia'], ['Mali', 'Mali', 'Mali'], ['Malta', 'Malte', 'Malta'], ['Myanmar', 'Myanmar', 'Myanmar'], ['Montenegro', 'Monténégro', 'Montenegro'], ['Mongolei', 'Mongolie', 'Mongolia'], ['Nördliche Mariana-Inseln', 'Mariannes du Nord, Îles', 'Marianne settentrionali'], ['Mosambik', 'Mozambique', 'Mozambico'], ['Mauretanien', 'Mauritanie', 'Mauritania'], ['Montserrat', 'Montserrat', 'Montserrat'], ['Martinique', 'Martinique', 'Martinica'], ['Mauritius', 'Maurice', 'Maurizio'], ['Malawi', 'Malawi', 'Malawi'], ['Malaysia', 'Malaisie', 'Malaysia'], ['Mayotte', 'Mayotte', 'Mayotte'], ['Namibia', 'Namibie', 'Namibia'], ['Neukaledonien', 'Nouvelle-Calédonie', 'Nuova Caledonia'], ['Nigeria oder Niger','Niger', 'Niger', 'Niger'], ['Norfolkinsel', 'Norfolk, Île', 'Isola Norfolk'], ['Nigeria', 'Nigeria', 'Nigéria', 'Nigeria'], ['Nicaragua', 'Nicaragua', 'Nicaragua'], ['Niue', 'Nioue', 'Niue'], ['Niederlande', 'Pays-Bas', 'Paesi Bassi'], ['Norwegen', 'Norvège', 'Norvegia'], ['Nepal', 'Népal', 'Nepal'], ['Nauru', 'Nauru', 'Nauru'], ['Neuseeland', 'Nouvelle-Zélande', 'Nuova Zelanda'], ['Oman', 'Oman', 'Oman'], ['Pakistan', 'Pakistan', 'Pakistan'], ['Panama', 'Panama', 'Panama'], ['Pitcairn', 'Pitcairn', 'Pitcairn'], ['Peru', 'Pérou', 'Perù'], ['Philippinen', 'Philippines', 'Filippine'], ['Palau', 'Palaos', 'Palau'], ['Papua-Neuguinea', 'Papouasie-Nouvelle-Guinée', 'Papua Nuova Guinea'], ['Polen', 'Pologne', 'Polonia'], ['Puerto Rico', 'Porto Rico', 'Puerto Rico'], ['Demokratische Volksrepublik Korea', 'Nordkorea', 'République populaire démocratique de Corée', 'Corée du Nord', 'Corea del Nord'], ['Portugal', 'Portugal', 'Portogallo'], ['Paraguay', 'Paraguay', 'Paraguay'], ['Palästina', 'Palestine', 'Palestina', \"d'origine palestinienne\", 'palästinensisch'], ['Französisch-Polynesien', 'Polynésie française', 'Polinesia francese'], ['Katar', 'Qatar', 'Qatar'], ['Réunion', 'Réunion, Île de la', 'Riunione'], ['Rumänien', 'Roumanie', 'Romania'], ['Russische Föderation', 'Russland', 'Russie', 'Russia'], ['Ruanda', 'Rwanda'], ['Saudi-Arabien', 'Arabie saoudite', 'Arabia Saudita'], ['Sudan', 'Soudan'], ['Senegal', 'Sénégal'], ['Singapur', 'Singapour', 'Singapore'], ['South Georgia und die Südlichen Sandwichinseln', 'Géorgie du Sud et les îles Sandwich du Sud', 'Georgia del Sud e Sandwich australi'], ['St. Helena, Ascension und Tristan da Cunha', 'Sainte-Hélène, Ascension et Tristan da Cunha', 'Saint Helena, Ascension e Tristan da Cunha'], ['Svalbard und Jan Mayen', 'Svalbard et île Jan Mayen', 'Svalbard e Jan Mayen'], ['Salomoninseln', 'Salomon, Îles', 'Isole Salomone'], ['Sierra Leone', 'Sierra Leone', 'Sierra Leone'], ['El Salvador', 'El Salvador', 'El Salvador'], ['San Marino', 'San Marin', 'San Marino'], ['Somalia', 'Somalie', 'Somalia'], ['St. Pierre und Miquelon', 'Saint-Pierre-et-Miquelon', 'Saint-Pierre e Miquelon'], ['Serbien', 'Serbie', 'Serbia'], ['Südsudan', 'Soudan du Sud', 'Sudan meridionale'], ['São Tomé und Príncipe', 'Sao Tomé-et-Principe', 'São Tomé e Príncipe'], ['Suriname', 'Surinam', 'Suriname'], ['Slowakei', 'Slovaquie', 'Slovacchia'], ['Slowenien', 'Slovénie', 'Slovenia'], ['Schweden', 'Suède', 'Svezia'], ['Swasiland', 'Swaziland', 'Swaziland'], ['Saint-Martin (Niederländischer Teil)', 'Saint-Martin (partie néerlandaise)', 'Sint Maarten (parte olandese)'], ['Seychellen', 'Seychelles', 'Seicelle'], ['Syrien', 'syrischer Herkunft', 'Syrie', 'Siria'], ['Turks- und Caicosinseln', 'Turks et Caïques, Îles', 'Isole Turks e Caicos'], ['Tschad', 'Tchad', 'Ciad'], ['Togo', 'Togo', 'Togo'], ['Thailand', 'Thaïlande', 'Thailandia'], ['Tadschikistan', 'Tadjikistan', 'Tagikistan'], ['Tokelau', 'Tokelau', 'Tokelau'], ['Turkmenistan', 'Turkménistan', 'Turkmenistan'], ['Timor-Leste', 'Timor orientale'], ['Tonga'], ['Trinidad'], ['Tunesien', 'Tunisie', 'Tunisia'], ['Türkei', 'Turquie', 'Turchia'], ['Tuvalu', 'Tuvalu', 'Tuvalu'], ['Taiwan, Chinesische Provinz', 'Taïwan, province de Chine', 'Taiwan'], ['Tansania', 'Tanzanie', 'Tanzania'], ['Uganda', 'Ouganda', 'Uganda'], ['Ukraine', 'Ukraine', 'Ucraina'], ['United States Minor Outlying Islands', \"Îles mineures éloignées des États-Unis d'Amérique\", 'Isole minori statunitensi'], ['Uruguay', 'Uruguay', 'Uruguay'], ['Vereinigte Staaten', 'États-Unis', 'Stati Uniti'], ['Usbekistan', 'Ouzbékistan', 'Uzbekistan'], ['Heiliger Stuhl (Staat Vatikanstadt)', 'Saint-Siège (état de la cité du Vatican)', 'Santa Sede (Stato della Città del Vaticano)'], ['St. Vincent und die Grenadinen', 'Saint-Vincent-et-les Grenadines', 'Saint Vincent e Grenadine'], ['Venezuela', 'Vénézuela', 'Venezuela'], ['Britische Jungferninseln', 'Îles Vierges britanniques', 'Isole Vergini britanniche'], ['Amerikanische Jungferninseln', 'Îles Vierges des États-Unis', 'Isole Vergini, U.S.'], ['Vietnam', 'Viet Nam', 'Vietnam'], ['Vanuatu', 'Vanuatu', 'Vanuatu'], ['Wallis und Futuna', 'Wallis et Futuna', 'Wallis e Futuna'], ['Samoa', 'Samoa', 'Samoa'], ['Jemen', 'Yémen', 'Yemen'], ['Südafrika', 'Afrique du Sud', 'Sud Africa'], ['Sambia', 'Zambie', 'Zambia'], ['Simbabwe', 'Zimbabwe', 'Zimbabwe']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nationality(head):\n",
    "    nationality = 'not found'\n",
    "    for i in range(0, len(nationalities)):   \n",
    "        for item in nationalities[i]:\n",
    "            if item.lower() in head.lower():\n",
    "                nationality=nationalities[i][0]\n",
    "    return nationality"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Find gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sex(ruling):\n",
    "    sex = 'unknown'\n",
    "    claimant = group_claimant = []\n",
    "    \n",
    "    # German\n",
    "    claimant = re.findall(r'beschwerdeführer[i]{0,1}[n]{0,1}', ruling.lower())\n",
    "    if len(claimant) > 0:\n",
    "        sex = 'M'\n",
    "        if 'die beschwerdeführer' in claimant:\n",
    "            sex = 'Group'\n",
    "        if 'beschwerdeführerin' in claimant:\n",
    "            sex = 'F'\n",
    "            \n",
    "    if 'die beschwerdeführenden' in ruling:\n",
    "        sex = 'Group'\n",
    "        \n",
    "    group_claimant = re.findall(r'die beschwerdeführer\\s{1}', ruling.lower())\n",
    "    \n",
    "    if len(group_claimant) > 0:\n",
    "        sex = 'Group'\n",
    "    \n",
    "    # French\n",
    "    if 'la recourante' in ruling:\n",
    "        sex = 'F'\n",
    "    elif 'le recourant' in ruling:\n",
    "        sex = 'M'\n",
    "    elif 'les recourants' in ruling:\n",
    "        sex = 'Group'\n",
    "    \n",
    "    # Italian\n",
    "    if \"nato\" in head:\n",
    "        sex = 'M'\n",
    "    elif \"nata\" in head:\n",
    "        sex = 'F'\n",
    "    elif ('nata' in head) & ('nato' in head):\n",
    "        sex = 'Group'\n",
    "    # Problem: families consisting only of women or men. People without birthdate\n",
    "    \n",
    "    \n",
    "    return sex"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Final verdict in the case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def verdict_italian(ruling):\n",
    "    try:\n",
    "        verdict = re.findall(r'Tribunale amministrativo federale pronuncia\\s*:\\s*1.([^.]*)', ruling)\n",
    "        verdict = verdict[:150]\n",
    "        if ('respinto' in verdict) | ('inammissibile' in verdict):\n",
    "            verdict = 'rejected'\n",
    "        elif 'accolto' in verdict:\n",
    "            verdict = 'granted'\n",
    "        return verdict\n",
    "    except:\n",
    "        None\n",
    "\n",
    "def verdict_french(ruling):\n",
    "    try:\n",
    "        verdict = re.findall(r'le Tribunal administratif fédéral prononce\\s*:\\s*1.([^.]*)', ruling)\n",
    "        verdict = verdict[0][:150]\n",
    "        if ('rejeté' in verdict)|('écarté'):\n",
    "            verdict = 'rejected'\n",
    "        elif 'admis' in verdict:\n",
    "            verdict = 'granted'\n",
    "        #elif \n",
    "        return verdict\n",
    "    except:\n",
    "        None\n",
    "\n",
    "def verdict_german(ruling):\n",
    "    try:\n",
    "        verdict = re.findall(r'erkennt das Bundesverwaltungsgericht\\s*:\\s*1.([^.]*)', ruling)\n",
    "        verdict = verdict[0][:150]\n",
    "        if ('abgewiesen' in verdict):\n",
    "            verdict = 'rejected'\n",
    "        elif 'gutgeheissen' in verdict:\n",
    "            verdict = 'granted'\n",
    "        return verdict\n",
    "    except:\n",
    "        None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper function: read case file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_ruling(file):\n",
    "    ruling_file = open('txtfiles/' + str(file) + '.txt', 'r')\n",
    "    ruling = ruling_file.read().replace('-\\n', '').replace('\\n', ' ')\n",
    "    ruling_file.close()\n",
    "    return ruling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper function: read statutes file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_statutes(file):\n",
    "    try:\n",
    "        federal_statutes_file = open('/Users/philiphanke/Dropbox/pythonprojects/bvger/swiss-asylum-judges/txtfiles/2017/' + str(file) + 'statutes.txt', 'r')\n",
    "        federal_statutes = federal_statutes_file.read()\n",
    "        federal_statutes_file.close()\n",
    "    except:\n",
    "        federal_statutes='NaN \\n NaN'\n",
    "    return federal_statutes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading the cases, applying helper function, build DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "500\n",
      "1000\n",
      "1500\n",
      "2000\n",
      "2500\n",
      "3000\n",
      "Done\n",
      "\u0007\u0007\u0007\n"
     ]
    }
   ],
   "source": [
    "df_deutsch = pd.DataFrame([['']], columns=['ruling'])\n",
    "df_francais = pd.DataFrame([['']], columns=['ruling'])\n",
    "df_italiano = pd.DataFrame([['']], columns=['ruling'])\n",
    "\n",
    "for file in range(0, 3256):\n",
    "    ruling = read_ruling(file)\n",
    "    federal_statutes = read_statutes(file)\n",
    "    \n",
    "    head = ruling[0:700]\n",
    "    \n",
    "    # Generate temporary DataFrame to be appended to the main DataFrame\n",
    "    df_temp = pd.DataFrame([[file, find_case_number(head), to_datetime(head), nationality(head), head, sex(ruling)]], columns=['filenumber', 'casenumber', 'date', 'nationality', 'head', 'sex'])\n",
    "\n",
    "\n",
    "    # Takes the statutes file, turns it into a list split by line break, and then generates a dictionary. Odd lines are the name of the law, even lines are the cited paragraphs.\n",
    "    statutes_list = federal_statutes.split('\\n')\n",
    "    statutes_dict = {}\n",
    "    for item in range(len(statutes_list)):\n",
    "        if item % 2 == 0:\n",
    "            statutes_dict[statutes_list[item]] = statutes_list[item + 1]\n",
    "    \n",
    "        \n",
    "    # Adds a column for every law and takes the cited paragraphs as observation.\n",
    "    for key in statutes_dict:\n",
    "        df_temp[key[0:-1]] = statutes_dict[key]\n",
    "   \n",
    "    if 'Urteil' in ruling:\n",
    "        df_temp['verdict'] = verdict_german(ruling)\n",
    "        df_deutsch = df_deutsch.append(df_temp)\n",
    "    if 'Arrêt' in ruling:\n",
    "        df_temp['verdict'] = verdict_french(ruling)\n",
    "        df_francais = df_francais.append(df_temp)\n",
    "    if 'Sentenza' in ruling:\n",
    "        df_temp['verdict'] = verdict_italian(ruling)\n",
    "        df_italiano = df_italiano.append(df_temp)\n",
    "    #print(file)\n",
    "    \n",
    "    if file % 500 == 0:\n",
    "        print(file)\n",
    "\n",
    "print('Done')\n",
    "print('\\a'*3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0.105</th>\n",
       "      <th>0.276.191.361</th>\n",
       "      <th>0.312.1</th>\n",
       "      <th>0.351.913.61</th>\n",
       "      <th>0.632.312.141.1</th>\n",
       "      <th>0.632.316.491.1</th>\n",
       "      <th>0.632.317.411.23</th>\n",
       "      <th>0.748.131.913.6</th>\n",
       "      <th>744.171</th>\n",
       "      <th>AHVG</th>\n",
       "      <th>...</th>\n",
       "      <th>ZGB</th>\n",
       "      <th>ZStV</th>\n",
       "      <th>casenumber</th>\n",
       "      <th>date</th>\n",
       "      <th>filenumber</th>\n",
       "      <th>head</th>\n",
       "      <th>nationality</th>\n",
       "      <th>ruling</th>\n",
       "      <th>sex</th>\n",
       "      <th>verdict</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Art.3,</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[E-6164/2014]</td>\n",
       "      <td>2017-01-03 00:00:00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Bundesverwaltungsgericht Tribunal administra...</td>\n",
       "      <td>Ägypten</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[D-8013/2016]</td>\n",
       "      <td>2017-01-03 00:00:00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Bundesverwaltungsgericht Tribunal administra...</td>\n",
       "      <td>Somalia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[D-7853/2016]</td>\n",
       "      <td>2017-01-03 00:00:00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Bundesverwaltungsgericht Tribunal administra...</td>\n",
       "      <td>Türkei</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Art.3,</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[D-6367/2015]</td>\n",
       "      <td>2017-01-03 00:00:00</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Bundesverwaltungsgericht Tribunal administra...</td>\n",
       "      <td>Usbekistan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0.105 0.276.191.361 0.312.1 0.351.913.61 0.632.312.141.1 0.632.316.491.1  \\\n",
       "0     NaN           NaN     NaN          NaN             NaN             NaN   \n",
       "0  Art.3,           NaN     NaN          NaN             NaN             NaN   \n",
       "0     NaN           NaN     NaN          NaN             NaN             NaN   \n",
       "0     NaN           NaN     NaN          NaN             NaN             NaN   \n",
       "0  Art.3,           NaN     NaN          NaN             NaN             NaN   \n",
       "\n",
       "  0.632.317.411.23 0.748.131.913.6 744.171 AHVG    ...     ZGB ZStV  \\\n",
       "0              NaN             NaN     NaN  NaN    ...     NaN  NaN   \n",
       "0              NaN             NaN     NaN  NaN    ...     NaN  NaN   \n",
       "0              NaN             NaN     NaN  NaN    ...     NaN  NaN   \n",
       "0              NaN             NaN     NaN  NaN    ...     NaN  NaN   \n",
       "0              NaN             NaN     NaN  NaN    ...     NaN  NaN   \n",
       "\n",
       "      casenumber                 date filenumber  \\\n",
       "0            NaN                  NaT        NaN   \n",
       "0  [E-6164/2014]  2017-01-03 00:00:00        0.0   \n",
       "0  [D-8013/2016]  2017-01-03 00:00:00        1.0   \n",
       "0  [D-7853/2016]  2017-01-03 00:00:00        2.0   \n",
       "0  [D-6367/2015]  2017-01-03 00:00:00        5.0   \n",
       "\n",
       "                                                head nationality ruling  sex  \\\n",
       "0                                                NaN         NaN         NaN   \n",
       "0    Bundesverwaltungsgericht Tribunal administra...     Ägypten    NaN    F   \n",
       "0    Bundesverwaltungsgericht Tribunal administra...     Somalia    NaN    F   \n",
       "0    Bundesverwaltungsgericht Tribunal administra...      Türkei    NaN    M   \n",
       "0    Bundesverwaltungsgericht Tribunal administra...  Usbekistan    NaN    F   \n",
       "\n",
       "    verdict  \n",
       "0       NaN  \n",
       "0  rejected  \n",
       "0  rejected  \n",
       "0  rejected  \n",
       "0  rejected  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_deutsch.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}