PII_Algorithm2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Algorithm: Making Determinations\n",
    "\n",
    "The next step, now that we can identify relevant records in each supporting dataset, is to use that information to make the PII redaction decision for a particular card. This will follow the procedure we outlined on the same flowchart as before. This is copied below for your reference.\n",
    "\n",
    "<img src=\"Pics/flowchart.png\" />\n",
    "\n",
    "The outer loop is there to indicate that we will process all of the 250 incident cards for PII. However, we can start by writing a Python function that makes that determination for just one card."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# reading in all of the data\n",
    "data_Card = pd.read_csv( \"Datasets/Cards_Box9.csv\" )\n",
    "data_Form26 = pd.read_csv(\"Datasets/WRAForm26.csv\")\n",
    "data_FAR = pd.read_csv(\"Datasets/TuleLake_FAR_ALL_FINAL4.csv\")\n",
    "\n",
    "# indexing the supporting datasets\n",
    "g_FAR = data_FAR.groupby(data_FAR['LastName'].str.lower())\n",
    "g_dataForm26 = data_Form26.groupby(data_Form26['LastName'].str.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lookup_form26(last, first, other):  # note the additional \"other\" parameter\n",
    "    \"\"\"Looks up a name in Form 26, starting with the last name index.\"\"\"\n",
    "    last = last.lower()\n",
    "    if g_dataForm26.groups.get(last) is None:\n",
    "        return None  # No match on last name\n",
    "    for i in list(g_dataForm26.groups.get(last)):\n",
    "        val = data_Form26.values[i]\n",
    "        if first is not None and pd.notna(val[1]):\n",
    "            if first.lower() == val[1].lower():\n",
    "                return val  # if both names match, return this record\n",
    "        elif other is not None and pd.notna(val[1]):\n",
    "            if other.lower() == val[1].lower():\n",
    "                return val  # card other name matches form 26 first name\n",
    "    return None  # if we reach the end of the family group w/o finding a matching value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lookup_far(last, first, other):\n",
    "    \"\"\"Looks up a name in FAR, starting with the last name index.\"\"\"\n",
    "    last = last.lower()\n",
    "    if g_FAR.groups.get(last) is None:\n",
    "        return None  # No match on last name\n",
    "    for i in list(g_FAR.groups.get(last)):\n",
    "        val = data_FAR.values[i]\n",
    "        if first is not None and pd.notna(val[2]):\n",
    "            if first.lower() == val[2].lower():\n",
    "                return val  # if both names match, return this record\n",
    "        elif other is not None and pd.notna(val[3]):\n",
    "            if other.lower() == val[3].lower():\n",
    "                return val  # card other name matches form 26 first name\n",
    "    return None  # if we reach the end of the family group w/o finding a matching value."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime, timedelta as delta\n",
    "\n",
    "age_of_majority = 18\n",
    "weeks_per_year = 52.1429\n",
    "\n",
    "\n",
    "def reasonsToRedact(index):  # i is the integer index of the card requested\n",
    "    \"\"\"Determines if a particular card, indicated by it's data_Card index, \n",
    "    is releasable. It returns a list of reasons the data cannot be released\n",
    "    or an empty list if it may be released.\"\"\"\n",
    "    result = []  # this is where we will add redaction reasons\n",
    "    \n",
    "    # check the not inmate column first, as these row don't have names\n",
    "    if pd.notna( data_Card.iloc[ index, 1 ] ):  \n",
    "        # if nan, that indicates not an incarceree\n",
    "        result.append('Not an incarceree')\n",
    "        return result\n",
    "    \n",
    "    # first we read some card data into local variables\n",
    "    # we have to make sure to replace NaN with None\n",
    "    last_name = data_Card.iloc[ index, 2 ]\n",
    "    last_name = None if pd.isna(last_name) else last_name\n",
    "    first_name = data_Card.iloc[ index, 3 ]\n",
    "    first_name = None if pd.isna(first_name) else first_name\n",
    "    other_name = data_Card.iloc[ index, 4 ]\n",
    "    other_name = None if pd.isna(other_name) else other_name\n",
    "    incident_date_str = data_Card.iloc[ index, 5 ]\n",
    "    incident_date = datetime.strptime(incident_date_str,'%m/%d/%y')\n",
    "    incident_date = incident_date.replace(year=incident_date.year - 100)  # b/c Python assumes 2-digit years are 20XX\n",
    "    incident_year = int(data_Card.iloc[ index, 6 ])\n",
    "    \n",
    "    far_match = lookup_far(last_name, first_name, other_name)\n",
    "    if far_match is not None:\n",
    "        birth_date_str = far_match[4]\n",
    "        birth_date = datetime.strptime(birth_date_str,'%m/%d/%Y')\n",
    "        weeks_until_majority = (age_of_majority + 1) * weeks_per_year\n",
    "        majority_date = birth_date + delta(weeks=weeks_until_majority)\n",
    "        if majority_date > incident_date:\n",
    "            result.append(\"FAR birthdate indicates still a minor\")\n",
    "    f26_match = lookup_form26(last_name, first_name, other_name)\n",
    "    if f26_match is not None:\n",
    "        birth_year = f26_match[2]\n",
    "        majority_year = birth_year + 19\n",
    "        if majority_year > incident_year:\n",
    "            result.append(\"Form 26 indicates still a minor\")\n",
    "    if far_match is None and f26_match is None:\n",
    "        result.append('No supporting data found')\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Box9-0692.jpg' nan 'Ebesu' 'Kikumatsu' nan '7/24/42' 1942]\n",
      "['No supporting data found']\n",
      "\n",
      "\n",
      "['Box9-0642.jpg' nan 'Doi' 'Satomi' nan '8/6/42' 1942]\n",
      "[]\n",
      "\n",
      "\n",
      "['Box9-0765.jpg' nan 'Endo' nan 'Herbert' '8/25/42' 1942]\n",
      "['FAR birthdate indicates still a minor', 'Form 26 indicates still a minor']\n",
      "\n",
      "\n",
      "['Box9-0632.jpg' nan 'Doi' 'Kanjiro' nan '10/3/42' 1942]\n",
      "[]\n",
      "\n",
      "\n",
      "['Box9-0196.jpg' 'Y' nan nan nan '3/11/44' 1944]\n",
      "['Not an incarceree']\n"
     ]
    }
   ],
   "source": [
    "# let's do some testing\n",
    "print(data_Card.values[1])\n",
    "print(reasonsToRedact(1))\n",
    "print('\\n')\n",
    "print(data_Card.values[2])\n",
    "print(reasonsToRedact(2))\n",
    "print('\\n')\n",
    "print(data_Card.values[3])\n",
    "print(reasonsToRedact(3))\n",
    "print('\\n')\n",
    "print(data_Card.values[6])\n",
    "print(reasonsToRedact(6))\n",
    "print('\\n')\n",
    "print(data_Card.values[112])\n",
    "print(reasonsToRedact(112))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing for a Different Age\n",
    "\n",
    "This box 9 incident card dataset has already been redacted, which is why we can share it with you. So\n",
    "how can we know that the algorithm works well and how many determinations it can make?\n",
    "\n",
    "Since we made the age_of_majority a variable, we can change it just for testing purposes.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Box9-1053.jpg' nan 'Fujii' 'Yasuko' nan '3/7/44' 1944]\n",
      "['FAR birthdate indicates still a minor']\n"
     ]
    }
   ],
   "source": [
    "age_of_majority = 75\n",
    "\n",
    "print(data_Card.values[111])\n",
    "print(reasonsToRedact(111))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE**: You will have to change the variable back to 18 before the code will run normally.\n",
    "\n",
    "Let's go ahead and do that and then make sure that all of the PII has already been redacted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Box9-0692.jpg' nan 'Ebesu' 'Kikumatsu' nan '7/24/42' 1942] for these reasons ['No supporting data found']\n",
      "['Box9-0765.jpg' nan 'Endo' nan 'Herbert' '8/25/42' 1942] for these reasons ['FAR birthdate indicates still a minor', 'Form 26 indicates still a minor']\n",
      "['Box9-0780.jpg' nan 'Enjoki' nan 'George' '1/10/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0015.jpg' nan 'Tujii' 'Yoshio' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0168.jpg' nan 'Antoku' 'Teru' 'Charles' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0169.jpg' nan 'Antoku' nan 'Charles' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0201.jpg' nan 'CHI-no-maki' nan nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0509.jpg' nan 'Ayoama' 'Dan' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0641.jpg' nan 'Doi' 'Noburo' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0729.jpg' nan 'Kitagawa' 'Eichi' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0759.jpg' nan 'Nogawa' 'Ematsu' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0820.jpg' nan 'Yamamoto' 'Euchi ?' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0951.jpg' nan 'Sugimoto' 'Frank' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0956.jpg' nan 'Kawaii' nan 'Frank' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0959.jog' nan 'Miyake' 'S.' 'Frank' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0964.jpg' nan 'Takashita' nan 'Frank' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0965.jpg' nan 'Terakami' 'M.' 'Frank' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0972.jpg' nan 'Kakimoto' 'Masa' 'Freddie' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0977.jpg' nan 'Mori' 'H.' 'Fred' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0978.jpg' nan 'Mori' 'H.' 'Fred' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0979.jpg' nan 'Mori' 'H.' 'Fred' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0981.jpg' nan 'Morita' 'O.' 'Fred' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0985.jpg' nan 'Yamasaki' 'Saku' 'Fred' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-1024.jpg' nan 'Fujii' 'Y.' 'Fred' '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-1076.jpg' nan 'Fujimoto' 'Akiro' nan '11/4/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0980.jpg' nan 'Mori' 'H.' 'Fred' '11/13/43' 1943] for these reasons ['No supporting data found']\n",
      "['Box9-0634.jpg' nan 'Doi' 'Kasumi' nan '2/11/44' 1944] for these reasons ['No supporting data found']\n",
      "['Box9-1015.jpg' nan 'Fujihiro' 'Mieko' nan '3/7/44' 1944] for these reasons ['No supporting data found']\n",
      "28\n"
     ]
    }
   ],
   "source": [
    "age_of_majority = 18\n",
    "\n",
    "count = 0\n",
    "for i in range(0, 113):\n",
    "    reasons = reasonsToRedact(i)\n",
    "    if(len(reasons) > 0):  # should be redacted!\n",
    "        if 'Not an incarceree' in reasons:\n",
    "            # oops nevermind, it was a staffer..\n",
    "            continue\n",
    "        else:\n",
    "            count = count + 1\n",
    "            print(str(data_Card.values[i]) +\" for these reasons \" + str(reasons))\n",
    "print(count)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}