preparation/3_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dynamic testing: AAA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### prepare results AAA Files, with gold standard and with result.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/prl222/OneDrive-University/multidisciplinary-data-challenge/preparation/Adversifier/datathon_results\n",
      "['team_1', 'team_2']\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "DATA_PATH = os.path.join(os.getcwd(), 'Adversifier', 'datathon_results')\n",
    "print(DATA_PATH)\n",
    "n = 2\n",
    "teams = [f'team_{i}' for i in range(1,n+1)]\n",
    "\n",
    "test_names = ['quoting_a_to_n', 'flip_n_to_a', 'corr_a_to_a', 'corr_n_to_n']\n",
    "other = ['f1_o', 'hashtag_check']\n",
    "\n",
    "print(teams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copy submissions\n",
    "os.mkdir(os.path.join(os.getcwd(), 'Adversifier', 'datathon_results', 'predictions'))\n",
    "\n",
    "# Submissions with answers are exported to 'answers' folder\n",
    "os.mkdir(os.path.join(os.getcwd(), 'Adversifier', 'datathon_results', 'answers'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for team in teams:\n",
    "    if not os.path.exists(os.path.join(DATA_PATH, 'answers', team)):\n",
    "        os.mkdir(os.path.join(DATA_PATH, 'answers', team))\n",
    "    for test in test_names+other:\n",
    "        res = pd.read_csv(os.path.join(DATA_PATH, 'predictions', team, test+'.tsv'), \n",
    "                        sep='\\t', \n",
    "                        names=['text', 'pred'])\n",
    "        f = pd.read_csv(os.path.join(os.getcwd(), 'Adversifier', 'mhs', 'aaa_files', test+'.tsv'), \n",
    "                        sep='\\t', \n",
    "                        names=['text', 'gtruth'])\n",
    "        f['pred'] = res['pred']\n",
    "        f.to_csv(os.path.join(DATA_PATH, 'answers', team, test+'.tsv'),  \n",
    "                 sep ='\\t', index=False, header=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### evaluate answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/prl222/OneDrive-University/multidisciplinary-data-challenge/preparation/Adversifier\r\n"
     ]
    }
   ],
   "source": [
    "# Open docker, to execute terminal eval.py on each folder\n",
    "os.chdir('Adversifier')\n",
    "!pwd\n",
    "!chmod 777 ../eval_variables.sh\n",
    "!../eval_variables.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# script runs command to generate results.tsv inside each team folder\n",
    "#!docker run --mount type=bind,source=$ANSWER_FILE_DIR,target=/aaa/output/answer_files aaa python3 eval.py --dataset_name $DATASET_NAME"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### display table results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1_o</th>\n",
       "      <th>quoting_a_to_n</th>\n",
       "      <th>corr_n_to_n</th>\n",
       "      <th>flip_n_to_a</th>\n",
       "      <th>corr_a_to_a</th>\n",
       "      <th>aaa</th>\n",
       "      <th>hashtag_check</th>\n",
       "      <th>team</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>49.39</td>\n",
       "      <td>49.06</td>\n",
       "      <td>50.34</td>\n",
       "      <td>49.89</td>\n",
       "      <td>49.65</td>\n",
       "      <td>49.73</td>\n",
       "      <td>50.05</td>\n",
       "      <td>team_2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>47.21</td>\n",
       "      <td>100.00</td>\n",
       "      <td>100.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>48.00</td>\n",
       "      <td>team_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    f1_o  quoting_a_to_n  corr_n_to_n  flip_n_to_a  corr_a_to_a    aaa  \\\n",
       "1  49.39           49.06        50.34        49.89        49.65  49.73   \n",
       "0  47.21          100.00       100.00         0.00         0.00   0.00   \n",
       "\n",
       "   hashtag_check    team  \n",
       "1          50.05  team_2  \n",
       "0          48.00  team_1  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For each team, get the AAA\n",
    "results = []\n",
    "for team in teams:\n",
    "    res = pd.read_csv(os.path.join(DATA_PATH, 'answers', team, 'results.tsv'),  sep ='\\t')\n",
    "    res['team'] = team\n",
    "    results.append(res)\n",
    "\n",
    "results = pd.concat(results, axis=0, ignore_index=True)\n",
    "# Sort by AAA column\n",
    "results = results.sort_values(by=['aaa'], ascending=False)\n",
    "results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Static testing: HATECHECK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### include all predictions to table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export hatecheck.csv to answers\n",
    "os.chdir('../')\n",
    "H_PATH = 'hatecheck-experiments/Data/Test Suite'\n",
    "hatecheck_df = pd.read_csv(os.path.join(H_PATH,'hatecheck_final_ACL.csv'), index_col=0)\n",
    "\n",
    "# load model results\n",
    "for team in teams: \n",
    "    res = pd.read_csv(os.path.join(DATA_PATH, 'predictions', team, 'static.tsv'), \n",
    "                      sep='\\t', \n",
    "                      names=['text', 'pred'])\n",
    "    res['pred'].replace(to_replace={1:'hateful', 0: 'non-hateful'}, inplace=True)\n",
    "    hatecheck_df[f'pred_{team}'] = res['pred']\n",
    "\n",
    "hatecheck_df.to_csv(os.path.join(DATA_PATH, 'answers','hatecheck.csv'))\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>functionality</th>\n",
       "      <th>case_id</th>\n",
       "      <th>test_case</th>\n",
       "      <th>label_gold</th>\n",
       "      <th>target_ident</th>\n",
       "      <th>direction</th>\n",
       "      <th>focus_words</th>\n",
       "      <th>focus_lemma</th>\n",
       "      <th>ref_case_id</th>\n",
       "      <th>ref_templ_id</th>\n",
       "      <th>templ_id</th>\n",
       "      <th>case_templ</th>\n",
       "      <th>pred_team_1</th>\n",
       "      <th>pred_team_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>derog_neg_emote_h</td>\n",
       "      <td>1</td>\n",
       "      <td>I hate women.</td>\n",
       "      <td>hateful</td>\n",
       "      <td>women</td>\n",
       "      <td>general</td>\n",
       "      <td>hate</td>\n",
       "      <td>hate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>I hate [IDENTITY_P].</td>\n",
       "      <td>non-hateful</td>\n",
       "      <td>hateful</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>derog_neg_emote_h</td>\n",
       "      <td>2</td>\n",
       "      <td>I hate trans people.</td>\n",
       "      <td>hateful</td>\n",
       "      <td>trans people</td>\n",
       "      <td>general</td>\n",
       "      <td>hate</td>\n",
       "      <td>hate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>I hate [IDENTITY_P].</td>\n",
       "      <td>hateful</td>\n",
       "      <td>hateful</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>derog_neg_emote_h</td>\n",
       "      <td>3</td>\n",
       "      <td>I hate gay people.</td>\n",
       "      <td>hateful</td>\n",
       "      <td>gay people</td>\n",
       "      <td>general</td>\n",
       "      <td>hate</td>\n",
       "      <td>hate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>I hate [IDENTITY_P].</td>\n",
       "      <td>hateful</td>\n",
       "      <td>non-hateful</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>derog_neg_emote_h</td>\n",
       "      <td>4</td>\n",
       "      <td>I hate black people.</td>\n",
       "      <td>hateful</td>\n",
       "      <td>black people</td>\n",
       "      <td>general</td>\n",
       "      <td>hate</td>\n",
       "      <td>hate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>I hate [IDENTITY_P].</td>\n",
       "      <td>non-hateful</td>\n",
       "      <td>hateful</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>derog_neg_emote_h</td>\n",
       "      <td>5</td>\n",
       "      <td>I hate disabled people.</td>\n",
       "      <td>hateful</td>\n",
       "      <td>disabled people</td>\n",
       "      <td>general</td>\n",
       "      <td>hate</td>\n",
       "      <td>hate</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>I hate [IDENTITY_P].</td>\n",
       "      <td>hateful</td>\n",
       "      <td>hateful</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       functionality  case_id                 test_case label_gold  \\\n",
       "0  derog_neg_emote_h        1            I hate women.     hateful   \n",
       "1  derog_neg_emote_h        2     I hate trans people.     hateful   \n",
       "2  derog_neg_emote_h        3       I hate gay people.     hateful   \n",
       "3  derog_neg_emote_h        4     I hate black people.     hateful   \n",
       "4  derog_neg_emote_h        5  I hate disabled people.     hateful   \n",
       "\n",
       "      target_ident direction focus_words focus_lemma  ref_case_id  \\\n",
       "0            women   general        hate        hate          NaN   \n",
       "1     trans people   general        hate        hate          NaN   \n",
       "2       gay people   general        hate        hate          NaN   \n",
       "3     black people   general        hate        hate          NaN   \n",
       "4  disabled people   general        hate        hate          NaN   \n",
       "\n",
       "   ref_templ_id  templ_id            case_templ  pred_team_1  pred_team_2  \n",
       "0           NaN         1  I hate [IDENTITY_P].  non-hateful      hateful  \n",
       "1           NaN         1  I hate [IDENTITY_P].      hateful      hateful  \n",
       "2           NaN         1  I hate [IDENTITY_P].      hateful  non-hateful  \n",
       "3           NaN         1  I hate [IDENTITY_P].  non-hateful      hateful  \n",
       "4           NaN         1  I hate [IDENTITY_P].      hateful      hateful  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#hatecheck_df = pd.read_csv(os.path.join(DATA_PATH, 'answers','hatecheck.csv'))\n",
    "hatecheck_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>functionality</th>\n",
       "      <th>team_1</th>\n",
       "      <th>team_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>derog_neg_emote_h</td>\n",
       "      <td>50.7%</td>\n",
       "      <td>49.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>derog_neg_attrib_h</td>\n",
       "      <td>46.4%</td>\n",
       "      <td>48.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>derog_dehum_h</td>\n",
       "      <td>53.6%</td>\n",
       "      <td>52.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>derog_impl_h</td>\n",
       "      <td>52.1%</td>\n",
       "      <td>43.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>threat_dir_h</td>\n",
       "      <td>52.6%</td>\n",
       "      <td>51.9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>threat_norm_h</td>\n",
       "      <td>48.6%</td>\n",
       "      <td>54.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>slur_h</td>\n",
       "      <td>50.7%</td>\n",
       "      <td>50.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>slur_homonym_nh</td>\n",
       "      <td>56.7%</td>\n",
       "      <td>60.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>slur_reclaimed_nh</td>\n",
       "      <td>55.6%</td>\n",
       "      <td>51.9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>profanity_h</td>\n",
       "      <td>56.4%</td>\n",
       "      <td>50.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>profanity_nh</td>\n",
       "      <td>47.0%</td>\n",
       "      <td>47.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ref_subs_clause_h</td>\n",
       "      <td>52.9%</td>\n",
       "      <td>45.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ref_subs_sent_h</td>\n",
       "      <td>49.6%</td>\n",
       "      <td>55.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>negate_pos_h</td>\n",
       "      <td>47.1%</td>\n",
       "      <td>58.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>negate_neg_nh</td>\n",
       "      <td>45.9%</td>\n",
       "      <td>52.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>phrase_question_h</td>\n",
       "      <td>44.3%</td>\n",
       "      <td>54.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>phrase_opinion_h</td>\n",
       "      <td>47.4%</td>\n",
       "      <td>43.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ident_neutral_nh</td>\n",
       "      <td>45.2%</td>\n",
       "      <td>50.8%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ident_pos_nh</td>\n",
       "      <td>58.2%</td>\n",
       "      <td>49.7%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>counter_quote_nh</td>\n",
       "      <td>52.0%</td>\n",
       "      <td>54.9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>counter_ref_nh</td>\n",
       "      <td>41.8%</td>\n",
       "      <td>47.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>target_obj_nh</td>\n",
       "      <td>58.5%</td>\n",
       "      <td>41.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>target_indiv_nh</td>\n",
       "      <td>46.2%</td>\n",
       "      <td>43.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>target_group_nh</td>\n",
       "      <td>46.8%</td>\n",
       "      <td>50.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>spell_char_swap_h</td>\n",
       "      <td>53.4%</td>\n",
       "      <td>44.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>spell_char_del_h</td>\n",
       "      <td>58.6%</td>\n",
       "      <td>54.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>spell_space_del_h</td>\n",
       "      <td>47.5%</td>\n",
       "      <td>55.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>spell_space_add_h</td>\n",
       "      <td>53.2%</td>\n",
       "      <td>42.8%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>spell_leet_h</td>\n",
       "      <td>45.1%</td>\n",
       "      <td>48.6%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         functionality team_1 team_2\n",
       "0    derog_neg_emote_h  50.7%  49.3%\n",
       "1   derog_neg_attrib_h  46.4%  48.6%\n",
       "2        derog_dehum_h  53.6%  52.1%\n",
       "3         derog_impl_h  52.1%  43.6%\n",
       "4         threat_dir_h  52.6%  51.9%\n",
       "5        threat_norm_h  48.6%  54.3%\n",
       "6               slur_h  50.7%  50.0%\n",
       "7      slur_homonym_nh  56.7%  60.0%\n",
       "8    slur_reclaimed_nh  55.6%  51.9%\n",
       "9          profanity_h  56.4%  50.7%\n",
       "10        profanity_nh  47.0%  47.0%\n",
       "11   ref_subs_clause_h  52.9%  45.7%\n",
       "12     ref_subs_sent_h  49.6%  55.6%\n",
       "13        negate_pos_h  47.1%  58.6%\n",
       "14       negate_neg_nh  45.9%  52.6%\n",
       "15   phrase_question_h  44.3%  54.3%\n",
       "16    phrase_opinion_h  47.4%  43.6%\n",
       "17    ident_neutral_nh  45.2%  50.8%\n",
       "18        ident_pos_nh  58.2%  49.7%\n",
       "19    counter_quote_nh  52.0%  54.9%\n",
       "20      counter_ref_nh  41.8%  47.5%\n",
       "21       target_obj_nh  58.5%  41.5%\n",
       "22     target_indiv_nh  46.2%  43.1%\n",
       "23     target_group_nh  46.8%  50.0%\n",
       "24   spell_char_swap_h  53.4%  44.4%\n",
       "25    spell_char_del_h  58.6%  54.3%\n",
       "26   spell_space_del_h  47.5%  55.3%\n",
       "27   spell_space_add_h  53.2%  42.8%\n",
       "28        spell_leet_h  45.1%  48.6%"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# compute accuracy\n",
    "func_accuracy_dict = {}\n",
    "for m in teams:\n",
    "    func_accuracy_dict[m] = []\n",
    "    for func in pd.unique(hatecheck_df.functionality):\n",
    "        n_cases = hatecheck_df[hatecheck_df.functionality==func].shape[0]\n",
    "        n_correct = hatecheck_df[(hatecheck_df.functionality==func)&(hatecheck_df['label_gold']==hatecheck_df['pred_{}'.format(m)])].shape[0]\n",
    "        func_accuracy_dict[m].append('{:.1%}'.format(n_correct/n_cases))\n",
    "    \n",
    "    # convert list to series\n",
    "    func_accuracy_dict[m] = pd.Series(func_accuracy_dict[m])\n",
    "    func_accuracy_dict[m].name = m\n",
    "\n",
    "# create df from dict\n",
    "func_accuracy_df = pd.Series(pd.unique(hatecheck_df.functionality))\n",
    "func_accuracy_df.name = 'functionality'\n",
    "\n",
    "for arc_data in func_accuracy_dict:\n",
    "    func_accuracy_df = pd.concat([func_accuracy_df, pd.Series(func_accuracy_dict[arc_data])], axis =1)\n",
    "func_accuracy_df  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "data-challenge",
   "language": "python",
   "name": "data-challenge"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}