SummaC - Additional Experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-10-31 22:47:43,882 [3845] WARNING  datasets.builder:355: [JupyterRequire] Using custom data configuration default\n",
      "2021-10-31 22:47:43,888 [3845] WARNING  datasets.builder:510: [JupyterRequire] Reusing dataset xsum (/home/phillab/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499)\n",
      "2021-10-31 22:47:48,828 [3845] WARNING  datasets.builder:510: [JupyterRequire] Reusing dataset cnn_dailymail (/home/phillab/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/3cb851bf7cf5826e45d49db2863f627cba583cbc32342df7349dfe6c38060234)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        name     N  N_pos  N_neg  frac_pos\n",
      "0  cogensumm   400    312     88  0.780000\n",
      "1  xsumfaith  1250    130   1120  0.104000\n",
      "2   polytope   634     41    593  0.064669\n",
      "3     factcc   503    441     62  0.876740\n",
      "4   summeval   850    770     80  0.905882\n",
      "5      frank  1575    529   1046  0.335873\n"
     ]
    }
   ],
   "source": [
    "import sklearn, torch, numpy as np, json, os, tqdm, pandas as pd, nltk, utils_misc, seaborn as sns, sys, glob\n",
    "sys.path.insert(0, \"/home/phillab/summac/\")\n",
    "from model_summac import SummaCConv, SummaCZS, model_map\n",
    "from utils_summac_benchmark import SummaCBenchmark\n",
    "from utils_scoring import ScorerWrapper\n",
    "import utils_summac_benchmark\n",
    "\n",
    "cm = sns.light_palette(\"green\", as_cmap=True)\n",
    "benchmark = SummaCBenchmark(cut=\"test\")\n",
    "benchmark.print_stats()\n",
    "\n",
    "def path_to_model_info(file_path):\n",
    "    toks = file_path.split(\"/\")\n",
    "    file_name = toks[-1].replace(\".bin\", \"\")\n",
    "    model_type = \"histo\"\n",
    "    model_card, granularity, bins, nli_labels, acc = file_name.split(\"_\")\n",
    "    acc = float(acc.replace(\"bacc\", \"\").replace(\"f1\", \"\"))\n",
    "    return {\"model_type\": model_type, \"model_card\": model_card, \"granularity\": granularity, \"bins\": bins, \"acc\": acc, \"model_path\": file_path, \"nli_labels\": nli_labels}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Table 3: NLI Model Selection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n"
     ]
    }
   ],
   "source": [
    "scorers = []\n",
    "model_keys = list(model_map.keys())#+ [\"decomp\"]\n",
    "# model_keys = [\"decomp\"]\n",
    "\n",
    "for model_key in model_keys:\n",
    "    scorers.append({\"name\": \"ZS-%s\" % (model_key.upper().replace(\"-\", \"_\")), \"model\": SummaCZS(granularity=\"sentence\", model_name=model_key), \"sign\": 1, \"only_doc\": True})\n",
    "    \n",
    "    # Add a histogram based-model\n",
    "    model_files = glob.glob(\"/home/phillab/models/summac/%s_sentence*\" % (model_key))\n",
    "    if len(model_files) == 0:\n",
    "        print(\"No model for [%s] was found\" % (model_key))\n",
    "        continue\n",
    "    best = sorted([path_to_model_info(mf) for mf in model_files], key=lambda m: m[\"acc\"])[-1]\n",
    "    scorers.append({\"name\": \"Histo-%s\" % (model_key.upper().replace(\"-\", \"_\")), \"model\": SummaCConv(bins=best[\"bins\"], nli_labels=best[\"nli_labels\"], models=[model_key], granularity=\"sentence\", start_file=best[\"model_path\"]), \"sign\": 1})\n",
    "\n",
    "scorer_doc = ScorerWrapper(scorers, scoring_method=\"sum\", max_batch_size=20, use_caching=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n",
      "Reusing dataset xsum (/home/phillab/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499)\n",
      "Reusing dataset cnn_dailymail (/home/phillab/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/3cb851bf7cf5826e45d49db2863f627cba583cbc32342df7349dfe6c38060234)\n",
      "  0%|          | 0/400 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= cogensumm ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at microsoft/deberta-base-mnli were not used when initializing DebertaForSequenceClassification: ['config']\n",
      "- This IS expected if you are initializing DebertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at microsoft/deberta-base-mnli were not used when initializing DebertaForSequenceClassification: ['config']\n",
      "- This IS expected if you are initializing DebertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DebertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at roberta-large-mnli were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at roberta-large-mnli were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "100%|██████████| 400/400 [23:02<00:00,  3.46s/it]\n",
      "  0%|          | 0/1250 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= xsumfaith ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1250/1250 [13:06<00:00,  1.59it/s]\n",
      "  0%|          | 0/634 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= polytope ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 634/634 [19:13<00:00,  1.82s/it]\n",
      "  0%|          | 0/503 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= factcc ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 503/503 [08:13<00:00,  1.02it/s]\n",
      "  5%|▍         | 40/850 [00:00<00:02, 325.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= summeval ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 850/850 [00:02<00:00, 383.71it/s]\n",
      "  0%|          | 0/1575 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= frank ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1575/1575 [39:21<00:00,  1.50s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9b\" ><caption>Balanced Accuracy</caption><thead>    <tr>        <th class=\"blank\" ></th>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >score</th>    </tr>    <tr>        <th class=\"index_name level0\" >nli_name</th>        <th class=\"index_name level1\" >model_type</th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row0\" class=\"row_heading level0 row0\" rowspan=2>ANLI</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row0\" class=\"row_heading level1 row0\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow0_col0\" class=\"data row0 col0\" >0.699</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row1\" class=\"row_heading level1 row1\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow1_col0\" class=\"data row1 col0\" >0.717</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row2\" class=\"row_heading level0 row2\" rowspan=2>MNLI</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row2\" class=\"row_heading level1 row2\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow2_col0\" class=\"data row2 col0\" >0.73</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row3\" class=\"row_heading level1 row3\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow3_col0\" class=\"data row3 col0\" >0.709</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row4\" class=\"row_heading level0 row4\" rowspan=2>MNLI_BASE</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row4\" class=\"row_heading level1 row4\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow4_col0\" class=\"data row4 col0\" >0.698</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row5\" class=\"row_heading level1 row5\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow5_col0\" class=\"data row5 col0\" >0.695</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row6\" class=\"row_heading level0 row6\" rowspan=2>SNLI_BASE</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row6\" class=\"row_heading level1 row6\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow6_col0\" class=\"data row6 col0\" >0.64</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row7\" class=\"row_heading level1 row7\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow7_col0\" class=\"data row7 col0\" >0.666</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row8\" class=\"row_heading level0 row8\" rowspan=2>SNLI_LARGE</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row8\" class=\"row_heading level1 row8\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow8_col0\" class=\"data row8 col0\" >0.624</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row9\" class=\"row_heading level1 row9\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow9_col0\" class=\"data row9 col0\" >0.666</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row10\" class=\"row_heading level0 row10\" rowspan=2>VITC</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row10\" class=\"row_heading level1 row10\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow10_col0\" class=\"data row10 col0\" >0.74</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row11\" class=\"row_heading level1 row11\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow11_col0\" class=\"data row11 col0\" >0.721</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row12\" class=\"row_heading level0 row12\" rowspan=2>VITC_BASE</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row12\" class=\"row_heading level1 row12\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow12_col0\" class=\"data row12 col0\" >0.712</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row13\" class=\"row_heading level1 row13\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow13_col0\" class=\"data row13 col0\" >0.679</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel0_row14\" class=\"row_heading level0 row14\" rowspan=2>VITC_ONLY</th>\n",
       "                        <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row14\" class=\"row_heading level1 row14\" >Histo</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow14_col0\" class=\"data row14 col0\" >0.728</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9blevel1_row15\" class=\"row_heading level1 row15\" >ZS</th>\n",
       "                        <td id=\"T_2c022b3a_efd5_11eb_ac7d_47dbd7e43e9brow15_col0\" class=\"data row15 col0\" >0.711</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f71c7b8b910>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "benchmark = SummaCBenchmark(cut=\"test\")\n",
    "\n",
    "results = {}\n",
    "for dataset in benchmark.tasks:\n",
    "    print(\"======= %s ========\" % (dataset[\"name\"]))\n",
    "    datas = dataset[\"task\"]\n",
    "    labels = [d[\"label\"] for d in datas]\n",
    "    utils_summac_benchmark.compute_doc_level(scorer_doc, datas)\n",
    "    \n",
    "    for pred_label in datas[0].keys():\n",
    "        if \"pred_\" not in pred_label or \"total\" in pred_label: continue\n",
    "        balanced_acc = sklearn.metrics.balanced_accuracy_score(labels, [d[pred_label] for d in datas])\n",
    "        model_name, input_type = pred_label.replace(\"pred_\", \"\").split(\"|\")\n",
    "        model_type, nli_name = model_name.split(\"-\")\n",
    "        k = (model_type, nli_name)\n",
    "        if k not in results:\n",
    "            results[k] = []\n",
    "        results[k].append(balanced_acc)\n",
    "\n",
    "cleaned_results = []\n",
    "for (model_type, nli), vs in results.items():\n",
    "    cleaned_results.append({\"nli_name\": nli, \"model_type\": model_type, \"score\": np.mean(vs)})\n",
    "    \n",
    "pd.DataFrame(cleaned_results).groupby([\"nli_name\", \"model_type\"]).agg({\"score\": \"sum\"}).style.set_precision(3).set_caption(\"Balanced Accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for scorer in scorers:\n",
    "        scorer[\"model\"].save_imager_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Table 4: Choice of NLI Category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "scorers = []\n",
    "for model_key in [\"vitc\", \"mnli\", \"anli\"]:\n",
    "    for nli_labels in [\"e\", \"c\", \"n\", \"ec\", \"en\", \"cn\", \"ecn\"]:\n",
    "    \n",
    "        model_files = glob.glob(\"/home/phillab/models/summac/%s_sentence_percentile_%s*\" % (model_key, nli_labels))\n",
    "        if len(model_files) == 0:\n",
    "            print(\"No model for [%s, %s] was found\" % (model_key, nli_labels))\n",
    "            continue\n",
    "        best = sorted([path_to_model_info(mf) for mf in model_files], key=lambda m: m[\"acc\"])[-1]\n",
    "        scorers.append({\"name\": \"Histo-%s-%s\" % (model_key.upper().replace(\"-\", \"_\"), nli_labels), \"model\": SummaCConv(bins=best[\"bins\"], nli_labels=best[\"nli_labels\"], models=[model_key], granularity=\"sentence\", start_file=best[\"model_path\"]), \"sign\": 1})\n",
    "\n",
    "scorer_doc = ScorerWrapper(scorers, max_batch_size=20, use_caching=True)\n",
    "print(\"%d scorers loaded\" % (len(scorers)))\n",
    "\n",
    "benchmark = SummaCBenchmark(cut=\"test\")\n",
    "\n",
    "results = {}\n",
    "for dataset in benchmark.tasks:\n",
    "    print(\"======= %s ========\" % (dataset[\"name\"]))\n",
    "    datas = dataset[\"task\"]\n",
    "    labels = [d[\"label\"] for d in datas]\n",
    "    utils_summac_benchmark.compute_doc_level(scorer_doc, datas)\n",
    "    \n",
    "    for pred_label in datas[0].keys():\n",
    "        if \"pred_\" not in pred_label or \"total\" in pred_label: continue\n",
    "        balanced_acc = sklearn.metrics.balanced_accuracy_score(labels, [d[pred_label] for d in datas])\n",
    "        model_name, input_type = pred_label.replace(\"pred_\", \"\").split(\"|\")\n",
    "        model_type, nli_name, nli_labels = model_name.split(\"-\")\n",
    "        k = (nli_name, nli_labels)\n",
    "        if k not in results:\n",
    "            results[k] = []\n",
    "        results[k].append(balanced_acc)\n",
    "\n",
    "cleaned_results = []\n",
    "for (nli, nli_labels), vs in results.items():\n",
    "    cleaned_results.append({\"nli_name\": nli, \"nli_labels\": nli_labels, \"model_type\": model_type, \"score\": np.mean(vs)})\n",
    "    \n",
    "pd.DataFrame(cleaned_results).groupby([\"nli_name\", \"nli_labels\"]).agg({\"score\": \"sum\"}).style.set_precision(3).set_caption(\"Balanced Accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Table 5: Granularity Selection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "<All keys matched successfully>\n",
      "28 scorers loaded\n"
     ]
    }
   ],
   "source": [
    "scorers = []\n",
    "granularities = [\"document\", \"document-sentence\", \"paragraph-document\", \"paragraph-sentence\", \"2sents-document\",  \"2sents-sentence\", \"sentence-document\", \"sentence\"]\n",
    "\n",
    "for model_key in [\"mnli\", \"vitc\"]:\n",
    "#     for granularity in [\"document\", \"document-sentence\", \"paragraph-document\", \"paragraph-sentence\", \"2sents-sentence\", \"sentence-document\", \"sentence\"]:\n",
    "    for granularity in granularities:\n",
    "#     for granularity in [\"sentence-document\"]:\n",
    "        scorers.append({\"name\": \"ZS-%s-%s\" % (model_key.upper().replace(\"-\", \"_\"), granularity.replace(\"-\", \"_\")), \"model\": SummaCZS(granularity=granularity, model_name=model_key), \"sign\": 1})\n",
    "        if granularity.startswith(\"document\"):\n",
    "            continue\n",
    "        model_files = glob.glob(\"/home/phillab/models/summac/%s_%s*\" % (model_key, granularity))\n",
    "        if len(model_files) == 0:\n",
    "            print(\"No model for [%s, %s] was found\" % (model_key, granularity))\n",
    "            continue\n",
    "        best = sorted([path_to_model_info(mf) for mf in model_files], key=lambda m: m[\"acc\"])[-1]\n",
    "        scorers.append({\"name\": \"Histo-%s-%s\" % (model_key.upper().replace(\"-\", \"_\"), granularity.replace(\"-\", \"_\")), \"model\": SummaCConv(bins=best[\"bins\"], nli_labels=best[\"nli_labels\"], models=[model_key], granularity=granularity, start_file=best[\"model_path\"]), \"sign\": 1})\n",
    "\n",
    "scorer_doc = ScorerWrapper(scorers, max_batch_size=20, use_caching=True)\n",
    "print(\"%d scorers loaded\" % (len(scorers)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 40/400 [00:00<00:01, 304.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= cogensumm ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 400/400 [00:01<00:00, 326.26it/s]\n",
      "  3%|▎         | 40/1250 [00:00<00:03, 326.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= xsumfaith ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1250/1250 [00:02<00:00, 462.43it/s]\n",
      "  6%|▋         | 40/634 [00:00<00:01, 330.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= polytope ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 634/634 [00:01<00:00, 386.73it/s]\n",
      "  8%|▊         | 40/503 [00:00<00:01, 341.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= factcc ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 503/503 [00:01<00:00, 441.78it/s]\n",
      "  5%|▍         | 40/850 [00:00<00:02, 356.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= summeval ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 850/850 [00:02<00:00, 350.46it/s]\n",
      "  4%|▍         | 60/1575 [00:00<00:03, 423.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= frank ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1575/1575 [00:03<00:00, 394.82it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0\" ><caption>Balanced Accuracy</caption><thead>    <tr>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >score</th>    </tr>    <tr>        <th class=\"index_name level0\" >nli_name</th>        <th class=\"index_name level1\" >granularity</th>        <th class=\"index_name level2\" >model_type</th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level0_row0\" class=\"row_heading level0 row0\" rowspan=14>MNLI</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row0\" class=\"row_heading level1 row0\" rowspan=2>2sents_document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row0\" class=\"row_heading level2 row0\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row0_col0\" class=\"data row0 col0\" >0.638</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row1\" class=\"row_heading level2 row1\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row1_col0\" class=\"data row1 col0\" >0.64</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row2\" class=\"row_heading level1 row2\" rowspan=2>2sents_sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row2\" class=\"row_heading level2 row2\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row2_col0\" class=\"data row2 col0\" >0.737</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row3\" class=\"row_heading level2 row3\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row3_col0\" class=\"data row3 col0\" >0.712</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row4\" class=\"row_heading level1 row4\" >document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row4\" class=\"row_heading level2 row4\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row4_col0\" class=\"data row4 col0\" >0.564</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row5\" class=\"row_heading level1 row5\" >document_sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row5\" class=\"row_heading level2 row5\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row5_col0\" class=\"data row5 col0\" >0.574</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row6\" class=\"row_heading level1 row6\" rowspan=2>paragraph_document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row6\" class=\"row_heading level2 row6\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row6_col0\" class=\"data row6 col0\" >0.616</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row7\" class=\"row_heading level2 row7\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row7_col0\" class=\"data row7 col0\" >0.598</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row8\" class=\"row_heading level1 row8\" rowspan=2>paragraph_sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row8\" class=\"row_heading level2 row8\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row8_col0\" class=\"data row8 col0\" >0.647</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row9\" class=\"row_heading level2 row9\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row9_col0\" class=\"data row9 col0\" >0.652</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row10\" class=\"row_heading level1 row10\" rowspan=2>sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row10\" class=\"row_heading level2 row10\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row10_col0\" class=\"data row10 col0\" >0.73</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row11\" class=\"row_heading level2 row11\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row11_col0\" class=\"data row11 col0\" >0.703</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row12\" class=\"row_heading level1 row12\" rowspan=2>sentence_document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row12\" class=\"row_heading level2 row12\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row12_col0\" class=\"data row12 col0\" >0.62</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row13\" class=\"row_heading level2 row13\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row13_col0\" class=\"data row13 col0\" >0.587</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level0_row14\" class=\"row_heading level0 row14\" rowspan=14>VITC</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row14\" class=\"row_heading level1 row14\" rowspan=2>2sents_document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row14\" class=\"row_heading level2 row14\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row14_col0\" class=\"data row14 col0\" >0.713</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row15\" class=\"row_heading level2 row15\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row15_col0\" class=\"data row15 col0\" >0.697</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row16\" class=\"row_heading level1 row16\" rowspan=2>2sents_sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row16\" class=\"row_heading level2 row16\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row16_col0\" class=\"data row16 col0\" >0.741</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row17\" class=\"row_heading level2 row17\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row17_col0\" class=\"data row17 col0\" >0.725</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row18\" class=\"row_heading level1 row18\" >document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row18\" class=\"row_heading level2 row18\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row18_col0\" class=\"data row18 col0\" >0.721</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row19\" class=\"row_heading level1 row19\" >document_sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row19\" class=\"row_heading level2 row19\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row19_col0\" class=\"data row19 col0\" >0.731</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row20\" class=\"row_heading level1 row20\" rowspan=2>paragraph_document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row20\" class=\"row_heading level2 row20\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row20_col0\" class=\"data row20 col0\" >0.712</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row21\" class=\"row_heading level2 row21\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row21_col0\" class=\"data row21 col0\" >0.698</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row22\" class=\"row_heading level1 row22\" rowspan=2>paragraph_sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row22\" class=\"row_heading level2 row22\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row22_col0\" class=\"data row22 col0\" >0.743</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row23\" class=\"row_heading level2 row23\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row23_col0\" class=\"data row23 col0\" >0.726</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row24\" class=\"row_heading level1 row24\" rowspan=2>sentence</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row24\" class=\"row_heading level2 row24\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row24_col0\" class=\"data row24 col0\" >0.74</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row25\" class=\"row_heading level2 row25\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row25_col0\" class=\"data row25 col0\" >0.718</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level1_row26\" class=\"row_heading level1 row26\" rowspan=2>sentence_document</th>\n",
       "                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row26\" class=\"row_heading level2 row26\" >Histo</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row26_col0\" class=\"data row26 col0\" >0.697</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                        <th id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0level2_row27\" class=\"row_heading level2 row27\" >ZS</th>\n",
       "                        <td id=\"T_4d755b48_34e5_11ec_ad19_1989e84ea8b0row27_col0\" class=\"data row27 col0\" >0.684</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f626af4fcd0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = {}\n",
    "for dataset in benchmark.tasks:\n",
    "    print(\"======= %s ========\" % (dataset[\"name\"]))\n",
    "    datas = dataset[\"task\"]\n",
    "    labels = [d[\"label\"] for d in datas]\n",
    "    utils_summac_benchmark.compute_doc_level(scorer_doc, datas)\n",
    "    \n",
    "    # Can be removed if re-running\n",
    "    for scorer in scorers:\n",
    "        scorer[\"model\"].save_imager_cache()\n",
    "    \n",
    "    for pred_label in datas[0].keys():\n",
    "        if \"pred_\" not in pred_label or \"total\" in pred_label: continue\n",
    "        balanced_acc = sklearn.metrics.balanced_accuracy_score(labels, [d[pred_label] for d in datas])\n",
    "        model_name, input_type = pred_label.replace(\"pred_\", \"\").split(\"|\")\n",
    "        \n",
    "        model_type, nli_name, gran = model_name.split(\"-\")\n",
    "        k = (model_type, nli_name, gran)\n",
    "        if k not in results:\n",
    "            results[k] = []\n",
    "        results[k].append(balanced_acc)\n",
    "\n",
    "cleaned_results = []\n",
    "for (model_type, nli, gran), vs in results.items():\n",
    "    cleaned_results.append({\"nli_name\": nli, \"granularity\": gran, \"model_type\": model_type, \"score\": np.mean(vs)})\n",
    "    \n",
    "pd.DataFrame(cleaned_results).groupby([\"nli_name\", \"granularity\", \"model_type\"]).agg({\"score\": \"sum\"}).style.set_precision(3).set_caption(\"Balanced Accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table 6: SummaCZS Operator Choice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9 scorers loaded\n"
     ]
    }
   ],
   "source": [
    "scorers = []\n",
    "for op1 in [\"min\", \"mean\", \"max\"]:\n",
    "    for op2 in [\"min\", \"mean\", \"max\"]:\n",
    "        scorers.append({\"name\": \"ZS-%s-%s\" % (op1, op2), \"model\": SummaCZS(granularity=\"sentence\", model_name=\"vitc\", op1=op1, op2=op2), \"sign\": 1})\n",
    "        \n",
    "scorer_doc = ScorerWrapper(scorers, max_batch_size=20, use_caching=True)\n",
    "print(\"%d scorers loaded\" % (len(scorers)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-07-31 14:15:34,909 [6185] WARNING  datasets.builder:355: [JupyterRequire] Using custom data configuration default\n",
      "2021-07-31 14:15:34,912 [6185] WARNING  datasets.builder:510: [JupyterRequire] Reusing dataset xsum (/home/phillab/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499)\n",
      "2021-07-31 14:15:39,162 [6185] WARNING  datasets.builder:510: [JupyterRequire] Reusing dataset cnn_dailymail (/home/phillab/.cache/huggingface/datasets/cnn_dailymail/3.0.0/3.0.0/3cb851bf7cf5826e45d49db2863f627cba583cbc32342df7349dfe6c38060234)\n",
      "100%|██████████| 400/400 [00:00<00:00, 4027.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= cogensumm ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 38%|███▊      | 480/1250 [00:00<00:00, 4655.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= xsumfaith ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1250/1250 [00:00<00:00, 4622.31it/s]\n",
      "100%|██████████| 634/634 [00:00<00:00, 4340.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= polytope ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 503/503 [00:00<00:00, 4521.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= factcc ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 52%|█████▏    | 440/850 [00:00<00:00, 4396.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= summeval ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 850/850 [00:00<00:00, 3651.48it/s]\n",
      " 30%|███       | 480/1575 [00:00<00:00, 4628.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======= frank ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1575/1575 [00:00<00:00, 4369.67it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2\" ><caption>Balanced Accuracy</caption><thead>    <tr>        <th class=\"blank\" ></th>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >score</th>    </tr>    <tr>        <th class=\"index_name level0\" >op1</th>        <th class=\"index_name level1\" >op2</th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level0_row0\" class=\"row_heading level0 row0\" rowspan=3>max</th>\n",
       "                        <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row0\" class=\"row_heading level1 row0\" >max</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row0_col0\" class=\"data row0 col0\" >0.691</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row1\" class=\"row_heading level1 row1\" >mean</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row1_col0\" class=\"data row1 col0\" >0.718</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row2\" class=\"row_heading level1 row2\" >min</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row2_col0\" class=\"data row2 col0\" >0.72</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level0_row3\" class=\"row_heading level0 row3\" rowspan=3>mean</th>\n",
       "                        <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row3\" class=\"row_heading level1 row3\" >max</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row3_col0\" class=\"data row3 col0\" >0.62</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row4\" class=\"row_heading level1 row4\" >mean</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row4_col0\" class=\"data row4 col0\" >0.628</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row5\" class=\"row_heading level1 row5\" >min</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row5_col0\" class=\"data row5 col0\" >0.605</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level0_row6\" class=\"row_heading level0 row6\" rowspan=3>min</th>\n",
       "                        <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row6\" class=\"row_heading level1 row6\" >max</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row6_col0\" class=\"data row6 col0\" >0.574</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row7\" class=\"row_heading level1 row7\" >mean</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row7_col0\" class=\"data row7 col0\" >0.557</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <th id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2level1_row8\" class=\"row_heading level1 row8\" >min</th>\n",
       "                        <td id=\"T_6376140c_f22b_11eb_a7ae_f3abb70468b2row8_col0\" class=\"data row8 col0\" >0.531</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7faab5878250>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "benchmark = SummaCBenchmark(cut=\"test\")\n",
    "\n",
    "results = {}\n",
    "for dataset in benchmark.tasks:\n",
    "    print(\"======= %s ========\" % (dataset[\"name\"]))\n",
    "    datas = dataset[\"task\"]\n",
    "    labels = [d[\"label\"] for d in datas]\n",
    "    utils_summac_benchmark.compute_doc_level(scorer_doc, datas)\n",
    "    \n",
    "    for pred_label in datas[0].keys():\n",
    "        if \"pred_\" not in pred_label or \"total\" in pred_label: continue\n",
    "        balanced_acc = sklearn.metrics.balanced_accuracy_score(labels, [d[pred_label] for d in datas])\n",
    "        model_name, input_type = pred_label.replace(\"pred_\", \"\").split(\"|\")\n",
    "        model_type, op1, op2 = model_name.split(\"-\")\n",
    "        k = (op1, op2)\n",
    "        if k not in results:\n",
    "            results[k] = []\n",
    "        results[k].append(balanced_acc)\n",
    "\n",
    "cleaned_results = []\n",
    "for (op1, op2), vs in results.items():\n",
    "    cleaned_results.append({\"op1\": op1, \"op2\": op2, \"score\": np.mean(vs)})\n",
    "    \n",
    "pd.DataFrame(cleaned_results).groupby([\"op1\", \"op2\"]).agg({\"score\": \"sum\"}).style.set_precision(3).set_caption(\"Balanced Accuracy\")"
   ]
  }
 ],
 "metadata": {
  "finalized": {
   "timestamp": 1625710190289,
   "trusted": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}