eval2tex.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from eval_paths import get_shortname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Q_to_subscript(s):\n",
    "    if 'GOODQ' in s:\n",
    "        match = re.search(r'\\d+', s)\n",
    "        name = r'GOOD\\textsubscript{' + match.group(0) + '}'\n",
    "        return name\n",
    "    else:\n",
    "        return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_list = [\n",
    "        r'Plain',\n",
    "        r'CEDA',\n",
    "        r'OE',\n",
    "        r'ACET',\n",
    "        r'CCU',\n",
    "        r'GOOD\\textsubscript{0}',\n",
    "        r'GOOD\\textsubscript{20}',\n",
    "        r'GOOD\\textsubscript{40}',\n",
    "        r'GOOD\\textsubscript{60}',\n",
    "        r'GOOD\\textsubscript{80}',\n",
    "        r'GOOD\\textsubscript{90}',\n",
    "        r'GOOD\\textsubscript{95}',\n",
    "        r'GOOD\\textsubscript{100}',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dset_log_names = {\n",
    "    'CIFAR-100'      : 'CIFAR100',\n",
    "    'CIFAR-10'       : 'CIFAR10',\n",
    "    'SVHN'           : 'SVHN',\n",
    "    'LSUN Classroom' : 'LSUN',\n",
    "    'Uniform Noise'  : 'Noise_Dataset_noise_uniform',\n",
    "    '80M Tiny Images': 'TinyImages',\n",
    "    'ImageNet-'      : 'ImageNet-',\n",
    "    'Smooth Noise'   : 'Noise_Dataset_noise_low_freq',\n",
    "    'Omniglot'       : 'Omniglot',\n",
    "    'notMNIST'       : 'ImageFolder',\n",
    "    'FashionMNIST'   : 'FashionMNIST',\n",
    "    'EMNIST Letters' : 'EMNIST',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dset_aauc_names = {\n",
    "    'CIFAR-100'      : None,\n",
    "    'CIFAR-10'       : 'GrayCIFAR10',\n",
    "    'SVHN'           : None,\n",
    "    'LSUN Classroom' : None,\n",
    "    'Uniform Noise'  : 'UniformNoise',\n",
    "    '80M Tiny Images': 'Tiny',\n",
    "    'ImageNet-'      : None,\n",
    "    'Smooth Noise'   : None,\n",
    "    'Omniglot'       : 'Omniglot',\n",
    "    'notMNIST'       : 'NotMNIST',\n",
    "    'FashionMNIST'   : 'FMNIST',\n",
    "    'EMNIST Letters' : 'EMNIST',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_c10 = 'evals/2020-10-14_16:34:38.log'\n",
    "file_svhn = 'evals/2020-10-14_18:14:40.log'\n",
    "file_mnist = 'evals/2020-10-15_15:13:11.log'\n",
    "file_mnist_04 = 'evals/2020-10-16_08:49:04.log'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "aauc_file_mnist = 'evals/aauc/MNIST_0.3_2020-10-22 12 47 03.842292.txt'\n",
    "aauc_file_mnist_04 = 'evals/aauc/MNIST_0.4_2020-10-22 12 49 06.954378.txt'\n",
    "aauc_file_mnist_add = 'evals/aauc/MNIST_0.3_2020-10-22 12 54 10.248237.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dset_list_C10_main = ['CIFAR-100', 'SVHN', 'LSUN Classroom', 'Uniform Noise']\n",
    "dset_list_C10_add = ['80M Tiny Images', 'ImageNet-', 'Smooth Noise']\n",
    "dset_list_svhn_main = ['CIFAR-100', 'CIFAR-10', 'LSUN Classroom', 'Uniform Noise']\n",
    "dset_list_svhn_add = ['80M Tiny Images', 'ImageNet-', 'Smooth Noise']\n",
    "dset_list_mnist_main = ['FashionMNIST', 'EMNIST Letters', 'CIFAR-10', 'Uniform Noise']\n",
    "dset_list_mnist_add = ['80M Tiny Images', 'Omniglot', 'notMNIST']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tex_length(c):\n",
    "    c = c.strip()\n",
    "    if len(c) == 4:\n",
    "        c = r'\\noindent\\phantom{0}' + c\n",
    "    elif len(c) == 3:\n",
    "        c = r'\\noindent\\phantom{00}' + c\n",
    "    return c.ljust(17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_table(logfile_dest, model_list, dset_list):\n",
    "    from_log = {}\n",
    "    dset_pos_in_log = {}\n",
    "    got_datasets = False\n",
    "    logfile = open(logfile_dest)\n",
    "    for l in logfile:\n",
    "        if 'Model & Acc. &' in l and not got_datasets:\n",
    "            got_datasets = True\n",
    "            for i, d in enumerate(l.split('&')):\n",
    "                if d[-3:] == ' P ':\n",
    "                    dset_pos_in_log[d[1:-3]] = i\n",
    "        if '.pt &' in l:\n",
    "            from_log[Q_to_subscript(get_shortname(l))] = l\n",
    "    for name in model_list:\n",
    "        if name not in from_log.keys():\n",
    "            row = []\n",
    "            row += [name, 'X']\n",
    "            for d in dset_list:\n",
    "                row += 3*['X']\n",
    "        else:\n",
    "            values = from_log[name][:-3].split('&')\n",
    "            row = [] \n",
    "            row += [name, values[1]]\n",
    "            for d in dset_list:\n",
    "                p = dset_pos_in_log[dset_log_names[d]]\n",
    "                row += values[p:p+3]\n",
    "        tex_row = row[:1] + [tex_length(e) for e in row[1:]]\n",
    "        print(' & '.join(tex_row) + ' \\\\\\\\')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def name_from_aauc_line(l):\n",
    "    f = l.split('&')[0].strip()\n",
    "    if f in {'Plain', 'CEDA', 'OE', 'ACET'}:\n",
    "        return f\n",
    "    elif f[:4] == 'GOOD':\n",
    "        match = re.search(r'\\d+', f)\n",
    "        name = r'GOOD\\textsubscript{' + match.group(0) + r'}'\n",
    "        return name\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_table_aauc(logfile_dest, aaucfile_dest, model_list, dset_list):\n",
    "    from_log = {}\n",
    "    dset_pos_in_log = {}\n",
    "    got_datasets = False\n",
    "    logfile = open(logfile_dest)\n",
    "    for l in logfile:\n",
    "        if 'Model & Acc. &' in l and not got_datasets:\n",
    "            got_datasets = True\n",
    "            for i, d in enumerate(l.split('&')):\n",
    "                if d[-3:] == ' P ':\n",
    "                    dset_pos_in_log[d[1:-3]] = i\n",
    "        if '.pt &' in l:\n",
    "            from_log[Q_to_subscript(get_shortname(l))] = l\n",
    "    aaucfile = open(aaucfile_dest)\n",
    "    dset_pos_in_aauc = {}\n",
    "    aaucs = {}\n",
    "    for l in aaucfile:\n",
    "        if '{} & ' in l:\n",
    "            for i, d in enumerate(l[:-3].split('&')):\n",
    "                for n in dset_aauc_names.keys():\n",
    "                    if dset_aauc_names[n] == d.strip():\n",
    "                        dset_pos_in_aauc[n] = i\n",
    "        lname = name_from_aauc_line(l)\n",
    "        if lname not in model_list:\n",
    "            continue\n",
    "        else:\n",
    "            aaucs[lname] = {}\n",
    "            for d in dset_pos_in_aauc.keys():\n",
    "                aaucs[lname][d] = l[:-3].split('&')[dset_pos_in_aauc[d]].strip()\n",
    "    for name in model_list:\n",
    "        if name not in from_log.keys():\n",
    "            row = []\n",
    "            row += [name, 'X']\n",
    "            for d in dset_list:\n",
    "                row += 3*['X']\n",
    "        else:\n",
    "            values = from_log[name][:-3].split('&')\n",
    "            row = [] \n",
    "            row += [name, values[1]]\n",
    "            for d in dset_list:\n",
    "                p = dset_pos_in_log[dset_log_names[d]]\n",
    "                row.append(values[p])\n",
    "                if name not in aaucs.keys():\n",
    "                    row.append('A')\n",
    "                elif d in aaucs[name].keys():\n",
    "                    row.append(aaucs[name][d])\n",
    "                else:\n",
    "                    row.append('A')\n",
    "                row.append(values[p+2])\n",
    "        tex_row = row[:1] + [tex_length(e) for e in row[1:]]\n",
    "        print(' & '.join(tex_row) + ' \\\\\\\\')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plain & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}98.0 & \\noindent\\phantom{0}34.2 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}88.0 & \\noindent\\phantom{0}31.4 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}98.8 & \\noindent\\phantom{0}36.6 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}99.1 & \\noindent\\phantom{0}36.5 & \\noindent\\phantom{00}0.0 \\\\\n",
      "CEDA & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}82.1 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}92.6 & \\noindent\\phantom{0}52.8 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}95.1 & \\noindent\\phantom{00}0.0 & 100.0             & 100.0             & \\noindent\\phantom{00}0.0 \\\\\n",
      "OE & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}76.8 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}92.7 & \\noindent\\phantom{0}50.9 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}92.4 & \\noindent\\phantom{00}0.0 & 100.0             & 100.0             & \\noindent\\phantom{00}0.0 \\\\\n",
      "ACET & \\noindent\\phantom{0}99.4 & 100.0             & \\noindent\\phantom{0}98.4 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}95.9 & \\noindent\\phantom{0}61.5 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}99.3 & \\noindent\\phantom{00}0.0 & 100.0             & 100.0             & \\noindent\\phantom{00}0.0 \\\\\n",
      "CCU & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 \\\\\n",
      "GOOD\\textsubscript{0} & \\noindent\\phantom{0}99.5 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}82.3 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}92.9 & \\noindent\\phantom{0}55.0 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}94.7 & \\noindent\\phantom{00}0.0 & 100.0             & 100.0             & \\noindent\\phantom{00}0.0 \\\\\n",
      "GOOD\\textsubscript{20} & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}99.8 & \\noindent\\phantom{0}88.2 & \\noindent\\phantom{00}9.7 & \\noindent\\phantom{0}95.3 & \\noindent\\phantom{0}54.3 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}97.6 & \\noindent\\phantom{0}28.3 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{40} & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}99.8 & \\noindent\\phantom{0}88.0 & \\noindent\\phantom{0}29.1 & \\noindent\\phantom{0}95.7 & \\noindent\\phantom{0}56.6 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}97.7 & \\noindent\\phantom{0}65.2 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{60} & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}88.8 & \\noindent\\phantom{0}42.0 & \\noindent\\phantom{0}96.6 & \\noindent\\phantom{0}57.9 & \\noindent\\phantom{00}0.1 & 100.0             & \\noindent\\phantom{0}97.9 & \\noindent\\phantom{0}85.3 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{80} & \\noindent\\phantom{0}99.1 & \\noindent\\phantom{0}99.8 & \\noindent\\phantom{0}90.3 & \\noindent\\phantom{0}55.5 & \\noindent\\phantom{0}97.9 & \\noindent\\phantom{0}63.1 & \\noindent\\phantom{00}3.4 & 100.0             & \\noindent\\phantom{0}98.4 & \\noindent\\phantom{0}94.7 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{90} & \\noindent\\phantom{0}98.8 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}91.4 & \\noindent\\phantom{0}66.9 & \\noindent\\phantom{0}98.0 & \\noindent\\phantom{0}59.4 & \\noindent\\phantom{00}5.1 & 100.0             & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}97.8 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{95} & \\noindent\\phantom{0}98.8 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}93.1 & \\noindent\\phantom{0}73.9 & \\noindent\\phantom{0}98.7 & \\noindent\\phantom{0}59.2 & \\noindent\\phantom{00}5.6 & 100.0             & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}98.8 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{100} & \\noindent\\phantom{0}98.7 & 100.0             & \\noindent\\phantom{0}96.5 & \\noindent\\phantom{0}78.0 & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}53.8 & \\noindent\\phantom{00}3.3 & 100.0             & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}99.4 & 100.0             & 100.0             & 100.0             \\\\\n"
     ]
    }
   ],
   "source": [
    "get_table_aauc(file_mnist, aauc_file_mnist, model_list, dset_list_mnist_main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plain & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}98.7 & \\noindent\\phantom{0}36.9 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}97.9 & \\noindent\\phantom{0}38.6 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}91.9 & \\noindent\\phantom{0}38.8 & \\noindent\\phantom{00}0.0 \\\\\n",
      "CEDA & \\noindent\\phantom{0}99.4 & 100.0             & \\noindent\\phantom{0}94.3 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}98.5 & \\noindent\\phantom{0}53.1 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}97.8 & \\noindent\\phantom{00}0.0 \\\\\n",
      "OE & \\noindent\\phantom{0}99.4 & 100.0             & \\noindent\\phantom{0}91.5 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}98.5 & \\noindent\\phantom{0}51.0 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}96.8 & \\noindent\\phantom{00}0.0 \\\\\n",
      "ACET & \\noindent\\phantom{0}99.4 & 100.0             & \\noindent\\phantom{0}99.2 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}99.5 & \\noindent\\phantom{0}76.5 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}99.5 & \\noindent\\phantom{00}0.0 \\\\\n",
      "CCU & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 \\\\\n",
      "GOOD\\textsubscript{0} & \\noindent\\phantom{0}99.5 & 100.0             & \\noindent\\phantom{0}93.8 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}98.6 & \\noindent\\phantom{0}55.7 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}97.7 & \\noindent\\phantom{00}0.0 \\\\\n",
      "GOOD\\textsubscript{20} & \\noindent\\phantom{0}99.0 & 100.0             & \\noindent\\phantom{0}97.1 & \\noindent\\phantom{0}32.7 & \\noindent\\phantom{0}97.0 & \\noindent\\phantom{0}42.4 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}99.6 & \\noindent\\phantom{0}19.3 \\\\\n",
      "GOOD\\textsubscript{40} & \\noindent\\phantom{0}99.0 & 100.0             & \\noindent\\phantom{0}97.2 & \\noindent\\phantom{0}59.5 & \\noindent\\phantom{0}96.9 & \\noindent\\phantom{0}36.8 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}99.7 & \\noindent\\phantom{0}44.7 \\\\\n",
      "GOOD\\textsubscript{60} & \\noindent\\phantom{0}99.0 & 100.0             & \\noindent\\phantom{0}97.3 & \\noindent\\phantom{0}77.8 & \\noindent\\phantom{0}96.3 & \\noindent\\phantom{0}31.3 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}99.8 & \\noindent\\phantom{0}76.2 \\\\\n",
      "GOOD\\textsubscript{80} & \\noindent\\phantom{0}99.1 & 100.0             & \\noindent\\phantom{0}97.8 & \\noindent\\phantom{0}89.4 & \\noindent\\phantom{0}96.9 & \\noindent\\phantom{0}34.2 & \\noindent\\phantom{00}1.2 & 100.0             & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}96.7 \\\\\n",
      "GOOD\\textsubscript{90} & \\noindent\\phantom{0}98.8 & 100.0             & \\noindent\\phantom{0}98.7 & \\noindent\\phantom{0}94.2 & \\noindent\\phantom{0}97.8 & \\noindent\\phantom{0}40.5 & \\noindent\\phantom{00}2.2 & 100.0             & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}99.2 \\\\\n",
      "GOOD\\textsubscript{95} & \\noindent\\phantom{0}98.8 & 100.0             & \\noindent\\phantom{0}99.2 & \\noindent\\phantom{0}96.1 & \\noindent\\phantom{0}97.8 & \\noindent\\phantom{0}42.2 & \\noindent\\phantom{00}2.4 & 100.0             & 100.0             & \\noindent\\phantom{0}99.5 \\\\\n",
      "GOOD\\textsubscript{100} & \\noindent\\phantom{0}98.7 & 100.0             & \\noindent\\phantom{0}99.5 & \\noindent\\phantom{0}97.7 & \\noindent\\phantom{0}98.6 & \\noindent\\phantom{0}50.7 & \\noindent\\phantom{00}1.8 & 100.0             & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}99.3 \\\\\n"
     ]
    }
   ],
   "source": [
    "get_table_aauc(file_mnist, aauc_file_mnist_add, model_list, dset_list_mnist_add)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plain & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}98.0 & \\noindent\\phantom{0}28.6 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}88.0 & \\noindent\\phantom{0}26.9 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}98.8 & \\noindent\\phantom{0}32.4 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}99.2 & \\noindent\\phantom{0}34.3 & \\noindent\\phantom{00}0.0 \\\\\n",
      "CEDA & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}69.9 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}92.6 & \\noindent\\phantom{0}49.1 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}82.8 & \\noindent\\phantom{00}0.0 & 100.0             & 100.0             & \\noindent\\phantom{00}0.0 \\\\\n",
      "OE & \\noindent\\phantom{0}99.4 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}63.6 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}92.7 & \\noindent\\phantom{0}47.4 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}76.0 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{00}0.0 \\\\\n",
      "ACET & \\noindent\\phantom{0}99.4 & 100.0             & \\noindent\\phantom{0}91.3 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}95.9 & \\noindent\\phantom{0}47.8 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}92.3 & \\noindent\\phantom{00}0.0 & 100.0             & 100.0             & \\noindent\\phantom{00}0.0 \\\\\n",
      "CCU & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 & X                 \\\\\n",
      "GOOD\\textsubscript{0} & \\noindent\\phantom{0}99.5 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}70.8 & \\noindent\\phantom{00}0.0 & \\noindent\\phantom{0}92.9 & \\noindent\\phantom{0}51.8 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}81.5 & \\noindent\\phantom{00}0.0 & 100.0             & 100.0             & \\noindent\\phantom{00}0.0 \\\\\n",
      "GOOD\\textsubscript{20} & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}99.8 & \\noindent\\phantom{0}81.9 & \\noindent\\phantom{00}3.6 & \\noindent\\phantom{0}95.3 & \\noindent\\phantom{0}46.2 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}91.4 & \\noindent\\phantom{00}6.4 & 100.0             & 100.0             & \\noindent\\phantom{0}99.9 \\\\\n",
      "GOOD\\textsubscript{40} & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}99.8 & \\noindent\\phantom{0}81.6 & \\noindent\\phantom{0}18.5 & \\noindent\\phantom{0}95.7 & \\noindent\\phantom{0}46.9 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}92.0 & \\noindent\\phantom{0}26.3 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{60} & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}82.5 & \\noindent\\phantom{0}30.6 & \\noindent\\phantom{0}96.6 & \\noindent\\phantom{0}47.1 & \\noindent\\phantom{00}0.0 & 100.0             & \\noindent\\phantom{0}92.7 & \\noindent\\phantom{0}55.4 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{80} & \\noindent\\phantom{0}99.1 & \\noindent\\phantom{0}99.8 & \\noindent\\phantom{0}84.5 & \\noindent\\phantom{0}41.9 & \\noindent\\phantom{0}97.9 & \\noindent\\phantom{0}52.1 & \\noindent\\phantom{00}1.0 & 100.0             & \\noindent\\phantom{0}93.8 & \\noindent\\phantom{0}77.3 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{90} & \\noindent\\phantom{0}98.8 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}86.3 & \\noindent\\phantom{0}45.5 & \\noindent\\phantom{0}98.0 & \\noindent\\phantom{0}48.6 & \\noindent\\phantom{00}1.4 & 100.0             & \\noindent\\phantom{0}95.7 & \\noindent\\phantom{0}77.6 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{95} & \\noindent\\phantom{0}98.8 & \\noindent\\phantom{0}99.9 & \\noindent\\phantom{0}87.8 & \\noindent\\phantom{0}49.0 & \\noindent\\phantom{0}98.7 & \\noindent\\phantom{0}47.0 & \\noindent\\phantom{00}1.6 & 100.0             & \\noindent\\phantom{0}96.8 & \\noindent\\phantom{0}79.8 & 100.0             & 100.0             & 100.0             \\\\\n",
      "GOOD\\textsubscript{100} & \\noindent\\phantom{0}98.7 & 100.0             & \\noindent\\phantom{0}92.0 & \\noindent\\phantom{0}48.8 & \\noindent\\phantom{0}99.0 & \\noindent\\phantom{0}39.1 & \\noindent\\phantom{00}0.8 & 100.0             & \\noindent\\phantom{0}98.2 & \\noindent\\phantom{0}75.9 & 100.0             & 100.0             & 100.0             \\\\\n"
     ]
    }
   ],
   "source": [
    "get_table_aauc(file_mnist_04, aauc_file_mnist_04, model_list, dset_list_mnist_main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}