comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This notebook contains the code for comparing the estimation value, time required to obtain it, and expected performance profiles of the resulting stopping methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys, json\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "from matplotlib.ticker import FormatStrFormatter, MultipleLocator\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ALL_DATASETS = ['midv500', 'midv2019', 'ic15', 'yvt']\n",
    "ESTIMATION_PLOT_DATASETS = [['midv500', 'midv2019'], ['ic15', 'yvt']]\n",
    "TIMING_PLOT_DATASETS = [['midv500', 'midv2019'], ['ic15', 'yvt']]\n",
    "\n",
    "# EPPS_DATASETS = ['midv500', 'midv2019']\n",
    "EPPS_DATASETS = ['ic15', 'yvt']\n",
    "\n",
    "EPPS_YLIMITS = {\n",
    "    'midv500': [0.06, 0.125],\n",
    "    'midv2019': [0.09, 0.25],\n",
    "    'ic15': [0.15, 0.35],\n",
    "    'yvt': [0.195, 0.24]\n",
    "}\n",
    "\n",
    "DATASET_LABELS = {\n",
    "    'midv500': 'MIDV-500',\n",
    "    'midv2019': 'MIDV-2019',\n",
    "    'ic15': 'IC15-Train',\n",
    "    'yvt': 'YVT'\n",
    "}\n",
    "\n",
    "METHODS = ['summation', 'treap', 'base']\n",
    "\n",
    "PRECALC_DIRECTORIES = {}\n",
    "for dataset in ALL_DATASETS:\n",
    "    PRECALC_DIRECTORIES[dataset] = {\n",
    "        'base': './precalc_base_%s' % dataset,\n",
    "        'summation': './precalc_summation_%s' % dataset,\n",
    "        'treap': './precalc_treap_%s' % dataset\n",
    "    }\n",
    "\n",
    "PLOT_COLOR = { 'base': '0.0', 'summation': '0.5', 'treap': '0.2' }\n",
    "PLOT_LINESTYLE = { 'base': '-', 'summation': '--', 'treap': ':' }\n",
    "PLOT_MARKER = { 'base': None, 'summation': 'o', 'treap': None }\n",
    "PLOT_MARKERSIZE = { 'base': None, 'summation': 6, 'treap': None }\n",
    "PLOT_LINEWIDTH = { 'base': 1.5, 'summation': 1.5, 'treap': 2.0 }\n",
    "\n",
    "PLOT_LABEL = {\n",
    "    'base': 'Base method', \n",
    "    'summation': 'Method A',\n",
    "    'treap': 'Method B'\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comparing the estimation values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_estimation_datapoints(method, dataset):\n",
    "    '''\n",
    "    Collects precalculated values for estimation \n",
    "    '''\n",
    "    precalc_dir = PRECALC_DIRECTORIES[dataset][method]\n",
    "    SMALL_DELTA = 0.1\n",
    "\n",
    "    x = [1.0 * (i + 1) for i in range(30)]\n",
    "    y = [0.0 for i in range(30)]\n",
    "    \n",
    "    precalc_files = [os.path.join(precalc_dir, x) for x in os.listdir(precalc_dir)]\n",
    "    for precalc_file in precalc_files:\n",
    "        precalc_data = None\n",
    "        with open(precalc_file) as js:\n",
    "            precalc_data = json.load(js)\n",
    "        for i in range(30):\n",
    "            y[i] += (SMALL_DELTA + precalc_data[i][1]) / (i + 2)\n",
    "    \n",
    "    for i in range(30):\n",
    "        y[i] /= len(precalc_files)\n",
    "    \n",
    "    return x, y, len(precalc_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (8, 4)\n",
    "plt.rcParams.update({'font.size': 12})\n",
    "\n",
    "plt.clf()\n",
    "plt.cla()\n",
    "\n",
    "for i_plot, plot in enumerate(ESTIMATION_PLOT_DATASETS):\n",
    "    plt.subplot(100 + 10 * len(ESTIMATION_PLOT_DATASETS) + i_plot + 1)\n",
    "    plt.title(('%s) ' % chr(ord('a') + i_plot)) + ' and '.join([DATASET_LABELS[dataset] for dataset in plot]))\n",
    "\n",
    "    X = {'summation': None, 'treap': None, 'base': None}\n",
    "    Y = {'summation': None, 'treap': None, 'base': None}\n",
    "    D = {'summation': 0, 'treap': 0, 'base': 0}\n",
    "    \n",
    "    for i_dataset, dataset in enumerate(plot):\n",
    "        for method in METHODS:\n",
    "            x, y, d = collect_estimation_datapoints(method, dataset)\n",
    "            X[method] = x\n",
    "            D[method] += d\n",
    "            for i in range(len(y)):\n",
    "                y[i] *= d\n",
    "            if Y[method] is None:\n",
    "                Y[method] = y\n",
    "            else:\n",
    "                for i in range(len(y)):\n",
    "                    Y[method][i] += y[i]\n",
    "                \n",
    "    for method in METHODS:\n",
    "        for i in range(len(Y[method])):\n",
    "            Y[method][i] /= D[method]\n",
    "    \n",
    "    plt.gca().xaxis.set_minor_locator(MultipleLocator(1))\n",
    "    plt.gca().set_xticks([1] + [5 * (i + 1) for i in range(6)])\n",
    "    plt.gca().yaxis.set_major_locator(MultipleLocator(0.01))\n",
    "    plt.grid(which = 'minor', alpha = 0.3)\n",
    "    plt.grid(which = 'major', alpha = 0.6)\n",
    "\n",
    "    for method in METHODS:\n",
    "        plt.plot(X[method], Y[method], \\\n",
    "                 label = PLOT_LABEL[method], \\\n",
    "                 color = PLOT_COLOR[method], \\\n",
    "                 linestyle = PLOT_LINESTYLE[method], \\\n",
    "                 marker = PLOT_MARKER[method], \\\n",
    "                 markersize = PLOT_MARKERSIZE[method], \\\n",
    "                 linewidth = PLOT_LINEWIDTH[method])\n",
    "\n",
    "    plt.xlim([1, 30])\n",
    "    plt.ylim([0.005, 0.065])\n",
    "\n",
    "    plt.legend(loc='upper right', prop={'size': 11})\n",
    "\n",
    "    plt.xlabel(r'Number of processed frame results')\n",
    "    plt.ylabel('Mean estimation value')\n",
    "\n",
    "plt.tight_layout(w_pad=1, h_pad=0)\n",
    "plt.savefig('estimations-composite.pdf', dpi=1200, bbox_inches='tight', pad_inches=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comparing the estimation time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_timing_datapoints(method, dataset):\n",
    "    '''\n",
    "    Collects timing values for estimation \n",
    "    '''\n",
    "    precalc_dir = PRECALC_DIRECTORIES[dataset][method]\n",
    "    \n",
    "    x = [1.0 * (i + 1) for i in range(30)]\n",
    "    y = [0.0 for i in range(30)]\n",
    "    \n",
    "    precalc_files = [os.path.join(precalc_dir, x) for x in os.listdir(precalc_dir)]\n",
    "    for precalc_file in precalc_files:\n",
    "        precalc_data = None\n",
    "        with open(precalc_file) as js:\n",
    "            precalc_data = json.load(js)\n",
    "        for i in range(30):\n",
    "            y[i] += precalc_data[i][2] + precalc_data[i][3]\n",
    "    \n",
    "    for i in range(30):\n",
    "        y[i] /= len(precalc_files)\n",
    "    \n",
    "    return x, y, len(precalc_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (8, 4)\n",
    "plt.rcParams.update({'font.size': 12})\n",
    "\n",
    "plt.clf()\n",
    "plt.cla()\n",
    "\n",
    "for i_plot, plot in enumerate(TIMING_PLOT_DATASETS):\n",
    "    plt.subplot(100 + 10 * len(TIMING_PLOT_DATASETS) + i_plot + 1)\n",
    "    plt.title(('%s) ' % chr(ord('a') + i_plot)) + ' and '.join([DATASET_LABELS[dataset] for dataset in plot]))\n",
    "\n",
    "    X = {'summation': None, 'treap': None, 'base': None}\n",
    "    Y = {'summation': None, 'treap': None, 'base': None}\n",
    "    D = {'summation': 0, 'treap': 0, 'base': 0}\n",
    "    \n",
    "    for i_dataset, dataset in enumerate(plot):\n",
    "        for method in METHODS:\n",
    "            x, y, d = collect_timing_datapoints(method, dataset)\n",
    "            X[method] = x\n",
    "            D[method] += d\n",
    "            for i in range(len(y)):\n",
    "                y[i] *= d\n",
    "            if Y[method] is None:\n",
    "                Y[method] = y\n",
    "            else:\n",
    "                for i in range(len(y)):\n",
    "                    Y[method][i] += y[i]\n",
    "    \n",
    "    for method in METHODS:\n",
    "        for i in range(len(Y[method])):\n",
    "            Y[method][i] /= D[method]\n",
    "\n",
    "    plt.gca().xaxis.set_minor_locator(MultipleLocator(1))\n",
    "    plt.gca().set_xticks([1] + [5 * (i + 1) for i in range(6)])\n",
    "    plt.grid(which = 'minor', alpha = 0.3)\n",
    "    plt.grid(which = 'major', alpha = 0.6)\n",
    "\n",
    "    for method in METHODS:\n",
    "        plt.plot(X[method], Y[method], \\\n",
    "                 label = PLOT_LABEL[method], \\\n",
    "                 color = PLOT_COLOR[method], \\\n",
    "                 linestyle = PLOT_LINESTYLE[method], \\\n",
    "                 marker = PLOT_MARKER[method], \\\n",
    "                 markersize = PLOT_MARKERSIZE[method], \\\n",
    "                 linewidth = PLOT_LINEWIDTH[method])\n",
    "\n",
    "    plt.xlim([1, 30])\n",
    "    plt.ylim([-0.05, 0.65])\n",
    "\n",
    "    plt.legend(loc='upper right', prop={'size': 11})\n",
    "\n",
    "    plt.xlabel(r'Number of processed frame results')\n",
    "    plt.ylabel('Mean time in sec. per decision')\n",
    "\n",
    "plt.tight_layout(w_pad=1, h_pad=0)\n",
    "plt.savefig('timing-composite.pdf', dpi=1200, bbox_inches='tight', pad_inches=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Printing the table\n",
    "for i_tbl, tbl in enumerate(TIMING_PLOT_DATASETS):\n",
    "    print('Table %d: ' % (1 + i_tbl) + ' and '.join([DATASET_LABELS[dataset] for dataset in tbl]))\n",
    "\n",
    "    XS = [5, 10, 15, 20, 25]\n",
    "    \n",
    "    X = {'summation': None, 'treap': None, 'base': None}\n",
    "    Y = {'summation': None, 'treap': None, 'base': None}\n",
    "    D = {'summation': 0, 'treap': 0, 'base': 0}\n",
    "    \n",
    "    for i_dataset, dataset in enumerate(tbl):\n",
    "        for method in METHODS:\n",
    "            x, y, d = collect_timing_datapoints(method, dataset)\n",
    "            X[method] = x\n",
    "            D[method] += d\n",
    "            for i in range(len(y)):\n",
    "                y[i] *= d\n",
    "            if Y[method] is None:\n",
    "                Y[method] = y\n",
    "            else:\n",
    "                for i in range(len(y)):\n",
    "                    Y[method][i] += y[i]\n",
    "    \n",
    "    print('          ' + ' ; '.join(['n = %02d' % x for x in XS]))\n",
    "    \n",
    "    for method in ['base', 'summation', 'treap']:\n",
    "        for i in range(len(Y[method])):\n",
    "            Y[method][i] /= D[method]\n",
    "        ys = []\n",
    "        for x, y in zip(X[method], Y[method]):\n",
    "            if x in XS:\n",
    "                ys.append(y)\n",
    "        print('% 9s  ' % method + '    '.join(['%.3f' % y for y in ys]))\n",
    "    \n",
    "    print('')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comparing expected performance profiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_counting_stopper_epp(dataset):\n",
    "    '''\n",
    "    Collects expected performance profile for a simple stopper which \n",
    "    stops after a fixed number of processed frames\n",
    "    '''\n",
    "    precalc_dir = PRECALC_DIRECTORIES[dataset]['base']\n",
    "    x = [1.0 * (i + 1) for i in range(30)]\n",
    "    y = [0.0 for i in range(30)]\n",
    "    \n",
    "    precalc_files = [os.path.join(precalc_dir, x) for x in os.listdir(precalc_dir)]\n",
    "    for precalc_file in precalc_files:\n",
    "        precalc_data = None\n",
    "        with open(precalc_file) as js:\n",
    "            precalc_data = json.load(js)\n",
    "        for i in range(30):\n",
    "            y[i] += precalc_data[i][0]\n",
    "    \n",
    "    for i in range(30):\n",
    "        y[i] /= len(precalc_files)\n",
    "    \n",
    "    return x, y\n",
    "\n",
    "def collect_modelling_stopper_epp(method, dataset):\n",
    "    '''\n",
    "    Collects expected performance profile for a next combination result \n",
    "    modelling stopping method\n",
    "    '''\n",
    "    precalc_dir = PRECALC_DIRECTORIES[dataset][method]\n",
    "    SMALL_DELTA = 0.1\n",
    "    \n",
    "    DATAPOINTS_COUNT = 300\n",
    "    MIN_THRESHOLD = -0.001\n",
    "    MAX_THRESHOLD = 0.15\n",
    "    THRESHOLDS = [MIN_THRESHOLD + (MAX_THRESHOLD - MIN_THRESHOLD) * i / (DATAPOINTS_COUNT - 1) \\\n",
    "                  for i in range(DATAPOINTS_COUNT)]\n",
    "    \n",
    "    x = []\n",
    "    y = []\n",
    "    \n",
    "    precalc = []\n",
    "    precalc_files = [os.path.join(precalc_dir, x) for x in os.listdir(precalc_dir)]\n",
    "    for precalc_file in precalc_files:\n",
    "        precalc_data = None\n",
    "        with open(precalc_file) as js:\n",
    "            precalc_data = json.load(js)\n",
    "        precalc.append(precalc_data)\n",
    "    \n",
    "    for threshold in THRESHOLDS:\n",
    "        sum_clip_length = 0.0\n",
    "        sum_error_level = 0.0\n",
    "        \n",
    "        for precalc_data in precalc:\n",
    "            stopped = False\n",
    "            clip_start = 1 if threshold <= 1.0 else 0\n",
    "            for i in range(clip_start, 30):\n",
    "                delta = (SMALL_DELTA + precalc_data[i][1]) / (i + 2)\n",
    "                if delta <= threshold:\n",
    "                    sum_clip_length += (i + 1)\n",
    "                    sum_error_level += precalc_data[i][0]\n",
    "                    stopped = True\n",
    "                    break\n",
    "            if not stopped:\n",
    "                sum_clip_length += 30\n",
    "                sum_error_level += precalc_data[-1][0]\n",
    "                \n",
    "        x.append(sum_clip_length / len(precalc))\n",
    "        y.append(sum_error_level / len(precalc))\n",
    "    \n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = (14, 4)\n",
    "plt.rcParams.update({'font.size': 12})\n",
    "\n",
    "plt.clf()\n",
    "plt.cla()\n",
    "\n",
    "for i_dataset, dataset in enumerate(EPPS_DATASETS):\n",
    "    plt.subplot(100 + 10 * len(EPPS_DATASETS) + i_dataset + 1)\n",
    "    plt.title(('%s) ' % chr(ord('a') + i_dataset)) + DATASET_LABELS[dataset])\n",
    "\n",
    "    plt.gca().xaxis.set_minor_locator(MultipleLocator(0.5))\n",
    "    plt.gca().xaxis.set_major_locator(MultipleLocator(2))\n",
    "\n",
    "    c_x, c_y = collect_counting_stopper_epp(dataset)\n",
    "    plt.plot(c_x, c_y, label=r'Stopping at fixed stage', color='0.5', linestyle='--', linewidth = 1)\n",
    "\n",
    "    PLOT_LINEWIDTH = { 'base': 2.0, 'summation': 2.0, 'treap': 2.5 }\n",
    "\n",
    "    for method in METHODS:\n",
    "        x, y = collect_modelling_stopper_epp(method, dataset)\n",
    "        plt.plot(x, y, \\\n",
    "                 label = PLOT_LABEL[method], \\\n",
    "                 color = PLOT_COLOR[method], \\\n",
    "                 linestyle = PLOT_LINESTYLE[method], \\\n",
    "                 linewidth = PLOT_LINEWIDTH[method])\n",
    "\n",
    "    plt.xlim([1, 20])\n",
    "    plt.ylim(EPPS_YLIMITS[dataset])\n",
    "    plt.grid()\n",
    "    plt.legend()\n",
    "\n",
    "    plt.xlabel(r'Mean number of frames')\n",
    "    plt.ylabel('Mean error level')\n",
    "    plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%.2f'))\n",
    "\n",
    "plt.savefig('epps_%s.pdf' % '_'.join(EPPS_DATASETS), dpi=1200, bbox_inches='tight', pad_inches=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}