examples/notebooks/plotting/plot_search_perfomance_search_types.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# %matplotlib inline  \n",
    "from __init__ import *\n",
    "import numpy as np\n",
    "import core.steps as steps\n",
    "from core import data_store as ds\n",
    "from core.common.ds_utils import print_ds_items_info\n",
    "from core.evaluation.retrieval_perfomance import extract_perfomances_from_arr\n",
    "from core.common.path_helper import DataStoreHelper\n",
    "from core.quantization.pq_quantizer import build_pq_params_str, extract_pq_params_from_str\n",
    "import math, os\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ds_helper=DataStoreHelper(r'C:\\data\\computation\\brodatz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot search types perfomance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def define_memory_per_descriptor(descriptor_name):\n",
    "    if 'product' in descriptor_name or 'bincount' in descriptor_name:\n",
    "        pq_params=ds_helper.extract_pq_params_from_descriptor_name(descriptor_name)\n",
    "        m=pq_params['n_quantizers']\n",
    "        K=pq_params['n_clusters']\n",
    "        if 'product' in descriptor_name:\n",
    "            mem=m*K*4\n",
    "        elif 'bincount' in descriptor_name:\n",
    "            mem=(K**m)*4\n",
    "    elif 'lbp' in descriptor_name:\n",
    "        mem=512*4\n",
    "    elif 'hist' in descriptor_name:\n",
    "        mem=256*4\n",
    "    elif 'glcm' in descriptor_name:\n",
    "        mem=256*256*4\n",
    "   \n",
    "    return mem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def build_mem_str(descriptor_name):\n",
    "    mem=define_memory_per_descriptor(descriptor_name)\n",
    "    if \"bovw\" in descriptor_name:\n",
    "        pq_params=extract_pq_params_from_str(descriptor_name)\n",
    "        pq_params_str=build_pq_params_str(pq_params)\n",
    "        descriptor_name=\"bovw_{}\".format(pq_params_str)\n",
    "    mem_str=descriptor_name\n",
    "#     mem_str=\"{}\\n(bytes: {})\".format(descriptor_name, mem)\n",
    "    return mem_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, perfomance_type, n_nearest_cutoffs, save_to_file=False, bar=False):\n",
    "    subplotname__label__x__y = {}\n",
    "    label__kwargs={}\n",
    "    for n_nearest_cutoff in n_nearest_cutoffs:\n",
    "        label__x__y = subplotname__label__x__y.setdefault(n_nearest_cutoff,{})\n",
    "        for pq_params in pq_params_arr:\n",
    "            x_arr=[build_mem_str(descriptor_name) for descriptor_name in descriptor_names]\n",
    "            ex_y_arr=[]\n",
    "            sdc_y_arr=[]\n",
    "            adc_y_arr=[]\n",
    "            imi_y_arr=[]\n",
    "            for descriptor_name in descriptor_names:\n",
    "                ex_perfomances_ds=ds_helper.ex_search_perfomances_ds(descriptor_name)\n",
    "                sdc_perfomances_ds=ds_helper.pq_search_perfomances_plot_path('sdc', descriptor_name, perfomance_type)\n",
    "                adc_perfomances_ds=ds_helper.pq_search_perfomances_plot_path('adc', descriptor_name, perfomance_type)\n",
    "                imi_perfomances_ds=ds_helper.pq_search_perfomances_plot_path('imi', descriptor_name, perfomance_type)\n",
    "\n",
    "                ex_perfomances_ds=ds_helper.ex_search_perfomances_ds(descriptor_name)\n",
    "                perfomances_arr=ds_helper.perfomance_arr(ex_perfomances_ds)\n",
    "                y = extract_perfomances_from_arr(perfomances_arr, perfomance_type)[n_nearest_cutoff-1]\n",
    "                ex_y_arr.append(y)\n",
    "\n",
    "                perfomances_ds=ds_helper.pq_search_perfomances_ds('sdc', descriptor_name, pq_params)\n",
    "                perfomances_arr=ds_helper.perfomance_arr(perfomances_ds)\n",
    "                y = extract_perfomances_from_arr(perfomances_arr, perfomance_type)[n_nearest_cutoff-1]\n",
    "                sdc_y_arr.append(y)\n",
    "\n",
    "                perfomances_ds=ds_helper.pq_search_perfomances_ds('adc', descriptor_name, pq_params)\n",
    "                perfomances_arr=ds_helper.perfomance_arr(perfomances_ds)\n",
    "                y = extract_perfomances_from_arr(perfomances_arr, perfomance_type)[n_nearest_cutoff-1]\n",
    "                adc_y_arr.append(y)\n",
    "\n",
    "                perfomances_ds=ds_helper.pq_search_perfomances_ds('imi', descriptor_name, pq_params)\n",
    "                perfomances_arr=ds_helper.perfomance_arr(perfomances_ds)\n",
    "                y = extract_perfomances_from_arr(perfomances_arr, perfomance_type)[n_nearest_cutoff-1]\n",
    "                imi_y_arr.append(y)\n",
    "\n",
    "            ex_x__y=dict(zip(x_arr,ex_y_arr))\n",
    "            sdc_x__y=dict(zip(x_arr,sdc_y_arr))\n",
    "            adc_x__y=dict(zip(x_arr,adc_y_arr))\n",
    "            imi_x__y=dict(zip(x_arr,imi_y_arr))\n",
    "\n",
    "            label__x__y['exh']=ex_x__y\n",
    "            label__x__y['sdc'+'-'+build_pq_params_str(pq_params)]=sdc_x__y\n",
    "            label__x__y['adc'+'-'+build_pq_params_str(pq_params)]=adc_x__y\n",
    "            label__x__y['imi'+'-'+build_pq_params_str(pq_params)]=imi_x__y\n",
    "\n",
    "\n",
    "            label__kwargs['exh']={'markersize':9.0}\n",
    "            label__kwargs['sdc'+'-'+build_pq_params_str(pq_params)]={'markersize':9.0}\n",
    "            label__kwargs['adc'+'-'+build_pq_params_str(pq_params)]={'markersize':9.0}\n",
    "            label__kwargs['imi'+'-'+build_pq_params_str(pq_params)]={'markersize':9.0}\n",
    "\n",
    "    #         label__kwargs[0]={'linewidth':3.0}\n",
    "    \n",
    "    if save_to_file:\n",
    "        plot_path=ds_helper.base_dir+'search_types_'+perfomance_type\n",
    "    else:\n",
    "        plot_path=None\n",
    "#     label_callback=lambda label: 'original' if label==0 else 'm = {}'.format(label)\n",
    "    steps.plotting_step3(subplotname__label__x__y, 'n_nearest: ', str, '', perfomance_type, save_to_file=plot_path, label__kwargs=label__kwargs, bar=bar,\n",
    "                        figsize=dict(width=16, height=15))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_nearest_cutoffs=[1,5,10,25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "descriptor_names=['histograms', 'lbphistograms', 'glcms']\n",
    "choosen_bovwproductbincounts_pq_params= [{'n_clusters': K, 'n_quantizers': m} for K,m in [(64,4),(128,1),(16,1)]]\n",
    "descriptor_names+=ds_helper.bovw_descriptors_names('bovwproductbincounts',choosen_bovwproductbincounts_pq_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pq_params_arr = [{'n_clusters': K, 'n_quantizers': m} for K,m in [(64, 2),(128, 2),(256, 2)]]\n",
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'precision', n_nearest_cutoffs, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'precision', n_nearest_cutoffs, False, bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'precision', n_nearest_cutoffs, True, bar=True)\n",
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'recall', n_nearest_cutoffs, True, bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'precision', n_nearest_cutoffs, False, bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pq_params_arr = [{'n_clusters': K, 'n_quantizers': m} for K,m in [(64, 2),(128, 2),(256, 2)]]\n",
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'recall', n_nearest_cutoffs, False)\n",
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'recall', n_nearest_cutoffs, False, bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pq_params_arr = [{'n_clusters': K, 'n_quantizers': m} for K,m in [(64, 4),(128, 2),(256, 2)]]\n",
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'precision', n_nearest_cutoffs, False, bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'mAP', n_nearest_cutoffs, False)\n",
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, 'mAP', n_nearest_cutoffs, False, bar=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Prodcut quantization approach and search types based on it are only 0.025-0.05 worser than exhaustive straightforward search.\n",
    "Even imi(inverted multi-index) search which is approximate search shows good results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pq_params_arr = [{'n_clusters': K, 'n_quantizers': m} for K,m in [(128,1),(128,2),(64,2)]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "perfomance_names=['precision', 'recall', 'mAP']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for perfomance_name in perfomance_names:\n",
    "    plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, perfomance_name, n_nearest_cutoffs, True, bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_pq_search_params_perfomance(descriptor_names, pq_params_arr, perfomance_name, n_nearest_cutoffs, False, bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}