eval_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation Demo\n",
    "ref: https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pycocotools.coco import COCO\n",
    "from pycocotools.cocoeval import COCOeval\n",
    "\n",
    "from fashionpedia.fp import Fashionpedia\n",
    "from fashionpedia.fp_eval import FPEval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running demo for *segm* results.\n"
     ]
    }
   ],
   "source": [
    "# set up for Fashionpedia eval type\n",
    "annType = ['segm','bbox']\n",
    "annType = annType[0]      #specify type here\n",
    "print('Running demo for *%s* results.'%(annType))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# annotation and prediction file names here\n",
    "anno_file = \"data/sample.json\"\n",
    "res_file = \"data/fake_results.json\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "loading attributes...\n",
      "attributes index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "loading attributes...\n",
      "attributes index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.07s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.69s).\n"
     ]
    }
   ],
   "source": [
    "# initialize Fashionpedia groudtruth and prediction api\n",
    "fpGt=Fashionpedia(anno_file)\n",
    "fpDt=fpGt.loadRes(res_file)\n",
    "imgIds=sorted(fpGt.getImgIds())\n",
    "\n",
    "# run evaluation\n",
    "fp_eval = FPEval(fpGt,fpDt,annType)\n",
    "fp_eval.params.imgIds  = imgIds\n",
    "fp_eval.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "results with both IoU and F1 thresholds\n",
      "================================================================================\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=100 | superCat=outerwear] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=100 | superCat=     part] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=100 | superCat=      all] = 0.590\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=0.50:0.95 |area=all | maxDets=100 | superCat=      all] = 0.590\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=0.50:0.95 |area=all | maxDets=100 | superCat=      all] = 0.590\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.75      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=0.50      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=0.75      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=0.50      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=0.75      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  s | maxDets=100 | superCat=      all] = 0.505\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  m | maxDets=100 | superCat=      all] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  l | maxDets=100 | superCat=      all] = 0.756\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=  1 | superCat=      all] = 0.585\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets= 10 | superCat=      all] = 0.594\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=100 | superCat=      all] = 0.594\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  s | maxDets=100 | superCat=      all] = 0.500\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  m | maxDets=100 | superCat=      all] = 0.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  l | maxDets=100 | superCat=      all] = 0.756\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "# print out results\n",
    "fp_eval.print() # print out result using both Iou AND F1 constraint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "results with IoU thresholds (f1_threshold = -1.0)\n",
      "================================================================================\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=100 | superCat=outerwear] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=100 | superCat=accessory] = 0.626\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=100 | superCat=     part] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=none      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=none      |area=all | maxDets=100 | superCat=      all] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=  s | maxDets=100 | superCat=      all] = 0.505\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=  m | maxDets=100 | superCat=      all] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=  l | maxDets=100 | superCat=      all] = 0.800\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=  1 | superCat=      all] = 0.500\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets= 10 | superCat=      all] = 0.562\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=100 | superCat=      all] = 0.562\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=  s | maxDets=100 | superCat=      all] = 0.500\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=  m | maxDets=100 | superCat=      all] = 0.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=  l | maxDets=100 | superCat=      all] = 0.800\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "# print out result using IoU only, \n",
    "# here results should be the same as what coco API produces\n",
    "fp_eval.print(f1=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "results with F1 thresholds (iou_threshold = -1.0)\n",
      "================================================================================\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=100 | superCat=outerwear] = 1.000\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=100 | superCat=     part] = 0.240\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=100 | superCat=      all] = 0.699\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50      |area=all | maxDets=100 | superCat=      all] = 0.688\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.75      |area=all | maxDets=100 | superCat=      all] = 0.688\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=  s | maxDets=100 | superCat=      all] = 1.000\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=  m | maxDets=100 | superCat=      all] = 0.476\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=  l | maxDets=100 | superCat=      all] = 0.756\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=  1 | superCat=      all] = 0.642\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets= 10 | superCat=      all] = 0.698\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=100 | superCat=      all] = 0.698\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=  s | maxDets=100 | superCat=      all] = 1.000\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=  m | maxDets=100 | superCat=      all] = 0.476\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=  l | maxDets=100 | superCat=      all] = 0.756\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "fp_eval.print(iou=False) # print out result using f1 only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [Optional] Get per-class and/or per-superclass results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp_eval.summarize_class(perSuperClass=False, perCls=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "shirt, blouse results with both IoU and F1 thresholds\n",
      "================================================================================\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=0.50:0.95 |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=0.50:0.95 |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.75      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=0.50      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=0.75      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=0.50      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=0.75      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  s | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  m | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  l | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=  1 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets= 10 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=all | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  s | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  m | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=0.50:0.95 |area=  l | maxDets=100 ] = 1.000\n",
      "================================================================================\n",
      "================================================================================\n",
      "shirt, blouse results with IoU thresholds (f1_threshold = -1.0)\n",
      "================================================================================\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50      | F1=none      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.75      | F1=none      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=  s | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=  m | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | F1=none      |area=  l | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=  1 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets= 10 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=all | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=  s | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=  m | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | F1=none      |area=  l | maxDets=100 ] = 1.000\n",
      "================================================================================\n",
      "================================================================================\n",
      "shirt, blouse results with F1 thresholds (iou_threshold = -1.0)\n",
      "================================================================================\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.75      |area=all | maxDets=100 ] = 1.000\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=  s | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=  m | maxDets=100 ] = -1.000\n",
      " Average Precision  (AP) @[ IoU=none      | F1=0.50:0.95 |area=  l | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=  1 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets= 10 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=all | maxDets=100 ] = 1.000\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=  s | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=  m | maxDets=100 ] = -1.000\n",
      " Average Recall     (AR) @[ IoU=none      | F1=0.50:0.95 |area=  l | maxDets=100 ] = 1.000\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "# check out results for individual class\n",
    "class_name = 'shirt, blouse'\n",
    "\n",
    "fp_eval.print_class_result(class_name, f1=True, iou=True)\n",
    "fp_eval.print_class_result(class_name, f1=False, iou=True)\n",
    "fp_eval.print_class_result(class_name, f1=True, iou=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sanity check with COCO API\n",
    "COCO get detection result only, equivalent to $AP_{IoU}$ in Fashionpedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Loading and preparing results...\n",
      "DONE (t=0.00s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=0.02s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=0.08s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.532\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.505\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.000\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.800\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.500\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.562\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.562\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.500\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.000\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.800\n"
     ]
    }
   ],
   "source": [
    "cocoGt=COCO(anno_file)\n",
    "cocoDt=cocoGt.loadRes(res_file)\n",
    "imgIds=sorted(cocoGt.getImgIds())\n",
    "\n",
    "cocoEval = COCOeval(cocoGt,cocoDt,annType)\n",
    "cocoEval.params.imgIds  = imgIds\n",
    "cocoEval.evaluate()\n",
    "cocoEval.accumulate()\n",
    "cocoEval.summarize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}