SSD_debug.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "import os\n",
    "\n",
    "from ssd_model import SSD300, SSD512\n",
    "from ssd_utils import PriorUtil\n",
    "from ssd_data import InputGenerator\n",
    "from ssd_data import preprocess\n",
    "from ssd_training import SSDLoss\n",
    "from utils.model import load_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from data_voc import GTUtility\n",
    "gt_util = GTUtility('data/VOC2007/')\n",
    "\n",
    "gt_util_train, gt_util_val = gt_util.split(0.8)\n",
    "\n",
    "experiment = 'ssd_voc'\n",
    "num_classes = gt_util.num_classes\n",
    "\n",
    "#model = SSD300(num_classes=num_classes)\n",
    "model = SSD512(num_classes=num_classes)\n",
    "\n",
    "image_size = model.image_size\n",
    "\n",
    "_, inputs, images, data = gt_util_train.sample_random_batch(batch_size=16, input_size=image_size)\n",
    "\n",
    "test_idx = 5\n",
    "test_img = images[test_idx]\n",
    "test_gt = data[test_idx]\n",
    "\n",
    "plt.imshow(test_img)\n",
    "gt_util.plot_gt(test_gt, show_labels=True)\n",
    "plt.show()\n",
    "\n",
    "prior_util = PriorUtil(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "egt = prior_util.encode(test_gt)\n",
    "idxs = np.where(np.logical_not(egt[:,4]))[0]\n",
    "egt[idxs][:,:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#x = prior_util.encode(test_gt)\n",
    "#y = prior_util.decode(x)\n",
    "\n",
    "\n",
    "self = prior_util\n",
    "\n",
    "from utils.vis import to_rec\n",
    "\n",
    "def plot_assignment(self, map_idx):\n",
    "    ax = plt.gca()\n",
    "    im = plt.gci()\n",
    "    image_height, image_width = image_size = im.get_size()\n",
    "    \n",
    "    # ground truth\n",
    "    boxes = self.gt_boxes\n",
    "    boxes_x = (boxes[:,0] + boxes[:,2]) / 2. * image_height\n",
    "    boxes_y = (boxes[:,1] + boxes[:,3]) / 2. * image_width\n",
    "    for box in boxes:\n",
    "        xy_rec = to_rec(box[:4], image_size)\n",
    "        ax.add_patch(plt.Polygon(xy_rec, fill=False, edgecolor='b', linewidth=2))\n",
    "    plt.plot(boxes_x, boxes_y, 'bo',  markersize=8)\n",
    "    \n",
    "    # prior boxes\n",
    "    for idx, box_idx in self.match_indices.items():\n",
    "        if idx >= self.map_offsets[map_idx] and idx < self.map_offsets[map_idx+1]:\n",
    "            x, y = self.priors_xy[idx]\n",
    "            w, h = self.priors_wh[idx]\n",
    "            plt.plot(x, y, 'ro',  markersize=4)\n",
    "            plt.plot([x, boxes_x[box_idx]], [y, boxes_y[box_idx]], '-r', linewidth=1)\n",
    "            ax.add_patch(plt.Rectangle((x-w/2, y-h/2), w+1, h+1, \n",
    "                    fill=False, edgecolor='y', linewidth=2))\n",
    "\n",
    "\n",
    "\n",
    "from ssd_utils import iou\n",
    "\n",
    "def encode(self, gt_data, overlap_threshold=0.5, debug=False):\n",
    "    # calculation is done with normalized sizes\n",
    "\n",
    "    gt_boxes = self.gt_boxes = np.copy(gt_data[:,:4]) # normalized xmin, ymin, xmax, ymax\n",
    "    gt_labels = self.gt_labels = np.copy(gt_data[:,4:]) # one_hot classes including background\n",
    "\n",
    "    num_priors = self.priors.shape[0]\n",
    "    num_classes = gt_labels.shape[1]\n",
    "\n",
    "    # TODO: empty ground truth\n",
    "    if gt_data.shape[0] == 0:\n",
    "        print('gt_data', type(gt_data), gt_data.shape)\n",
    "\n",
    "    gt_iou = np.array([iou(b, self.priors_norm) for b in gt_boxes]).T\n",
    "    \n",
    "    # assigne gt to priors\n",
    "    max_idxs = np.argmax(gt_iou, axis=1)\n",
    "    max_val = gt_iou[np.arange(num_priors), max_idxs]\n",
    "    prior_mask = max_val > overlap_threshold\n",
    "    match_indices = max_idxs[prior_mask]\n",
    "\n",
    "    self.match_indices = dict(zip(list(np.argwhere(prior_mask)[:,0]), list(match_indices)))\n",
    "\n",
    "    # prior labels\n",
    "    confidence = np.zeros((num_priors, num_classes))\n",
    "    confidence[:,0] = 1\n",
    "    confidence[prior_mask] = gt_labels[match_indices]\n",
    "\n",
    "    # compute local offsets from ground truth boxes\n",
    "    gt_xy = (gt_boxes[:,2:4] + gt_boxes[:,0:2]) / 2.\n",
    "    gt_wh = gt_boxes[:,2:4] - gt_boxes[:,0:2]\n",
    "    gt_xy = gt_xy[match_indices]\n",
    "    gt_wh = gt_wh[match_indices]\n",
    "    priors_xy = self.priors_xy[prior_mask] / self.image_size\n",
    "    priors_wh = self.priors_wh[prior_mask] / self.image_size\n",
    "    offsets = np.zeros((num_priors, 4))\n",
    "    offsets[prior_mask, 0:2] = (gt_xy - priors_xy) / priors_wh\n",
    "    offsets[prior_mask, 2:4] = np.log(gt_wh / priors_wh)\n",
    "    offsets[prior_mask, :] /= self.priors[prior_mask,-4:] # variances\n",
    "\n",
    "    return np.concatenate([offsets, confidence], axis=1)\n",
    "    \n",
    "def decode(self, model_output, confidence_threshold=0.01, keep_top_k=200):\n",
    "    # calculation is done with normalized sizes\n",
    "\n",
    "    offsets = model_output[:,:4]\n",
    "    confidence = model_output[:,4:]\n",
    "\n",
    "    num_priors = offsets.shape[0]\n",
    "    num_classes = confidence.shape[1]\n",
    "\n",
    "    priors_xy = self.priors_xy / self.image_size\n",
    "    priors_wh = self.priors_wh / self.image_size\n",
    "\n",
    "    # compute bounding boxes from local offsets\n",
    "    boxes = np.empty((num_priors, 4))\n",
    "    offsets *= self.priors[:,-4:] # variances\n",
    "    boxes_xy = priors_xy + offsets[:,0:2] * priors_wh\n",
    "    boxes_wh = priors_wh * np.exp(offsets[:,2:4])\n",
    "    boxes[:,0:2] = boxes_xy - boxes_wh / 2. # xmin, ymin\n",
    "    boxes[:,2:4] = boxes_xy + boxes_wh / 2. # xmax, ymax\n",
    "    boxes = np.clip(boxes, 0.0, 1.0)\n",
    "\n",
    "    prior_mask = confidence > confidence_threshold\n",
    "\n",
    "    # TODO: number of confident boxes, compute bounding boxes only for those?\n",
    "    #print(np.sum(np.any(prior_mask[:,1:], axis=1)))\n",
    "\n",
    "    # do non maximum suppression\n",
    "    results = []\n",
    "    for c in range(1, num_classes):\n",
    "        mask = prior_mask[:,c]\n",
    "        boxes_to_process = boxes[mask]\n",
    "        if len(boxes_to_process) > 0:\n",
    "            confs_to_process = confidence[mask, c]\n",
    "            feed_dict = {\n",
    "                self.boxes: boxes_to_process,\n",
    "                self.scores: confs_to_process\n",
    "            }\n",
    "            idx = self.sess.run(self.nms, feed_dict=feed_dict)\n",
    "            good_boxes = boxes_to_process[idx]\n",
    "            good_confs = confs_to_process[idx][:, None]\n",
    "            labels = np.ones((len(idx),1)) * c\n",
    "            c_pred = np.concatenate((good_boxes, good_confs, labels), axis=1)\n",
    "            results.extend(c_pred)\n",
    "    results = np.array(results)\n",
    "    if len(results) > 0:\n",
    "        order = np.argsort(results[:, 1])[::-1]\n",
    "        results = results[order]\n",
    "        results = results[:keep_top_k]\n",
    "    self.results = results\n",
    "    return results\n",
    "\n",
    "x = encode(self, test_gt)\n",
    "y = decode(self, x)\n",
    "\n",
    "\n",
    "#for idx in range(len(prior_util.prior_maps)):\n",
    "for idx in [1,2,3,4]:\n",
    "    m = prior_util.prior_maps[idx]\n",
    "    plt.figure(figsize=[10]*2)\n",
    "    plt.imshow(test_img)\n",
    "    m.plot_locations()\n",
    "    #m.plot_boxes([0, 10, 100])\n",
    "    #gt_util.plot_gt(test_gt)\n",
    "    plot_assignment(self, idx)\n",
    "    prior_util.plot_results(y, show_labels=False)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "enc = prior_util.encode(test_gt)\n",
    "prior_util.decode(enc)[:,:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_gt[:,:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# plot ground truth\n",
    "for i in range(4):\n",
    "    plt.figure(figsize=[8]*2)\n",
    "    plt.imshow(images[i])\n",
    "    gt_util.plot_gt(data[i])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# plot prior boxes\n",
    "for i, m in enumerate(prior_util.prior_maps):\n",
    "    plt.figure(figsize=[8]*2)\n",
    "    #plt.imshow(images[7])\n",
    "    plt.imshow(images[8])\n",
    "    m.plot_locations()\n",
    "    #m.plot_boxes([0, 10, 100])\n",
    "    m.plot_boxes([0])\n",
    "    plt.axis('off')\n",
    "    #plt.savefig('plots/ssd_priorboxes_%i.pgf' % (i), bbox_inches='tight')\n",
    "    #print(m.map_size)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for l in model.layers:\n",
    "    try:\n",
    "        ks = l.weights[0].shape\n",
    "    except:\n",
    "        ks = \"\"\n",
    "    print(\"%30s %16s %24s %20s\" % (l.name, l.__class__.__name__, l.output_shape, ks))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 24\n",
    "\n",
    "gen = InputGenerator(gt_util_train, prior_util, batch_size, model.image_size, \n",
    "        augmentation=False,\n",
    "        vflip_prob=0.0, do_crop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "g = gen.generate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "for j in range(3000):\n",
    "    i, o = next(g)\n",
    "    print(j, len(i))\n",
    "    if len(i) != batch_size:\n",
    "        print('FOOOOOOOOOOO')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import cProfile\n",
    "\n",
    "g = gen.generate()\n",
    "\n",
    "p = cProfile.Profile()\n",
    "\n",
    "p.enable()\n",
    "\n",
    "i, o = next(g)\n",
    "\n",
    "p.disable()\n",
    "\n",
    "p.print_stats(sort='cumulative')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}