multi process batch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import PIL\n",
    "from keras.applications import VGG16,imagenet_utils\n",
    "from keras.preprocessing.image import img_to_array\n",
    "from keras.preprocessing.image import load_img\n",
    "from keras.models import Model\n",
    "import numpy as np\n",
    "import glob\n",
    "import os\n",
    "from IPython.display import Image,display\n",
    "\n",
    "import pickle\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bbox = pd.read_table('Data/DeepFashion/list_bbox.txt',sep='\\s+')\n",
    "cat = pd.read_table('Data/DeepFashion/list_category_img.txt',sep='\\s+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41    72158\n",
       "18    36887\n",
       "3     24557\n",
       "32    19666\n",
       "17    15429\n",
       "33    14773\n",
       "6     13311\n",
       "16    13123\n",
       "11    10467\n",
       "19    10078\n",
       "2      7495\n",
       "48     7408\n",
       "26     7076\n",
       "42     6153\n",
       "30     5013\n",
       "29     4416\n",
       "10     4048\n",
       "34     3048\n",
       "44     2294\n",
       "39     2120\n",
       "24     1669\n",
       "35     1106\n",
       "15      791\n",
       "12      748\n",
       "9       716\n",
       "13      676\n",
       "27      594\n",
       "22      527\n",
       "23      486\n",
       "36      386\n",
       "5       330\n",
       "7       324\n",
       "4       309\n",
       "1       160\n",
       "47      150\n",
       "20      146\n",
       "43      126\n",
       "14       97\n",
       "21       77\n",
       "46       70\n",
       "37       54\n",
       "25       49\n",
       "28       45\n",
       "31       32\n",
       "8        17\n",
       "40       17\n",
       "Name: category_label, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat['category_label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inputShape = (224, 224)\n",
    "preprocess = imagenet_utils.preprocess_input\n",
    "def crop_resize_DF(bbox):\n",
    "    img = PIL.Image.open('Data/DeepFashion/'+bbox['image_name'])\n",
    "    lx = bbox['x_1']\n",
    "    ly = bbox['y_1']\n",
    "    ux = bbox['x_2']\n",
    "    uy = bbox['y_2']\n",
    "    \n",
    "    img = img.crop((lx,ly,ux,uy))\n",
    "    img = img.resize(inputShape, PIL.Image.ANTIALIAS)\n",
    "    return img\n",
    "\n",
    "def preprocess_DF(bbox):\n",
    "    image = crop_resize_DF(bbox)\n",
    "    image = img_to_array(image)\n",
    "    image = np.expand_dims(image,axis=0)\n",
    "    \n",
    "    return preprocess(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14.145 s\n"
     ]
    }
   ],
   "source": [
    "N = 2000\n",
    "start = time.time()\n",
    "subset = np.random.randint(bbox.shape[0],size=N)\n",
    "proc_img = []\n",
    "for i in subset:\n",
    "    proc_img.append(np.squeeze(preprocess_DF(bbox.iloc[i])))\n",
    "proc_img = np.array(proc_img)\n",
    "print('{:.3f} s'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from multiprocessing import Pool\n",
    "p = Pool()\n",
    "def proc(i):\n",
    "    return preprocess_DF(bbox.iloc[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.784 s\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "r = np.squeeze(p.map(proc,subset))\n",
    "print('{:.3f} s'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 224, 224, 3)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.squeeze(r).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>x_1</th>\n",
       "      <th>y_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>y_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>89273</th>\n",
       "      <td>img/Faux_Fur-Lined_Parka/img_00000063.jpg</td>\n",
       "      <td>33</td>\n",
       "      <td>31</td>\n",
       "      <td>201</td>\n",
       "      <td>295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140986</th>\n",
       "      <td>img/Mid-Rise_-_Distressed_Skinny_Jeans/img_000...</td>\n",
       "      <td>111</td>\n",
       "      <td>2</td>\n",
       "      <td>211</td>\n",
       "      <td>283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63125</th>\n",
       "      <td>img/Champagne_Slub_Knit_Top/img_00000008.jpg</td>\n",
       "      <td>100</td>\n",
       "      <td>72</td>\n",
       "      <td>208</td>\n",
       "      <td>266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47865</th>\n",
       "      <td>img/Tasseled_Tribal_Print_Cardigan/img_0000004...</td>\n",
       "      <td>77</td>\n",
       "      <td>34</td>\n",
       "      <td>214</td>\n",
       "      <td>263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8545</th>\n",
       "      <td>img/Abstract_Paint_Splatter_Blouse/img_0000006...</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>200</td>\n",
       "      <td>268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20171</th>\n",
       "      <td>img/Floral_Crochet_Cutout_Blouse/img_00000008.jpg</td>\n",
       "      <td>43</td>\n",
       "      <td>1</td>\n",
       "      <td>264</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157497</th>\n",
       "      <td>img/Shimmering_High-Waist_Leggings/img_0000002...</td>\n",
       "      <td>95</td>\n",
       "      <td>1</td>\n",
       "      <td>205</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285986</th>\n",
       "      <td>img/Collarless_A-Line_Boucl&amp;eacute;_Coat/img_0...</td>\n",
       "      <td>35</td>\n",
       "      <td>36</td>\n",
       "      <td>162</td>\n",
       "      <td>208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279957</th>\n",
       "      <td>img/Crisscross-Back_Skater_Dress/img_00000054.jpg</td>\n",
       "      <td>66</td>\n",
       "      <td>13</td>\n",
       "      <td>244</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273067</th>\n",
       "      <td>img/Ornate_Lace_Romper/img_00000007.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>48</td>\n",
       "      <td>162</td>\n",
       "      <td>261</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               image_name  x_1  y_1  x_2  y_2\n",
       "89273           img/Faux_Fur-Lined_Parka/img_00000063.jpg   33   31  201  295\n",
       "140986  img/Mid-Rise_-_Distressed_Skinny_Jeans/img_000...  111    2  211  283\n",
       "63125        img/Champagne_Slub_Knit_Top/img_00000008.jpg  100   72  208  266\n",
       "47865   img/Tasseled_Tribal_Print_Cardigan/img_0000004...   77   34  214  263\n",
       "8545    img/Abstract_Paint_Splatter_Blouse/img_0000006...    1   16  200  268\n",
       "20171   img/Floral_Crochet_Cutout_Blouse/img_00000008.jpg   43    1  264  300\n",
       "157497  img/Shimmering_High-Waist_Leggings/img_0000002...   95    1  205  300\n",
       "285986  img/Collarless_A-Line_Boucl&eacute;_Coat/img_0...   35   36  162  208\n",
       "279957  img/Crisscross-Back_Skater_Dress/img_00000054.jpg   66   13  244  279\n",
       "273067            img/Ornate_Lace_Romper/img_00000007.jpg    1   48  162  261"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bbox.iloc[subset]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}