Deep Fashion.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import PIL\n",
    "from keras.applications import VGG16,imagenet_utils\n",
    "from keras.preprocessing.image import img_to_array\n",
    "from keras.preprocessing.image import load_img\n",
    "from keras.models import Model\n",
    "import numpy as np\n",
    "import glob\n",
    "import os\n",
    "from IPython.display import Image,display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bbox = pd.read_table('Data/DeepFashion/list_bbox.txt',sep='\\s+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bbox.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.patches as patches\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "\n",
    "#i = np.random.randint(bbox.shape[0])\n",
    "\n",
    "i=8873\n",
    "im = np.array(Image.open('Data/DeepFashion/'+bbox['image_name'][i]), dtype=np.uint8)\n",
    "\n",
    "# Create figure and axes\n",
    "fig,ax = plt.subplots(1)\n",
    "\n",
    "# Display the image\n",
    "ax.imshow(im)\n",
    "\n",
    "lx = bbox['x_1'][i]\n",
    "ly = bbox['y_1'][i]\n",
    "w = bbox['x_2'][i] - lx\n",
    "h = bbox['y_2'][i] - ly\n",
    "\n",
    "# Create a Rectangle patch\n",
    "rect = patches.Rectangle((lx,ly),w,h,linewidth=1,edgecolor='r',facecolor='none')\n",
    "\n",
    "# Add the patch to the Axes\n",
    "ax.add_patch(rect)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cropDF(bbox):\n",
    "    im = np.array(PIL.Image.open('Data/DeepFashion/'+bbox['image_name']), dtype=np.uint8)\n",
    "    lx = bbox['x_1']\n",
    "    ly = bbox['y_1']\n",
    "    ux = bbox['x_2']\n",
    "    uy = bbox['y_2']\n",
    "    return im[ly:uy,lx:ux]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(1)\n",
    "\n",
    "ax.imshow(cropDF(bbox.iloc[56]))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.applications import VGG16,imagenet_utils\n",
    "from keras.preprocessing.image import img_to_array\n",
    "from keras.preprocessing.image import load_img\n",
    "from keras.models import Model\n",
    "import numpy as np\n",
    "import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inputShape = (224, 224)\n",
    "preprocess = imagenet_utils.preprocess_input\n",
    "model = VGG16(weights='imagenet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def crop_resize_DF(bbox):\n",
    "    img = PIL.Image.open('Data/DeepFashion/'+bbox['image_name'])\n",
    "    lx = bbox['x_1']\n",
    "    ly = bbox['y_1']\n",
    "    ux = bbox['x_2']\n",
    "    uy = bbox['y_2']\n",
    "    \n",
    "    img = img.crop((lx,ly,ux,uy))\n",
    "    img = img.resize(inputShape, PIL.Image.ANTIALIAS)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(1)\n",
    "\n",
    "ax.imshow(crop_resize_DF(bbox.iloc[56]))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def predictLabel(bbox):\n",
    "    image = crop_resize_DF(bbox)\n",
    "    file = 'Data/DeepFashion/'+bbox['image_name']\n",
    "    image = np.expand_dims(image,axis=0)\n",
    "\n",
    "    image = preprocess(image)\n",
    "    preds = model.predict(image)\n",
    "    P = imagenet_utils.decode_predictions(preds)\n",
    "    display(Image(filename=file,width=inputShape[0],height=inputShape[1]))\n",
    "    for (i, (imagenetID, label, prob)) in enumerate(P[0]):\n",
    "        # other way\n",
    "        print(f'{i+1}. {label}: {prob*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i = np.random.randint(bbox.shape[0])\n",
    "predictLabel(bbox.iloc[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_features(model,bbox):\n",
    "    image = crop_resize_DF(bbox)\n",
    "    image = img_to_array(image)\n",
    "    image = np.expand_dims(image,axis=0)\n",
    "    \n",
    "    image = preprocess(image)\n",
    "    feature = model.predict(image)\n",
    "    return np.squeeze(feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def preprocess_DF(bbox):\n",
    "    image = crop_resize_DF(bbox)\n",
    "    image = img_to_array(image)\n",
    "    image = np.expand_dims(image,axis=0)\n",
    "    \n",
    "    return preprocess(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fc6_model = Model(inputs=model.input,outputs=model.get_layer('fc1').output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = 1000\n",
    "subset = np.random.randint(bbox.shape[0],size=N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "start = time.time()\n",
    "proc_imgs = []\n",
    "for i in subset:\n",
    "    proc_imgs.append(np.squeeze(preprocess_DF(bbox.iloc[i])))\n",
    "proc_imgs = np.array(proc_imgs)\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "feature_matrix = fc6_model.predict(proc_imgs)\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PIL import ImageDraw\n",
    "def draw_bbox(bbox):\n",
    "    img = PIL.Image.open('Data/DeepFashion/'+bbox['image_name']).convert('RGBA')\n",
    "    \n",
    "    lx = bbox['x_1']\n",
    "    ly = bbox['y_1']\n",
    "    ux = bbox['x_2']\n",
    "    uy = bbox['y_2']\n",
    "    rect = PIL.Image.new('RGBA', img.size, (255,255,255,0))\n",
    "    d = ImageDraw.Draw(img)\n",
    "    d.rectangle(((lx,ly), (ux,uy)), outline=\"red\")\n",
    "    return PIL.Image.alpha_composite(img,rect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_match(i,image_df,model,feature_matrix):\n",
    "    img_feature = model.predict(preprocess_DF(image_df.iloc[i]))\n",
    "    L2 = np.sum((feature_matrix-img_feature)**2,axis=1)\n",
    "    match = np.argmin(L2)\n",
    "\n",
    "    print(f'distance: {np.sqrt(np.min(L2))}')\n",
    "    display(Image(filename='Data/DeepFashion/'+image_df.iloc[i]['image_name'],\n",
    "                  width=inputShape[0],height=inputShape[1]))\n",
    "    display(Image(filename='Data/DeepFashion/'+image_df.iloc[match]['image_name'],\n",
    "                  width=inputShape[0],height=inputShape[1]))\n",
    "    \n",
    "def gen_pairs(N,image_df,feature_matrix):\n",
    "    img_index = np.random.randint(image_df.shape[0],size=N)\n",
    "    for i in img_index:\n",
    "        img_feature = feature_matrix[i,:]\n",
    "        L2 = np.sum((feature_matrix-img_feature)**2,axis=1)\n",
    "        match = np.argsort(L2)[1]\n",
    "\n",
    "        print(f'distance: {np.sqrt(L2[match])}')\n",
    "        display(draw_bbox(image_df.iloc[i]))\n",
    "        display(draw_bbox(image_df.iloc[match]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gen_pairs(3,bbox.iloc[subset],feature_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "tsne = TSNE()\n",
    "fc6_embed = tsne.fit_transform(feature_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from matplotlib.offsetbox import OffsetImage, AnnotationBbox\n",
    "\n",
    "# Plots an image at each x and y location. \n",
    "def plotImage(x, y, image, ax):\n",
    "    im = OffsetImage(load_img(image,target_size=inputShape), zoom=0.15)\n",
    "    ab = AnnotationBbox(im, (x, y), xycoords='data', frameon=False)\n",
    "    ax.add_artist(ab)\n",
    "\n",
    "# Create figure\n",
    "def plot_tsne(N,embed,image_df,file):\n",
    "    fig = plt.figure(figsize=(20,20),dpi=300)\n",
    "    ax = fig.add_subplot(111)\n",
    "\n",
    "    for i in range(N):\n",
    "        plotImage(embed[i,0],embed[i,1],\n",
    "                  'Data/DeepFashion/'+image_df.iloc[i]['image_name'],ax)\n",
    "\n",
    "    ax.set_ylim(np.min(embed[:N,1]),np.max(embed[:N,1]))\n",
    "    ax.set_xlim(np.min(embed[:N,0]),np.max(embed[:N,0]))\n",
    "\n",
    "    plt.savefig(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plot_tsne(fc6_embed,bbox.iloc[subset],'DF_1000_tsne.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cat = pd.read_table('Data/DeepFashion/list_category_img.txt',sep='\\s+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tops = bbox.iloc[cat[(cat.category_label>=1) & (cat.category_label<=20)].index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = 2000\n",
    "subset = np.random.randint(tops.shape[0],size=N)\n",
    "proc_imgs = []\n",
    "for i in subset:\n",
    "    proc_imgs.append(np.squeeze(preprocess_DF(tops.iloc[i])))\n",
    "proc_imgs = np.array(proc_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "feature_matrix_tops = fc6_model.predict(proc_imgs)\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gen_pairs(3,tops.iloc[subset],feature_matrix_tops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fc6_tops_embed = tsne.fit_transform(feature_matrix_tops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_tsne(500,fc6_tops_embed,tops.iloc[subset],'output/DF_tops_500_tsne.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tops = bbox.iloc[cat[(cat.category_label>=1) & (cat.category_label<=20)].index]\n",
    "bottoms = bbox.iloc[cat[(cat.category_label>=21) & (cat.category_label<=36)].index]\n",
    "dresses = bbox.iloc[cat[(cat.category_label>36)].index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = 10000\n",
    "subset_tops = np.random.randint(tops.shape[0],size=N)\n",
    "proc_tops = []\n",
    "for i in subset_tops:\n",
    "    proc_tops.append(np.squeeze(preprocess_DF(tops.iloc[i])))\n",
    "proc_tops = np.array(proc_tops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = 10000\n",
    "subset_bottoms = np.random.randint(bottoms.shape[0],size=N)\n",
    "proc_bottoms = []\n",
    "for i in subset_bottoms:\n",
    "    proc_bottoms.append(np.squeeze(preprocess_DF(bottoms.iloc[i])))\n",
    "proc_bottoms = np.array(proc_bottoms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = 10000\n",
    "subset_dresses = np.random.randint(dresses.shape[0],size=N)\n",
    "proc_dresses = []\n",
    "for i in subset_dresses:\n",
    "    proc_dresses.append(np.squeeze(preprocess_DF(dresses.iloc[i])))\n",
    "proc_dresses = np.array(proc_dresses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(tops.iloc[subset_tops] ,open('Data/feature_matrix/tops_10000_df.p','wb'))\n",
    "pickle.dump(bottoms.iloc[subset_bottoms] ,open('Data/feature_matrix/bottoms_10000_df.p','wb'))\n",
    "pickle.dump(dresses.iloc[subset_dresses] ,open('Data/feature_matrix/dresses_10000_df.p','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense\n",
    "\n",
    "model = VGG16(weights='imagenet')\n",
    "FC6_relu = Model(inputs=model.input,outputs=model.get_layer('fc1').output)\n",
    "FC8_linear = Model(inputs = model.input,\n",
    "                   output = Dense(1000, activation=None, name='FC8')(model.get_layer('predictions').input))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "fc6_tops = FC6_relu.predict(proc_tops)\n",
    "pickle.dump(FC6_relu_tops,open('Data/feature_matrix/FC6_relu_tops.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(fc6_tops,open('Data/feature_matrix/FC6_relu_tops.p','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "fc8_tops = FC8_linear.predict(proc_tops)\n",
    "pickle.dump(fc8_tops,open('Data/feature_matrix/FC8_linear_tops.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "fc6_bottoms = FC6_relu.predict(proc_bottoms)\n",
    "pickle.dump(fc6_bottoms,open('Data/feature_matrix/FC6_relu_bottoms.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "fc8_bottoms = FC8_linear.predict(proc_bottoms)\n",
    "pickle.dump(fc8_bottoms,open('Data/feature_matrix/FC8_linear_bottoms.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "fc6_dresses = FC6_relu.predict(proc_dresses)\n",
    "pickle.dump(FC6_relu_dresses,open('Data/feature_matrix/FC6_relu_dresses.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "fc8_dresses = FC8_linear.predict(proc_dresses)\n",
    "pickle.dump(FC8_linear_dresses,open('Data/feature_matrix/FC8_linear_dresses.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fc6_tops_embed = tsne.fit_transform(fc6_tops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plot_tsne(500,fc6_tops_embed,tops.iloc[subset_tops],'output/DF_tops_FC6_10000_tsne.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fc8_relu_tops = np.where(fc8_tops>0,fc8_tops,0)\n",
    "fc8_relu_tops_embed = tsne.fit_transform(fc8_relu_tops)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plot_tsne(1000,fc8_relu_tops_embed,tops.iloc[subset_tops],'output/DF_tops_FC8_Relu_10000_tsne.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense\n",
    "\n",
    "model = VGG16(weights='imagenet')\n",
    "\n",
    "FC8 = Dense(1000, activation=None, name='FC8')\n",
    "FC8_linear = Model(inputs = model.input,\n",
    "                   output = FC8(model.get_layer('predictions').input))\n",
    "FC8.set_weights(model.get_layer('predictions').get_weights())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import pickle\n",
    "start = time.time()\n",
    "fc8_tops = FC8_linear.predict(proc_tops)\n",
    "pickle.dump(fc8_tops,open('Data/feature_matrix/FC8_linear_tops_loaded_weights.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = VGG16(weights='imagenet')\n",
    "FC7 = Model(inputs = model.input, output = model.get_layer('fc2').output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "fc7_relu_tops = FC7.predict(proc_tops)\n",
    "pickle.dump(fc7_relu_tops,open('Data/feature_matrix/FC7_relu_tops.p','wb'))\n",
    "print(f'{time.time()-start:.2f} s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}