prep_binary_masks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install face-alignment package\n",
    "Environment: Google colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 139
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 46673,
     "status": "ok",
     "timestamp": 1529151506883,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "NK7_yFjNV-wY",
    "outputId": "091fda26-6e40-4c9d-fd39-2d48cdd7d14f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting torch\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/43/380514bd9663f1bf708abeb359b8b48d3fabb1c8e95bb3427a980a064c57/torch-0.4.0-cp36-cp36m-manylinux1_x86_64.whl (484.0MB)\n",
      "\u001b[K    100% |████████████████████████████████| 484.0MB 24kB/s \n",
      "tcmalloc: large alloc 1073750016 bytes == 0x5c3f8000 @  0x7fe09f1ac1c4 0x46d6a4 0x5fcbcc 0x4c494d 0x54f3c4 0x553aaf 0x54e4c8 0x54f4f6 0x553aaf 0x54efc1 0x54f24d 0x553aaf 0x54efc1 0x54f24d 0x553aaf 0x54efc1 0x54f24d 0x551ee0 0x54e4c8 0x54f4f6 0x553aaf 0x54efc1 0x54f24d 0x551ee0 0x54efc1 0x54f24d 0x551ee0 0x54e4c8 0x54f4f6 0x553aaf 0x54e4c8\n",
      "\u001b[?25hInstalling collected packages: torch\n",
      "Successfully installed torch-0.4.0\n"
     ]
    }
   ],
   "source": [
    "# Update pyorch to 0.4\n",
    "#!pip install --upgrade torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6783,
     "status": "ok",
     "timestamp": 1529151513699,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "n79ip5KbWH0s",
    "outputId": "34fd0b04-52d3-45a9-a4e8-feb48013d5b2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized empty Git repository in /content/.git/\n",
      "remote: Counting objects: 277, done.\u001b[K\n",
      "remote: Compressing objects: 100% (14/14), done.\u001b[K\n",
      "remote: Total 277 (delta 1), reused 3 (delta 0), pack-reused 263\u001b[K\n",
      "Receiving objects: 100% (277/277), 3.46 MiB | 13.68 MiB/s, done.\n",
      "Resolving deltas: 100% (156/156), done.\n",
      "From https://github.com/1adrianb/face-alignment\n",
      " * [new branch]      master     -> origin/master\n",
      " * [new tag]         v1.0.0     -> v1.0.0\n",
      "Branch master set up to track remote branch master from origin.\n",
      "Already on 'master'\n"
     ]
    }
   ],
   "source": [
    "#!git init .\n",
    "#!git remote add origin https://github.com/1adrianb/face-alignment.git\n",
    "#!git fetch origin\n",
    "#!git checkout master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "48vu2IoAXNcU"
   },
   "outputs": [],
   "source": [
    "#%%capture\n",
    "#!apt update\n",
    "#!apt install -y cmake"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Install dlib (CUDA enabled or CPU version)**\n",
    "\n",
    "CUDA enabled dlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%capture\n",
    "#!git clone https://github.com/davisking/dlib.git dlib/\n",
    "#cd dlib/\n",
    "#!python setup.py install –yes USE_AVX_INSTRUCTIONS –yes DLIB_USE_CUDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dlib w/o CUDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CPU dlib\n",
    "#!pip install dlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1784,
     "status": "ok",
     "timestamp": 1529131895182,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "c8b776O8WH5_",
    "outputId": "1adbfb1a-1399-4396-c05c-8685573ee25a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datalab     examples\t    README.md\t      setup.cfg  tox.ini\r\n",
      "Dockerfile  face_alignment  README.rst\t      setup.py\r\n",
      "docs\t    LICENSE\t    requirements.txt  test\r\n"
     ]
    }
   ],
   "source": [
    "#!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7848,
     "status": "ok",
     "timestamp": 1529131778373,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "HLqaUVyQWH3V",
    "outputId": "3c814038-afca-4934-bd40-c2e7b6a344e5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.4.0'"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#import torch\n",
    "#torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "QNdDVGxkblKX"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RT_PlazfbldI"
   },
   "source": [
    "# Generate binary masks\n",
    "\n",
    "```bash\n",
    "    Inputs:\n",
    "        Images from ./faceA and ./faceB\n",
    "    Outputs:\n",
    "        Eyes binary masks, which are saved to ./binary_masks/faceA_eyes and ./binary_masks/faceB_eyes respectively\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "zD5XgAEJbkjS"
   },
   "outputs": [],
   "source": [
    "import face_alignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "hmSj4zaXXz5W"
   },
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "from glob import glob\n",
    "from pathlib import PurePath, Path\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "34x3mp7wb2bR"
   },
   "outputs": [],
   "source": [
    "dir_faceA = \"./faceA\"\n",
    "dir_faceB = \"./faceB\"\n",
    "dir_bm_faceA_eyes = \"./binary_masks/faceA_eyes\"\n",
    "dir_bm_faceB_eyes = \"./binary_masks/faceB_eyes\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "17GYV20FXz78"
   },
   "outputs": [],
   "source": [
    "fns_faceA = glob(f\"{dir_faceA}/*.*\")\n",
    "fns_faceB = glob(f\"{dir_faceB}/*.*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11767,
     "status": "ok",
     "timestamp": 1529152245581,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "fwD2DVvQcEMw",
    "outputId": "698867a7-2b85-4703-cdd2-fa7531bc6b52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading the Face Alignment Network(FAN). Please wait...\n"
     ]
    }
   ],
   "source": [
    "fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, enable_cuda=True, flip_input=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "cYCCJK1ehpNi"
   },
   "outputs": [],
   "source": [
    "# !mkdir -p binary_masks/faceA_eyes\n",
    "Path(f\"binary_masks/faceA_eyes\").mkdir(parents=True, exist_ok=True)\n",
    "# !mkdir -p binary_masks/faceB_eyes\n",
    "Path(f\"binary_masks/faceB_eyes\").mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 2397
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 121564,
     "status": "ok",
     "timestamp": 1529152370522,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "iMZHXBmzcEUY",
    "outputId": "2dc0737f-6990-40db-adb9-c5e2baf49ae9"
   },
   "outputs": [],
   "source": [
    "fns_face_not_detected = []\n",
    "\n",
    "for idx, fns in enumerate([fns_faceA, fns_faceB]):\n",
    "    if idx == 0:\n",
    "        save_path = dir_bm_faceA_eyes\n",
    "    elif idx == 1:\n",
    "        save_path = dir_bm_faceB_eyes     \n",
    "    \n",
    "    # create binary mask for each training image\n",
    "    for fn in fns:\n",
    "        raw_fn = PurePath(fn).parts[-1]\n",
    "\n",
    "        x = plt.imread(fn)\n",
    "        x = cv2.resize(x, (256,256))\n",
    "        preds = fa.get_landmarks(x)\n",
    "        \n",
    "        if preds is not None:\n",
    "            preds = preds[0]\n",
    "            mask = np.zeros_like(x)\n",
    "            \n",
    "            # Draw right eye binary mask\n",
    "            pnts_right = [(preds[i,0],preds[i,1]) for i in range(36,42)]\n",
    "            hull = cv2.convexHull(np.array(pnts_right)).astype(np.int32)\n",
    "            mask = cv2.drawContours(mask,[hull],0,(255,255,255),-1)\n",
    "\n",
    "            # Draw left eye binary mask\n",
    "            pnts_left = [(preds[i,0],preds[i,1]) for i in range(42,48)]\n",
    "            hull = cv2.convexHull(np.array(pnts_left)).astype(np.int32)\n",
    "            mask = cv2.drawContours(mask,[hull],0,(255,255,255),-1)\n",
    "\n",
    "            # Draw mouth binary mask\n",
    "            #pnts_mouth = [(preds[i,0],preds[i,1]) for i in range(48,60)]\n",
    "            #hull = cv2.convexHull(np.array(pnts_mouth)).astype(np.int32)\n",
    "            #mask = cv2.drawContours(mask,[hull],0,(255,255,255),-1)\n",
    "            \n",
    "            mask = cv2.dilate(mask, np.ones((13,13), np.uint8), iterations=1)\n",
    "            mask = cv2.GaussianBlur(mask, (7,7), 0)\n",
    "            \n",
    "        else:\n",
    "            mask = np.zeros_like(x)\n",
    "            print(f\"No faces were detected in image '{fn}''\")\n",
    "            fns_face_not_detected.append(fn)\n",
    "        \n",
    "        plt.imsave(fname=f\"{save_path}/{raw_fn}\", arr=mask, format=\"jpg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 559,
     "status": "ok",
     "timestamp": 1529152371122,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "uXgooPybq7PG",
    "outputId": "362246e2-9b58-487c-cb4e-99ce69136eda"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nuber of processed images: 694\n",
      "Number of image(s) with no face detected: 70\n"
     ]
    }
   ],
   "source": [
    "num_faceA = len(glob(dir_faceA+\"/*.*\"))\n",
    "num_faceB = len(glob(dir_faceB+\"/*.*\"))\n",
    "\n",
    "print(\"Nuber of processed images: \"+ str(num_faceA + num_faceB))\n",
    "print(\"Number of image(s) with no face detected: \" + str(len(fns_face_not_detected)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uRKzF42Wy0Ba"
   },
   "source": [
    "# Randomly diaplay a face image and its result binary mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 316
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1080,
     "status": "ok",
     "timestamp": 1529139911130,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "Ja4aOHNay0HH",
    "outputId": "c6e1b3d4-d427-4267-a33b-efc8882373e3"
   },
   "outputs": [],
   "source": [
    "face = np.random.choice([\"A\",\"B\"])\n",
    "\n",
    "dir_face = dir_faceA if face == \"A\" else dir_faceB\n",
    "fns_face = fns_faceA if face == \"A\" else fns_faceB\n",
    "num_face = len(glob(dir_face+\"/*.*\"))\n",
    "rand_idx = np.random.randint(num_face)\n",
    "rand_fn = fns_face[rand_idx]\n",
    "raw_fn = PurePath(rand_fn).parts[-1]\n",
    "mask_fn = f\"{dir_bm_faceA_eyes}/{raw_fn}\" if face == \"A\" else f\"{dir_bm_faceB_eyes}/{raw_fn}\"\n",
    "im = plt.imread(rand_fn)\n",
    "mask = plt.imread(mask_fn)\n",
    "\n",
    "if rand_fn in fns_face_not_detected:\n",
    "    print(\"========== No faces were detected in this image! ==========\")\n",
    "\n",
    "fig = plt.figure(figsize=(15,6))\n",
    "plt.subplot(1,3,1)\n",
    "plt.grid('off')\n",
    "plt.imshow(im)\n",
    "plt.subplot(1,3,2)\n",
    "plt.grid('off')\n",
    "plt.imshow(mask)\n",
    "plt.subplot(1,3,3)\n",
    "plt.grid('off')\n",
    "plt.imshow((mask/255*im).astype(np.uint8))\n",
    "\n",
    "#fa.get_landmarks(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X6c22xxGsVQR"
   },
   "source": [
    "# Randomly diaplay an image that has no face detected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 697,
     "status": "ok",
     "timestamp": 1529139782680,
     "user": {
      "displayName": "Lu SA",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "109275333798683015269"
     },
     "user_tz": -480
    },
    "id": "aAgOLJ_WowlK",
    "outputId": "00138071-ab9e-4c91-aeb6-67d253c81cf5"
   },
   "outputs": [],
   "source": [
    "num_no_face_img = len(fns_face_not_detected)\n",
    "rand_idx = np.random.randint(num_no_face_img)\n",
    "x = plt.imread(fns_face_not_detected[rand_idx])\n",
    "#x = cv2.resize(x, (256,256))\n",
    "\n",
    "plt.grid('off')\n",
    "plt.imshow(x)\n",
    "\n",
    "#fa.get_landmarks(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "make_binary_masks.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}