Run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "import os\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "import _pickle as pkl\n",
    "\n",
    "from utils import *\n",
    "from FCNN import FCNN\n",
    "from sklearn.linear_model import LinearRegression, Ridge, LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LFW+"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_address = './LFWA+/lfw'\n",
    "dataset = get_dataset(dataset_address)\n",
    "lfw_raw, label_list = get_image_paths_and_labels(dataset)\n",
    "with open(\"./dataset_description.pkl\",\"rb\") as foo:\n",
    "    dc = pkl.load(foo)\n",
    "lfw_list = []\n",
    "for im in dc['image_list']:\n",
    "    lfw_list.append(os.path.join(dataset_address, im))\n",
    "lfw_attributes = (dc['attributes']>0).astype(int)\n",
    "lfw_labels = (lfw_attributes[:,0]>0).astype(int)\n",
    "lfw_latent_vars = dc['latent_vars'] ##Latent representation of the image in the VAE trained on CelebA\n",
    "lfw_data = ((read_images(lfw_list) - 127.5)/128, lfw_labels, lfw_latent_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sex = lfw_attributes[:,0]\n",
    "skin = lfw_attributes[:,3]\n",
    "dictionary = dc\n",
    "permuted_idxs = np.random.permutation(np.arange(len(lfw_data[0])))\n",
    "idxs_val, idxs_test = permuted_idxs[:6263], permuted_idxs[6263:]\n",
    "x_val, y_val, a_val = lfw_data[0][idxs_val], lfw_data[1][idxs_val], lfw_attributes[:,[3,38,40]][idxs_val]\n",
    "x_test, y_test, a_test = lfw_data[0][idxs_test], lfw_data[1][idxs_test], lfw_attributes[:,[3,38,40]][idxs_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load trained network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1.0)\n",
    "gender_ckpt = \"./model-20180428-135113_male.ckpt-111\"\n",
    "net = FCNN()\n",
    "sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))\n",
    "sess.run(tf.global_variables_initializer())\n",
    "sess.run(tf.local_variables_initializer())\n",
    "net.load_model(sess, gender_ckpt, initialize = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Network & VAE Representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prelogits_val = np.zeros((len(x_val),128))\n",
    "batch_size = 24\n",
    "for i in range(int(np.ceil(len(x_val)/batch_size))):\n",
    "    prelogits_val[i*batch_size:i*batch_size+batch_size] = sess.run(\n",
    "        net.prelogits ,feed_dict={net.in_ph:x_val[i*batch_size:i*batch_size+batch_size]})\n",
    "lts_val = dictionary['latent_vars'][idxs_val] #VAE representation\n",
    "prelogits_test = np.zeros((len(x_test),128))\n",
    "for i in range(int(np.ceil(len(x_test)/batch_size))):\n",
    "    prelogits_test[i*batch_size:i*batch_size+batch_size] = sess.run(\n",
    "        net.prelogits, feed_dict={net.in_ph:x_test[i*batch_size:i*batch_size+batch_size]})\n",
    "lts_test = dictionary['latent_vars'][idxs_test] #VAE representation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Black-box audting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sess_run(result, x, l, sess):\n",
    "    num = x.shape[0]\n",
    "    num_batch = np.ceil(num/200).astype(int)\n",
    "    output = np.zeros(num)\n",
    "    for batch in range(num_batch):\n",
    "        output[batch*200:(batch+1)*200] = sess.run(result, feed_dict={net.in_ph:x[batch*200:(batch+1)*200],\n",
    "                                                                         latent_ph:l[batch*200:(batch+1)*200]})       \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "control = tf.cast(tf.greater(net.output[:,1],net.output[:,0]), tf.float32)\n",
    "noharm = [control, 1 - control, control + 1 - control]\n",
    "latent_val, latent_test = lts_val, lts_test\n",
    "dim = latent_val.shape[-1]\n",
    "latent_ph = tf.placeholder(tf.float32, shape=(None, dim), name=\"latent_var\")\n",
    "logits = net.output[:,1] - net.output[:,0]\n",
    "max_T = 100\n",
    "thresh = 1e-4 # Hyper-parameter\n",
    "temp_harm = []\n",
    "for s in noharm:\n",
    "    temp_harm.append(sess_run(s, x_val, latent_val, sess))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def res(p, y):\n",
    "    return y * ((p>=0.1)/(p + 1e-20) + (p<0.1) * (20 - 100  * p)) +\\\n",
    "(1-y) * ((p < 0.9)/(1 - p + 1e-20) + (p>=0.9) * (100 * p - 80))\n",
    "plt.plot(np.arange(0, 1.0, 0.001), res(np.arange(0., 1, 0.001), 0))\n",
    "plt.plot(np.arange(0, 1.0, 0.001), res(np.arange(0., 1, 0.001), 1))\n",
    "plt.legend(['0', '1'])\n",
    "plt.xlabel('P')\n",
    "plt.ylabel('Residual')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_epoch, best_acc = -1,0\n",
    "(idxs1, idxs2, _), _ = split_data(np.arange(len(idxs_val)), ratio=[0.7,0.3,0.])\n",
    "coeffs = []\n",
    "for t in range(max_T):\n",
    "    control = tf.cast(tf.greater(net.output[:,1], net.output[:,0]), tf.float32)\n",
    "    noharm = [control, 1 - control, control + 1 - control]\n",
    "    probs_heldout = sess_run(tf.nn.sigmoid(logits), x_val[idxs2], latent_val[idxs2], sess)\n",
    "    heldout_loss = np.mean(-y_val[idxs2] * np.log(probs_heldout + 1e-20) - (1-y_val[idxs2]) * np.log(1-probs_heldout + 1e-20))\n",
    "    heldout_acc =  np.mean((probs_heldout>0.5)==y_val[idxs2])\n",
    "    probs = sess_run(tf.nn.sigmoid(logits), x_val, latent_val ,sess)\n",
    "    val_loss = np.mean(-y_val * np.log(probs + 1e-20) - (1 - y_val) * np.log(1 - probs + 1e-20))\n",
    "    val_acc = np.mean((probs > 0.5) == y_val)\n",
    "    if heldout_acc > best_acc:\n",
    "        best_epoch = t\n",
    "        best_acc = heldout_acc\n",
    "        best_logits = logits\n",
    "    delta = res(probs,y_val)\n",
    "    residual = probs - y_val\n",
    "    for i, s in enumerate(noharm):\n",
    "        temp_s = sess_run(noharm[i], x_val[idxs1], latent_val[idxs1], sess)\n",
    "        temp_s_heldout = sess_run(noharm[i], x_val[idxs2], latent_val[idxs2], sess)\n",
    "        samples1 = np.where(temp_s == 1)[0]\n",
    "        samples2 = np.where(temp_s_heldout == 1)[0]\n",
    "        clf = Ridge(alpha=1)\n",
    "        clf.fit(latent_val[idxs1][samples1],delta[idxs1][samples1])\n",
    "        clf_prediction = clf.predict(latent_val[idxs2][samples2])\n",
    "        corr = np.mean(clf_prediction * residual[idxs2][samples2])\n",
    "        print(t, i, corr)\n",
    "        if corr > 1e-4:\n",
    "            coeffs.append(clf.coef_)\n",
    "            h = (tf.matmul(latent_ph, tf.constant(np.expand_dims(clf.coef_,-1),\n",
    "                                                  dtype=tf.float32))[:,0] + clf.intercept_)\n",
    "            logits -= .1 * h * s\n",
    "            break\n",
    "    if i==2:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = sess_run(net.output[:,1] - net.output[:,0], x_test, latent_test, sess)\n",
    "groups = ['all', 'F', 'M', 'B', 'N', 'BF', 'BM', 'NF', 'NM']\n",
    "errs = []\n",
    "idxs = np.where((skin[idxs_test]>-1) * (sex[idxs_test]>-10))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]>-1) * (sex[idxs_test]==0))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]>-1) * (sex[idxs_test]==1))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==1) * (sex[idxs_test]>-10))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==0) * (sex[idxs_test]>-10))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==1) * (sex[idxs_test]==0))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==1) * (sex[idxs_test]==1))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==0) * (sex[idxs_test]==0))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==0) * (sex[idxs_test]==1))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "output = ''\n",
    "for group, err in zip(groups, errs):\n",
    "    output += group + ': ' + str(round(err, 1)) + ' & '\n",
    "print('Original: ', output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = sess_run(tf.nn.sigmoid(best_logits), x_test, latent_test, sess)\n",
    "groups = ['all', 'F', 'M', 'B', 'N', 'BF', 'BM', 'NF', 'NM']\n",
    "errs = []\n",
    "idxs = np.where((skin[idxs_test]>-1) * (sex[idxs_test]>-10))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]>-1) * (sex[idxs_test]==0))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]>-1) * (sex[idxs_test]==1))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==1) * (sex[idxs_test]>-10))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==0) * (sex[idxs_test]>-10))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==1) * (sex[idxs_test]==0))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==1) * (sex[idxs_test]==1))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==0) * (sex[idxs_test]==0))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "idxs = np.where((skin[idxs_test]==0) * (sex[idxs_test]==1))[0]\n",
    "errs.append(100 * np.mean((probs[idxs]>0.5)!=y_test[idxs]))\n",
    "output = ''\n",
    "for group, err in zip(groups, errs):\n",
    "    output += group + ': ' + str(round(err, 1)) + ' & '\n",
    "print('MultiAccuracy Boost: ', output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:modern]",
   "language": "python",
   "name": "conda-env-modern-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}