Visual_Feature_Engineering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Systematic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import clip\n",
    "from evaluation_utils import norm, denorm\n",
    "from general_utils import *\n",
    "from datasets.lvis_oneshot3 import LVIS_OneShot3\n",
    "\n",
    "clip_device = 'cuda'\n",
    "clip_model, preprocess = clip.load(\"ViT-B/16\", device=clip_device)\n",
    "clip_model.eval();\n",
    "\n",
    "from models.clipseg import CLIPDensePredTMasked\n",
    "\n",
    "clip_mask_model = CLIPDensePredTMasked(version='ViT-B/16').to(clip_device)\n",
    "clip_mask_model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lvis = LVIS_OneShot3('train_fixed', mask='separate', normalize=True, with_class_label=True, add_bar=False, \n",
    "                     text_class_labels=True, image_size=352, min_area=0.1,\n",
    "                     min_frac_s=0.05, min_frac_q=0.05, fix_find_crop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_data(lvis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import json\n",
    "\n",
    "lvis_raw = json.load(open(expanduser('~/datasets/LVIS/lvis_v1_train.json')))\n",
    "lvis_val_raw = json.load(open(expanduser('~/datasets/LVIS/lvis_v1_val.json')))\n",
    "\n",
    "objects_per_image = defaultdict(lambda : set())\n",
    "for ann in lvis_raw['annotations']:\n",
    "    objects_per_image[ann['image_id']].add(ann['category_id'])\n",
    "    \n",
    "for ann in lvis_val_raw['annotations']:\n",
    "    objects_per_image[ann['image_id']].add(ann['category_id'])    \n",
    "    \n",
    "objects_per_image = {o: [lvis.category_names[o] for o in v] for o, v in objects_per_image.items()}\n",
    "\n",
    "del lvis_raw, lvis_val_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bs = 32\n",
    "#batches = [get_batch(lvis, i*bs, (i+1)*bs, cuda=True) for i in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from general_utils import get_batch\n",
    "from functools import partial\n",
    "from evaluation_utils import img_preprocess\n",
    "import torch\n",
    "\n",
    "def get_similarities(batches_or_dataset, process, mask=lambda x: None, clipmask=False):\n",
    "\n",
    "    # base_words = [f'a photo of {x}' for x in ['a person', 'an animal', 'a knife', 'a cup']]\n",
    "\n",
    "    all_prompts = []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        valid_sims = []\n",
    "        torch.manual_seed(571)\n",
    "        \n",
    "        if type(batches_or_dataset) == list:\n",
    "            loader = batches_or_dataset  # already loaded\n",
    "            max_iter = float('inf')\n",
    "        else:\n",
    "            loader = DataLoader(batches_or_dataset, shuffle=False, batch_size=32)\n",
    "            max_iter = 50\n",
    "        \n",
    "        global batch\n",
    "        for i_batch, (batch, batch_y) in enumerate(loader):\n",
    "            \n",
    "            if i_batch >= max_iter: break\n",
    "                \n",
    "            processed_batch = process(batch)\n",
    "            if type(processed_batch) == dict:\n",
    "                \n",
    "                # processed_batch =  {k: v.to(clip_device) for k, v in processed_batch.items()}\n",
    "                image_features = clip_mask_model.visual_forward(**processed_batch)[0].to(clip_device).half()\n",
    "            else:\n",
    "                processed_batch = process(batch).to(clip_device)\n",
    "                processed_batch = nnf.interpolate(processed_batch, (224, 224), mode='bilinear')\n",
    "                #image_features = clip_model.encode_image(processed_batch.to(clip_device)) \n",
    "                image_features = clip_mask_model.visual_forward(processed_batch)[0].to(clip_device).half()\n",
    "                \n",
    "            image_features = image_features / image_features.norm(dim=-1, keepdim=True)\n",
    "            bs = len(batch[0])\n",
    "            for j in range(bs):\n",
    "            \n",
    "                c, _, sid, qid = lvis.sample_ids[bs * i_batch + j]\n",
    "                support_image = basename(lvis.samples[c][sid])\n",
    "                \n",
    "                img_objs = [o for o in objects_per_image[int(support_image)]]\n",
    "                img_objs = [o.replace('_', ' ') for o in img_objs]\n",
    "                \n",
    "                other_words = [f'a photo of a {o.replace(\"_\", \" \")}' for o in img_objs \n",
    "                               if o != batch_y[2][j]]\n",
    "            \n",
    "                prompts = [f'a photo of a {batch_y[2][j]}'] + other_words\n",
    "                all_prompts += [prompts]\n",
    "                \n",
    "                text_cond = clip_model.encode_text(clip.tokenize(prompts).to(clip_device))\n",
    "                text_cond = text_cond / text_cond.norm(dim=-1, keepdim=True)            \n",
    "\n",
    "                global logits\n",
    "                logits = clip_model.logit_scale.exp() * image_features[j] @ text_cond.T\n",
    "\n",
    "                global sim\n",
    "                sim = torch.softmax(logits, dim=-1)\n",
    "            \n",
    "                valid_sims += [sim]\n",
    "                \n",
    "        #valid_sims = torch.stack(valid_sims)\n",
    "        return valid_sims, all_prompts\n",
    "    \n",
    "\n",
    "def new_img_preprocess(x):\n",
    "    return {'x_inp': x[1], 'mask': (11, 'cls_token', x[2])}\n",
    "    \n",
    "#get_similarities(lvis, partial(img_preprocess, center_context=0.5));\n",
    "get_similarities(lvis, lambda x: x[1]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessing_functions = [\n",
    "#     ['clip mask CLS L11', lambda x: {'x_inp': x[1].cuda(), 'mask': (11, 'cls_token', x[2].cuda())}],\n",
    "#     ['clip mask CLS all', lambda x: {'x_inp': x[1].cuda(), 'mask': ('all', 'cls_token', x[2].cuda())}],\n",
    "#     ['clip mask all all', lambda x: {'x_inp': x[1].cuda(), 'mask': ('all', 'all', x[2].cuda())}],\n",
    "#     ['colorize object red', partial(img_preprocess, colorize=True)],\n",
    "#     ['add red outline', partial(img_preprocess, outline=True)],\n",
    "    \n",
    "#     ['BG brightness 50%', partial(img_preprocess, bg_fac=0.5)],\n",
    "#     ['BG brightness 10%', partial(img_preprocess, bg_fac=0.1)],\n",
    "#     ['BG brightness 0%', partial(img_preprocess, bg_fac=0.0)],\n",
    "#     ['BG blur', partial(img_preprocess, blur=3)],\n",
    "#     ['BG blur & intensity 10%', partial(img_preprocess, blur=3, bg_fac=0.1)],\n",
    "   \n",
    "#     ['crop large context', partial(img_preprocess, center_context=0.5)],\n",
    "#     ['crop small context', partial(img_preprocess, center_context=0.1)],\n",
    "    ['crop & background blur', partial(img_preprocess, blur=3, center_context=0.5)],\n",
    "    ['crop & intensity 10%', partial(img_preprocess, blur=3, bg_fac=0.1)],\n",
    "#     ['crop & background blur & intensity 10%', partial(img_preprocess, blur=3, center_context=0.1, bg_fac=0.1)],\n",
    "]\n",
    "\n",
    "preprocessing_functions = preprocessing_functions\n",
    "\n",
    "base, base_p = get_similarities(lvis, lambda x: x[1])\n",
    "outs = [get_similarities(lvis, fun) for _, fun in preprocessing_functions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outs2 = [get_similarities(lvis, fun) for _, fun in  [['BG brightness 0%', partial(img_preprocess, bg_fac=0.0)]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(1):\n",
    "    print(np.mean([outs2[j][0][i][0].cpu() - base[i][0].cpu() for i in range(len(base)) if len(base_p[i]) >= 3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import DataFrame\n",
    "tab = dict()\n",
    "for j, (name, _) in enumerate(preprocessing_functions):\n",
    "    tab[name] =  np.mean([outs[j][0][i][0].cpu() - base[i][0].cpu() for i in range(len(base)) if len(base_p[i]) >= 3])\n",
    "    \n",
    "    \n",
    "print('\\n'.join(f'{k} & {v*100:.2f} \\\\\\\\' for k,v in tab.items()))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation_utils import denorm, norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_sample(filename, filename2):\n",
    "    from os.path import join\n",
    "    bp = expanduser('~/cloud/resources/sample_images')\n",
    "    tf = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "        transforms.Resize(224),\n",
    "        transforms.CenterCrop(224)\n",
    "    ])\n",
    "    tf2 = transforms.Compose([\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Resize(224),\n",
    "        transforms.CenterCrop(224)\n",
    "    ])\n",
    "    inp1 = [None, tf(Image.open(join(bp, filename))), tf2(Image.open(join(bp, filename2)))]\n",
    "    inp1[1] = inp1[1].unsqueeze(0)\n",
    "    inp1[2] = inp1[2][:1]   \n",
    "    return inp1\n",
    "\n",
    "def all_preprocessing(inp1):\n",
    "    return [\n",
    "        img_preprocess(inp1),\n",
    "        img_preprocess(inp1, colorize=True),\n",
    "        img_preprocess(inp1, outline=True),        \n",
    "        img_preprocess(inp1, blur=3),\n",
    "        img_preprocess(inp1, bg_fac=0.1),\n",
    "        #img_preprocess(inp1, bg_fac=0.5),\n",
    "        #img_preprocess(inp1, blur=3, bg_fac=0.5),        \n",
    "        img_preprocess(inp1, blur=3, bg_fac=0.5, center_context=0.5),\n",
    "    ]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "from PIL import Image\n",
    "from matplotlib import pyplot as plt\n",
    "from evaluation_utils import img_preprocess\n",
    "import clip\n",
    "\n",
    "images_queries = [\n",
    "    [load_sample('things1.jpg', 'things1_jar.png'), ['jug', 'knife', 'car', 'animal', 'sieve', 'nothing']],\n",
    "    [load_sample('own_photos/IMG_2017s_square.jpg', 'own_photos/IMG_2017s_square_trash_can.png'), ['trash bin', 'house', 'car', 'bike', 'window', 'nothing']],\n",
    "]\n",
    "\n",
    "\n",
    "_, ax = plt.subplots(2 * len(images_queries), 6, figsize=(14, 4.5 * len(images_queries)))\n",
    "\n",
    "for j, (images, objects) in enumerate(images_queries):\n",
    "    \n",
    "    joint_image = all_preprocessing(images)\n",
    "    \n",
    "    joint_image = torch.stack(joint_image)[:,0]\n",
    "    clip_model, preprocess = clip.load(\"ViT-B/16\", device='cpu')\n",
    "    image_features = clip_model.encode_image(joint_image)\n",
    "    image_features = image_features / image_features.norm(dim=-1, keepdim=True)\n",
    "    \n",
    "    prompts = [f'a photo of a {obj}'for obj in objects]\n",
    "    text_cond = clip_model.encode_text(clip.tokenize(prompts))\n",
    "    text_cond = text_cond / text_cond.norm(dim=-1, keepdim=True)\n",
    "    logits = clip_model.logit_scale.exp() * image_features @ text_cond.T\n",
    "    sim = torch.softmax(logits, dim=-1).detach().cpu()\n",
    "\n",
    "    for i, img in enumerate(joint_image):\n",
    "        ax[2*j, i].axis('off')\n",
    "        \n",
    "        ax[2*j, i].imshow(torch.clamp(denorm(joint_image[i]).permute(1,2,0), 0, 1))\n",
    "        ax[2*j+ 1, i].grid(True)\n",
    "        \n",
    "        ax[2*j + 1, i].set_ylim(0,1)\n",
    "        ax[2*j + 1, i].set_yticklabels([])\n",
    "        ax[2*j + 1, i].set_xticks([])  # set_xticks(range(len(prompts)))\n",
    "#         ax[1, i].set_xticklabels(objects, rotation=90)\n",
    "        for k in range(len(sim[i])):\n",
    "            ax[2*j + 1, i].bar(k, sim[i][k], color=plt.cm.tab20(1) if k!=0 else plt.cm.tab20(3))\n",
    "            ax[2*j + 1, i].text(k, 0.07, objects[k], rotation=90, ha='center', fontsize=15)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('figures/prompt_engineering.pdf', bbox_inches='tight')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env2",
   "language": "python",
   "name": "env2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}