FiddlingAround.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pickle\n",
    "import random\n",
    "import numpy as np\n",
    "np.set_printoptions(formatter={'float_kind':(lambda x: \"%6.2f\" % x)})\n",
    "\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import importlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import argparse\n",
    "#parser = argparse.ArgumentParser(\n",
    "#            description='PyTorch Relations-from-Stream sort-of-CLVR Example')\n",
    "#args = parser.parse_args()\n",
    "\n",
    "from attrdict import AttrDict\n",
    "args = AttrDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "args.batch_size = 32\n",
    "args.cuda = torch.cuda.is_available()\n",
    "args.dtype = torch.cuda.FloatTensor if args.cuda else torch.FloatTensor\n",
    "args.lr   = 0.0001\n",
    "args.seed = 5\n",
    "args.process_coords=False\n",
    "args.debug = True\n",
    "args.rnn_hidden_size = 32\n",
    "args.seq_len = 2+2+2\n",
    "args.coord_extra_len = 6\n",
    "args.highway = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(args.seed)\n",
    "if args.cuda:\n",
    "    torch.cuda.manual_seed(args.seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in a batch of images to test dimensions, etc...\n",
    "bs = args.batch_size\n",
    "\n",
    "input_img = torch.FloatTensor(bs, 3, 75, 75)\n",
    "input_qst = torch.FloatTensor(bs, 11)\n",
    "label = torch.LongTensor(bs)\n",
    "\n",
    "if args.cuda:\n",
    "    input_img = input_img.cuda()\n",
    "    input_qst = input_qst.cuda()\n",
    "    label = label.cuda()\n",
    "    \n",
    "input_img = Variable(input_img)\n",
    "input_qst = Variable(input_qst)\n",
    "label = Variable(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dirs = './data'\n",
    "\n",
    "filename = os.path.join(data_dirs, 'sort-of-clevr++.pickle')\n",
    "with open(filename, 'rb') as f:\n",
    "  train_datasets, test_datasets = pickle.load(f)\n",
    "\n",
    "def cvt_data_axis(data):\n",
    "    img = [e[0] for e in data]\n",
    "    qst = [e[1] for e in data]\n",
    "    ans = [e[2] for e in data]\n",
    "    return (img,qst,ans)\n",
    "\n",
    "norel_train, birel_train, trirel_train  = [], [], []\n",
    "for img, norelations, birelations, trirelations in train_datasets:\n",
    "    img = np.swapaxes(img,0,2)\n",
    "    for qst,ans in zip(norelations[0], norelations[1]):\n",
    "        norel_train.append((img,qst,ans))\n",
    "    for qst,ans in zip(birelations[0], birelations[1]):\n",
    "        birel_train.append((img,qst,ans))\n",
    "    for qst,ans in zip(trirelations[0], trirelations[1]):\n",
    "        trirel_train.append((img,qst,ans))\n",
    "\n",
    "norel = cvt_data_axis(norel_train)\n",
    "birel = cvt_data_axis(birel_train)\n",
    "trirel = cvt_data_axis(trirel_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tensor_data(data, i):\n",
    "    img = torch.from_numpy(np.asarray(data[0][bs*i:bs*(i+1)]))\n",
    "    qst = torch.from_numpy(np.asarray(data[1][bs*i:bs*(i+1)]))\n",
    "    ans = torch.from_numpy(np.asarray(data[2][bs*i:bs*(i+1)]))\n",
    "\n",
    "    input_img.data.resize_(img.size()).copy_(img)\n",
    "    input_qst.data.resize_(qst.size()).copy_(qst)\n",
    "    label.data.resize_(ans.size()).copy_(ans)\n",
    "\n",
    "# Loads batch 0 into input_img, input_qst amd label\n",
    "tensor_data(norel, 0)\n",
    "#tensor_data(birel, 0)    \n",
    "#tensor_data(trirel, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def from_pytorch(p): \n",
    "    #print(type(p))\n",
    "    if isinstance(p, Variable):\n",
    "        return p.cpu().data.numpy()\n",
    "    return p\n",
    "\n",
    "def show_image(im):\n",
    "    fig = plt.figure(figsize=(6,6))\n",
    "    ax = fig.add_subplot(1,1,1)      \n",
    "    plt.imshow(im[:,:,::-1], interpolation='nearest')  # BGR-> RGB\n",
    "    ax.set_xticks(np.arange(0, 75, 15))\n",
    "    ax.set_yticks(np.arange(0, 75, 15))\n",
    "    plt.grid(True)\n",
    "    #plt.axis('off')\n",
    "    \n",
    "def show_image_py(img):\n",
    "    show_image( np.swapaxes( from_pytorch(img), 0,2) )  # Undo the np->pytorch swap\n",
    "\n",
    "colors = 'red green blue orange gray yellow'.split()\n",
    "def describe_question(q):\n",
    "    question = list(from_pytorch(q))\n",
    "    color_arr = [colors[i] for i in range(0,6) if question[i]==1.]\n",
    "    #query = colors[question[0:6].index(1)]\n",
    "    query = '[%s] ' % ','.join(color_arr)\n",
    "\n",
    "    if question[6]==1 and question[7]==0 :  # NonRel Questions\n",
    "        if question[8] == 1:  query += 'shape?'\n",
    "        if question[9] == 1:  query += 'left?'\n",
    "        if question[10] == 1: query += 'up?'\n",
    "            \n",
    "    if question[6]==0 and question[7]==1:  # BiRel questions\n",
    "        if question[8] == 1:  query += 'closest shape?'\n",
    "        if question[9] == 1:  query += 'furthest shape?'\n",
    "        if question[10] == 1: query += 'shape count?'\n",
    "\n",
    "    if question[6]==1 and question[7]==1:  # Tricky questions\n",
    "        if question[8] == 1:  query += 'colinear count?'\n",
    "        if question[9] == 1:  query += 'equidistant count?'\n",
    "        if question[10] == 1: query += 'clockwise sweep count?'\n",
    "        if question[8] == -1:  query += 'big area triangle?'\n",
    "        if question[9] == -1:  query += 'clockwise?'\n",
    "        if question[10] == -1: query += 'most isolated shape?'\n",
    "    return query\n",
    "\n",
    "answer_sheet = ['yes', 'no', 'rectangle', 'circle', '1', '2', '3', '4', '5', '6']\n",
    "def describe_answer(a):\n",
    "    answer = from_pytorch(a)[0]\n",
    "    return answer_sheet[answer]\n",
    "\n",
    "def to_array(v): return v.cpu().data.numpy()\n",
    "\n",
    "def heat_map(d2, Xmap_to_zero=3.0):  # exp() factor below max that gets mapped to 0\n",
    "    d2_np = d2[0:25].view(5,5).t().cpu().data.numpy()\n",
    "    #d2_max = np.max(d2_np)\n",
    "    #d2_rescale = np.maximum(0., 1.0+(d2_np-d2_max)/map_to_zero)\n",
    "    print(d2_np)\n",
    "    plt.figure(figsize=(4,4))\n",
    "    plt.imshow(np.exp(d2_np), interpolation='nearest', cmap='gray')  # copper, binary\n",
    "\n",
    "def show_example(i):\n",
    "    print( describe_question( input_qst[i] ), describe_answer( label[i] ) )\n",
    "    show_image_py(input_img[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_choice = 1\n",
    "show_example(example_choice)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from model import RN, CNN_MLP, RFS\n",
    "import model_hard\n",
    "importlib.reload(model_hard)\n",
    "\n",
    "args.process_coords=False\n",
    "\n",
    "m = model_hard.RFSH(args)\n",
    "if args.cuda:\n",
    "    m.cuda()\n",
    "m.train();  # sets state only\n",
    "\n",
    "#accuracy_rel = m.train_(input_img, input_qst, label)\n",
    "#accuracy_norel = m.train_(input_img, input_qst, label)\n",
    "\n",
    "# Load a snapshot\n",
    "#m.load_state_dict( torch.load('model/RFS_2item-span-again-seed10_050.pth') )\n",
    "#m.load_state_dict( torch.load('model/RFS_2item-span-again-seed10_050.pth', map_location=lambda storage, loc: storage) )\n",
    "#m.load_state_dict( torch.load('model/RFS_2item-span-seed10-tricky_080.pth', map_location=lambda storage, loc: storage) )\n",
    "#m.load_state_dict( torch.load('model/RFSH_4item-span-seed10-6coord-fuzz1.0-emph5.0_050.pth', map_location=lambda storage, loc: storage) )\n",
    "#m.load_state_dict( torch.load('model/RFSH_4item-span-seed10-6coord-fuzz1.0-plain_010.pth', map_location=lambda storage, loc: storage) )\n",
    "m.load_state_dict( torch.load('model/RFSH_6item-span-seed10-6coord-fuzz1.0-plain-hardtest_150.pth', map_location=lambda storage, loc: storage) )\n",
    "\n",
    "#m.optimizer.zero_grad()\n",
    "m.eval()\n",
    "output = m(input_img, input_qst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print( to_array(m.ans_logits[example_choice]), \n",
    "       ' answer = %s' % answer_sheet[ np.argmax(to_array(m.ans_logits[example_choice])) ] )\n",
    "for i in range(len(m.ent_similarities)):\n",
    "    heat_map(m.ent_similarities[i][example_choice])\n",
    "    heat_map(m.ent_weights_arr [i][example_choice])\n",
    "#m.ent_similarities[example_choice].size()\n",
    "#m.ent_similarities[0][example_choice]-m.ent_similarities[1][example_choice]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heat_maplet(d2): \n",
    "    d2_np = d2[0:25].view(5,5).t().cpu().data.numpy()\n",
    "    plt.imshow(np.exp(d2_np), interpolation='nearest', cmap='gray')  # copper, binary\n",
    "    #plt.axis('off')\n",
    "    plt.gca().tick_params(bottom='off', left='off', labelbottom='off', labelleft='off', )    \n",
    "\n",
    "plt.figure(1, figsize=(16,5.5))\n",
    "\n",
    "glimpses = len(m.ent_similarities)\n",
    "for i in range(glimpses):\n",
    "    plt.subplot(2, glimpses, i+1)\n",
    "    heat_maplet(m.ent_similarities[i][example_choice])\n",
    "    if i==0:\n",
    "        plt.ylabel('Attention Matches')\n",
    "    plt.subplot(2, glimpses, i+glimpses+1)\n",
    "    heat_maplet(m.ent_weights_arr [i][example_choice])\n",
    "    plt.xlabel('step=%d' % (i,) )\n",
    "    if i==0:\n",
    "        plt.ylabel('Hard Entity')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#[ (i,l,np.argmax(to_array(m.ans_logits[i]))) for i,l in enumerate(to_array(label)) ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sanity checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.zeros(8, 2)\n",
    "b = torch.ones(8, 7)\n",
    "\n",
    "c = torch.cat( (a,b), 1)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = torch.rand( (1,5) )\n",
    "p\n",
    "p.expand( (6,5) )\n",
    "p.expand( (6,5) ) + torch.rand( (6,5) )\n",
    "#p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = torch.from_numpy( np.array([[ 1.,2.,3.], [6.,1.,4. ] ], dtype=np.float32) )\n",
    "b = Variable(b)\n",
    "b\n",
    "torch.nn.functional.softmax(b)    # This is the expected one (probs add up to 1 along rows)\n",
    "#torch.nn.functional.log_softmax(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.sample_gumbel(b) \n",
    "#model.gumbel_softmax_sample(b, temperature=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the sort-of-CLEVR generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sort_of_clevr_generator\n",
    "importlib.reload(sort_of_clevr_generator);\n",
    "\n",
    "(img, norelations, birelations, trirelations) = sort_of_clevr_generator.build_dataset(1)\n",
    "\n",
    "show_image(img)\n",
    "\n",
    "example_choice = 0\n",
    "r = trirelations #, birelations, norelations\n",
    "print( describe_question(np.array(r[0][example_choice])) )\n",
    "print( describe_answer(np.array([r[1][example_choice]])) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#isinstance(np.array([]), np.ndarray)\n",
    "#np.cross( np.array( [-10,-10] ), np.array( [10,10] ) )\n",
    "np.linalg.norm( np.array( [10,10] ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}