densenet_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun May  5 18:37:20 2019       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 410.72       Driver Version: 410.72       CUDA Version: 10.0     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  Tesla K40c          Off  | 00000000:83:00.0 Off |                    0 |\r\n",
      "| 24%   46C    P0    65W / 235W |      0MiB / 11441MiB |    100%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID   Type   Process name                             Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|  No running processes found                                                 |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import scipy.misc\n",
    "import torchvision\n",
    "from torchvision import utils, transforms, models\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn as nn\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "from PIL import Image\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of classes:  4251\n"
     ]
    }
   ],
   "source": [
    "# Reading data to create id to label mapping\n",
    "data = pd.read_csv(\"/scratch/tmp/WhaleClassData/train.csv\")\n",
    "label_set = set(data['Id'])\n",
    "num_of_classes = len(label_set)\n",
    "print(\"Number of classes: \", len(label_set))\n",
    "\n",
    "label_to_id = {}\n",
    "id_to_label = {}\n",
    "\n",
    "def create_label_id_maps():\n",
    "    _id = 0 \n",
    "    for label in label_set: \n",
    "        label_to_id[label] = _id \n",
    "        id_to_label[_id] = label\n",
    "        _id += 1\n",
    "\n",
    "create_label_id_maps()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataset class definition \n",
    "class WhaleDataset(Dataset):\n",
    "    def __init__(self, csv_file, root_dir, transform=None):\n",
    "        self.root_dir = root_dir\n",
    "        if \"train\" in csv_file:\n",
    "            self.whales = pd.read_csv(csv_file)\n",
    "            self.test = 0\n",
    "        else:\n",
    "            self.whales = list(os.listdir(self.root_dir))\n",
    "            self.test = 1\n",
    "        self.transform = transform\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.whales)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        if self.test:\n",
    "            img_name = os.path.join(self.root_dir, self.whales[idx])\n",
    "        else:\n",
    "            img_name = os.path.join(self.root_dir, self.whales.iloc[idx, 0])\n",
    "            \n",
    "        image = Image.open(img_name).convert('RGB')\n",
    "        if self.test:\n",
    "            sample = {'image': image, 'image_name': self.whales[idx]}\n",
    "        else:\n",
    "            label = self.whales.iloc[idx, 1]\n",
    "            label = label_to_id[label]\n",
    "            sample = {'image': image, 'image_name': self.whales.iloc[idx, 0], 'label': label}\n",
    "        if self.transform:\n",
    "            sample['image'] = self.transform(sample['image'])\n",
    "        return sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining transformations to aid in data augmentation\n",
    "resize_shape = (200, 200)\n",
    "# resize_shape = (700, 1050)\n",
    "\n",
    "transforms_1 = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.Resize(resize_shape),\n",
    "    torchvision.transforms.ColorJitter(hue=.05, saturation=.05),\n",
    "    torchvision.transforms.RandomHorizontalFlip(),\n",
    "    torchvision.transforms.RandomVerticalFlip(p=0.5),\n",
    "    torchvision.transforms.RandomRotation(20, resample=Image.BILINEAR),\n",
    "    torchvision.transforms.RandomAffine(30),\n",
    "    torchvision.transforms.ToTensor()\n",
    "])\n",
    "\n",
    "transform_resize = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.Resize(resize_shape),\n",
    "    torchvision.transforms.ToTensor()\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating Train and Valid Datset samplers\n",
    "whale_dataset_1 = WhaleDataset(csv_file=\"/scratch/tmp/WhaleClassData/train.csv\", \n",
    "                             root_dir=\"/scratch/tmp/WhaleClassData/train\",\n",
    "                             transform=transform_resize)\n",
    "whale_dataset_2 = WhaleDataset(csv_file=\"/scratch/tmp/WhaleClassData/train.csv\", \n",
    "                             root_dir=\"/scratch/tmp/WhaleClassData/train\",\n",
    "                             transform=transforms_1)\n",
    "whale_dataset_test = WhaleDataset(csv_file=\"/scratch/tmp/WhaleClassData/test\", \n",
    "                             root_dir=\"/scratch/tmp/WhaleClassData/test/\",\n",
    "                             transform=transform_resize)\n",
    "\n",
    "concat_dataset = torch.utils.data.ConcatDataset([whale_dataset_1, whale_dataset_2])\n",
    "\n",
    "validation_split = .2\n",
    "shuffle_dataset = True\n",
    "random_seed = 42\n",
    "\n",
    "dataset_size = len(concat_dataset)\n",
    "indices = list(range(dataset_size))\n",
    "split = int(np.floor(validation_split * dataset_size))\n",
    "if shuffle_dataset:\n",
    "    np.random.seed(random_seed)\n",
    "    np.random.shuffle(indices)\n",
    "train_indices, val_indices = indices[split:], indices[:split]\n",
    "\n",
    "train_sampler = SubsetRandomSampler(train_indices)\n",
    "valid_sampler = SubsetRandomSampler(val_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checking what kind of system you are using\n",
    "try:\n",
    "  import google.colab\n",
    "  from google.colab import drive\n",
    "  from google.colab import files\n",
    "  IN_COLAB = True\n",
    "except:\n",
    "  IN_COLAB = False\n",
    "try:\n",
    "    hostname = !hostname\n",
    "    if 'lab' in hostname[0] and '.sci.utah.edu' in hostname[0]:\n",
    "        IN_CADE = True\n",
    "    else:\n",
    "        IN_CADE = False\n",
    "except:\n",
    "    IN_CADE = False\n",
    "\n",
    "assert(not IN_CADE or not IN_COLAB)\n",
    "\n",
    "#defining the folders where datasets will be, depending on the system\n",
    "machine_being_used = 'cade' if IN_CADE else ('colab' if IN_COLAB else 'other')\n",
    "pre_folder = '/scratch/tmp/' if machine_being_used == 'cade' else './'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_gpu_to_use(minimum_memory_mb = 3800):\n",
    "    gpu_to_use = None\n",
    "    try: \n",
    "        os.environ['CUDA_VISIBLE_DEVICES']\n",
    "        print('GPU already assigned before: ' + str(os.environ['CUDA_VISIBLE_DEVICES']))\n",
    "        return\n",
    "    except:\n",
    "        pass\n",
    "    torch.cuda.empty_cache()\n",
    "    for i in range(16):\n",
    "        free_memory = !nvidia-smi --query-gpu=memory.free -i $i --format=csv,nounits,noheader\n",
    "        if free_memory[0] == 'No devices were found':\n",
    "            break\n",
    "        free_memory = int(free_memory[0])\n",
    "        if free_memory>minimum_memory_mb-500:\n",
    "            gpu_to_use = i\n",
    "            break\n",
    "    if gpu_to_use is None:\n",
    "        print('Could not find any GPU available with the required free memory of ' +str(minimum_memory_mb) + 'MB. Please use a different system for this assignment.')\n",
    "    else:\n",
    "        os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_to_use)\n",
    "        print('Chosen GPU: ' + str(gpu_to_use))\n",
    "        x = torch.rand((256,1024,minimum_memory_mb-500)).cuda()\n",
    "        x = torch.rand((1,1)).cuda()\n",
    "        del x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chosen GPU: 0\n"
     ]
    }
   ],
   "source": [
    "define_gpu_to_use()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model 1 - Resnet18 - Dataset\n",
    "batch_size = 128\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(concat_dataset, batch_size=batch_size, sampler=train_sampler)\n",
    "validation_loader = torch.utils.data.DataLoader(concat_dataset, batch_size=1, sampler=valid_sampler)\n",
    "test_loader = torch.utils.data.DataLoader(whale_dataset_test, batch_size=1, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Identity(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Identity, self).__init__()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/densenet161-8d451a50.pth\" to /home/sci/sravan/.cache/torch/checkpoints/densenet161-8d451a50.pth\n",
      "100.0%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "CUDA out of memory. Tried to allocate 162.00 MiB (GPU 0; 11.17 GiB total capacity; 10.52 GiB already allocated; 137.31 MiB free; 170.80 MiB cached)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-42fba2426cce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0my_onehot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscatter_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m         \u001b[0my_onehot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_onehot\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_onehot\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torchvision/models/densenet.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    112\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 113\u001b[0;31m         \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    114\u001b[0m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madaptive_avg_pool2d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/env_dir/lib/python3.7/site-packages/torchvision/models/densenet.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop_rate\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0mnew_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdrop_rate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_features\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: CUDA out of memory. Tried to allocate 162.00 MiB (GPU 0; 11.17 GiB total capacity; 10.52 GiB already allocated; 137.31 MiB free; 170.80 MiB cached)"
     ]
    }
   ],
   "source": [
    "# Model 1 - alexnet\n",
    "num_epochs = 5\n",
    "\n",
    "model_1 = models.densenet161(pretrained=True)\n",
    "model_1.classifier = nn.Linear(1024, num_of_classes)\n",
    "# model_1.classifier[4] = nn.Linear(in_features=4096, out_features=num_of_classes, bias=True)\n",
    "# model_1.classifier[5] = Identity()\n",
    "# model_1.classifier[6] = Identity()\n",
    "model_1.cuda()\n",
    "\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = torch.optim.Adam(model_1.parameters(), lr=0.0001)\n",
    "   \n",
    "for epoch in range(num_epochs): \n",
    "    model_1.train() \n",
    "    losses = []\n",
    "    print('Epoch ' + str(epoch)) \n",
    "    for sample in train_loader:\n",
    "        optimizer.zero_grad() \n",
    "        images = sample['image'].cuda()\n",
    "        label = sample['label'].cpu().numpy()\n",
    "        label = label.reshape(label.shape[0],1)\n",
    "        # Creating one hot vector\n",
    "        y_onehot = torch.FloatTensor(label.shape[0], num_of_classes)\n",
    "        y_onehot.zero_()\n",
    "        y_onehot.scatter_(1, torch.LongTensor(label), 1)\n",
    "        y_onehot = y_onehot.cuda()\n",
    "        out = model_1(images)\n",
    "        loss = criterion(out, y_onehot)\n",
    "        loss.backward()\n",
    "        optimizer.step() \n",
    "        losses.append(loss.item())\n",
    "    print('loss: ' + str(np.mean(losses)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# writeFile = open('submission.csv', 'w')\n",
    "# writer = csv.writer(writeFile)\n",
    "# writer.writerow([\"Image\", \"Id\"])\n",
    "\n",
    "model_1.eval()\n",
    "counter = 0\n",
    "acc = 0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for sample in validation_loader:\n",
    "        images = sample['image'].cuda()\n",
    "#         print (sample['image_name'][0])\n",
    "#         print (counter)\n",
    "#         counter += 1\n",
    "        out = model_1(images)\n",
    "        out = nn.functional.softmax(out)\n",
    "        out = out.cpu().detach().numpy()\n",
    "        ind = np.argsort(out)[0][-5:]\n",
    "#         ind_last = np.argsort(out)[0][-5:]\n",
    "#         print(out)\n",
    "#         print(out[0][ind[0]], \" :: \", out[0][ind[1]])\n",
    "#         print(out[0][ind_last[0]], \" :: \", out[0][ind_last[1]])\n",
    "#         print (int(sample['label'][0]))\n",
    "#         print (ind[0])\n",
    "        if int(sample['label'][0]) in ind : \n",
    "            acc += 1\n",
    "#         if int(sample['label'][0]) == ind[0] or int(sample['label'][0]) == ind[1] or int(sample['label'][0]) == ind[2] or int(sample['label'][0]) == ind[3] or int(sample['label'][0]) == ind[4]:\n",
    "#             acc += 1\n",
    "#         writer.writerow([sample['image_name'][0], id_to_label[ind[0]] + \" \" + id_to_label[ind[1]] + \" \" \n",
    "#                          + id_to_label[ind[2]] + \" \" + id_to_label[ind[3]] + \" \" + id_to_label[ind[4]]])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(acc, len(validation_loader))\n",
    "print (\"Accuracy: %f\" % (acc/len(validation_loader)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write to the kaggle submission file \n",
    "\n",
    "print(\"# test files :: \", len(test_loader))\n",
    "writeFile = open('submission.csv', 'w')\n",
    "writer = csv.writer(writeFile)\n",
    "writer.writerow([\"Image\", \"Id\"])\n",
    "\n",
    "lines = [[\"Image\", \"Id\"]]\n",
    "\n",
    "model_1.eval()\n",
    "counter = 0\n",
    "acc = 0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for sample in test_loader:\n",
    "        images = sample['image'].cuda()\n",
    "#         print (sample['image_name'][0])\n",
    "#         print (counter)\n",
    "        counter += 1\n",
    "        out = model_1(images)\n",
    "        out = nn.functional.softmax(out)\n",
    "        out = out.cpu().detach().numpy()\n",
    "        ind = np.argsort(out)[0][-5:]\n",
    "#         print(ind)\n",
    "#         print (int(sample['label'][0]))\n",
    "#         print (ind[0])\n",
    "#         if int(sample['label'][0]) == ind[0] or int(sample['label'][0]) == ind[1] or int(sample['label'][0]) == ind[2] or int(sample['label'][0]) == ind[3] or int(sample['label'][0]) == ind[4]:\n",
    "#             acc += 1\n",
    "        row = [sample['image_name'][0], id_to_label[ind[-1]] + \" \" + id_to_label[ind[-2]] + \" \" \n",
    "                         + id_to_label[ind[-3]] + \" \" + id_to_label[ind[-4]] + \" \" + id_to_label[ind[-5]]]\n",
    "        lines.append(row)\n",
    "        writer.writerow([sample['image_name'][0], id_to_label[ind[0]] + \" \" + id_to_label[ind[1]] + \" \" \n",
    "                         + id_to_label[ind[2]] + \" \" + id_to_label[ind[3]] + \" \" + id_to_label[ind[4]]])\n",
    "    \n",
    "print(\"# counter:: \", counter)\n",
    "\n",
    "with open('submission_alexnet.csv', 'w') as writeFile:\n",
    "    writer = csv.writer(writeFile)\n",
    "    writer.writerows(lines)\n",
    "\n",
    "#check how many rows are present in the file.\n",
    "with open(\"submission.csv\") as f:\n",
    "    print(\"#rows in submission file :: \",sum(1 for line in f))\n",
    "    \n",
    "with open(\"submission_alexnet.csv\") as f:\n",
    "    print(\"#rows in submission_alexnet file :: \",sum(1 for line in f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun May  5 18:37:02 2019       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 410.72       Driver Version: 410.72       CUDA Version: 10.0     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  Tesla K40c          Off  | 00000000:83:00.0 Off |                    0 |\r\n",
      "| 24%   47C    P0    64W / 235W |      0MiB / 11441MiB |     98%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID   Type   Process name                             Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|  No running processes found                                                 |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env_dir",
   "language": "python",
   "name": "env_dir"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}