sample_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h4F7KVNZIClZ"
   },
   "source": [
    "### Load libraries and packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "loCEdjeVIKi_"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import PIL \n",
    "import time \n",
    "import torch \n",
    "import torchvision \n",
    "import random\n",
    "\n",
    "import numpy as np \n",
    "import torch.nn as nn \n",
    "import matplotlib.pyplot as plt \n",
    "import torch.nn.functional as F \n",
    "\n",
    "#data transforms\n",
    "from torch.autograd import Variable \n",
    "from torchvision import datasets, transforms \n",
    "from PIL import Image, ImageEnhance\n",
    "\n",
    "#data aug\n",
    "from augmentation.autoaugment   import CIFAR10Policy \n",
    "from augmentation.AugMix.AugMix import AugMixDataset \n",
    "from augmentation.cutout        import Cutout \n",
    "from augmentation.RandAugment   import RandAugment\n",
    "\n",
    "#optim and activation\n",
    "from optim.deepmemory import DeepMemory\n",
    "from optim.lookahead  import Lookahead \n",
    "from optim.radam      import RAdam \n",
    "from adamod           import AdaMod\n",
    "from activations      import Mish\n",
    "\n",
    "from models.model import CustomModel\n",
    "from loss_func.cross_entropy import CrossEntropyLoss\n",
    "from metrics import AverageMeter, accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "u_i2lQDDIClh"
   },
   "source": [
    "### Set Seed for Reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q4m4m3F-ICli"
   },
   "outputs": [],
   "source": [
    "def set_seed(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED']=str(seed)\n",
    "    \n",
    "set_seed(72)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "v25cvsAUIClm"
   },
   "source": [
    "### Preprocess and Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VQVgSBhIIClo"
   },
   "outputs": [],
   "source": [
    "preprocess = transforms.Compose([\n",
    "     transforms.ToTensor(),\n",
    "     transforms.Normalize((0.4914, 0.4822, 0.4465), (.2023, .1994, .2010)),  #CIFAR10\n",
    "#      transforms.Normalize((0.507, 0.487, 0.441), (0.267, 0.256, 0.276))      #CIFAR100\n",
    "#      Cutout(n_holes=1, length=16),               # CutOut\n",
    "])\n",
    "\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.RandomCrop(32, padding=4, fill=128),\n",
    "#     CIFAR10Policy(),                    # AutoAugment\n",
    "    preprocess\n",
    "\n",
    "])\n",
    "\n",
    "# train_transform.transforms.insert(0, RandAugment(1, 5))                    #RandAugment\n",
    "\n",
    "test_transform = preprocess\n",
    "\n",
    "# test_transform = transforms.Compose([\n",
    "#     transforms.ToTensor(),\n",
    "#      transforms.Normalize((0.4914, 0.4822, 0.4465), (.2023, .1994, .2010)),  #CIFAR10\n",
    "#      transforms.Normalize((0.507, 0.487, 0.441), (0.267, 0.256, 0.276))      #CIFAR100\n",
    "# ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5859,
     "status": "ok",
     "timestamp": 1578403552710,
     "user": {
      "displayName": "Enoch Tetteh",
      "photoUrl": "",
      "userId": "18429685187145851841"
     },
     "user_tz": -120
    },
    "id": "19uZ_-wXIClr",
    "outputId": "d505b40b-bf4c-479d-8424-d82b8acbc043"
   },
   "outputs": [],
   "source": [
    "batch_size = 4\n",
    "\n",
    "train_data = datasets.CIFAR10(root=\"./data\", train=True, download=True, transform=train_transform)\n",
    "\n",
    "test_data= datasets.CIFAR10(root=\"./data\", train=False, download=True, transform=test_transform)\n",
    "\n",
    "# load training data in batches\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "#                       AugMixDataset(train_data, preprocess, no_jsd=True),      # Augmix\n",
    "                      train_data,\n",
    "                      batch_size=batch_size,\n",
    "                      num_workers=8,\n",
    "                      shuffle=True, \n",
    "                      pin_memory=True\n",
    "                      )\n",
    "\n",
    "# load test data in batches\n",
    "test_loader = torch.utils.data.DataLoader(test_data,\n",
    "                      batch_size=batch_size,\n",
    "                      num_workers=8,\n",
    "                      shuffle=False,\n",
    "                      pin_memory=True\n",
    "                      )\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5796,
     "status": "ok",
     "timestamp": 1578403552712,
     "user": {
      "displayName": "Enoch Tetteh",
      "photoUrl": "",
      "userId": "18429685187145851841"
     },
     "user_tz": -120
    },
    "id": "3FKBwJoSIClv",
    "outputId": "e672b6bc-801d-4178-b496-0a280a85b2cb"
   },
   "outputs": [],
   "source": [
    "print(f'Length of train loader is {len(train_loader)}')\n",
    "print(f'Length of test loader is {len(test_loader)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "V5F0pZ3oGNn8"
   },
   "source": [
    "### Build Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CustomModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kbyc6OWovS1Z"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model = model.to(device)\n",
    "model = torch.nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5350,
     "status": "ok",
     "timestamp": 1578403552716,
     "user": {
      "displayName": "Enoch Tetteh",
      "photoUrl": "",
      "userId": "18429685187145851841"
     },
     "user_tz": -120
    },
    "id": "FTxHBIlNICl-",
    "outputId": "0883288d-75fd-4aff-bdfa-18881eb53c34"
   },
   "outputs": [],
   "source": [
    "print('Number of model parameters: {}'.format(\n",
    "        sum([p.data.nelement() for p in model.parameters()])\n",
    "        ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xmUuozs0ICmC"
   },
   "source": [
    "### Fresh Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Mnc4QeDrICmD"
   },
   "outputs": [],
   "source": [
    "best_top1 = 0 # train from start\n",
    "start_epoch = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5056,
     "status": "ok",
     "timestamp": 1578403552717,
     "user": {
      "displayName": "Enoch Tetteh",
      "photoUrl": "",
      "userId": "18429685187145851841"
     },
     "user_tz": -120
    },
    "id": "qSJ_eiJeICmG",
    "outputId": "2cfb71ad-947a-4f7b-c5e0-5b4f7e55791a"
   },
   "outputs": [],
   "source": [
    "num_epochs = 20\n",
    "criterion = CrossEntropyLoss(smooth_eps=0.1).to(device)\n",
    "params = [p for p in model.parameters()]\n",
    "\n",
    "optimizer = AdaMod(params, lr=0.1, betas=(0.999, 0.9999), weight_decay=1e-5)\n",
    "# optimizer = DeepMemory(params, betas=(0.999, 0.9999), len_memory=len(train_data.data)//batch_size, weight_decay=1e-4)\n",
    "# optimizer = Lookahead(DeepMemory(params, len_memory=len(train_data.data)//batch_size))\n",
    "# optimizer = Lookahead(AdaMod(params, betas=(0.999, 0.9999), weight_decay=1e-5))\n",
    "# optimizer = Lookahead(RAdam(params, lr=0.0015, weight_decay=0.0))          # rectified adam wtih lookahead\n",
    "lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_data.data)//batch_size) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "G9smYWDNICmJ"
   },
   "source": [
    "### Load Checkpoints to resume training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1TCmACuRICmK"
   },
   "outputs": [],
   "source": [
    "# checkpoint = torch.load('./checkpoint/CustomModel_standard_dmla_ckpt.pth')\n",
    "\n",
    "# model.module.load_state_dict(checkpoint['model_state_dict'], strict=False)\n",
    "# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "# start_epoch = checkpoint['epoch']\n",
    "# best_top1 = checkpoint['top1']   # resume training\n",
    "# best_top5 = checkpoint['top5']  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4842,
     "status": "ok",
     "timestamp": 1578403552719,
     "user": {
      "displayName": "Enoch Tetteh",
      "photoUrl": "",
      "userId": "18429685187145851841"
     },
     "user_tz": -120
    },
    "id": "j1S2Eei7ICmN",
    "outputId": "e1acd940-9dfe-4eda-d679-c9b496b1c0e3"
   },
   "outputs": [],
   "source": [
    "# print(f'Loaded checkpoint with \\n {best_top1}% Top-1 Accuracy, {best_top5}% Top-5 Accuracy, after training for {start_epoch} epochs.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rXU7su9tICmQ"
   },
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YrAeZyBNICmR"
   },
   "outputs": [],
   "source": [
    "def train(train_loader, model, criterion, optimizer, epoch):\n",
    "    print('Training model...\\n')\n",
    "    batch_time = AverageMeter()\n",
    "    data_time = AverageMeter()\n",
    "    losses = AverageMeter()\n",
    "    top1 = AverageMeter()\n",
    "    top5 = AverageMeter()\n",
    "\n",
    "    # switch to train mode\n",
    "    model.train()\n",
    "\n",
    "    end = time.time()\n",
    "    \n",
    "    for i, (input, target) in enumerate(train_loader):\n",
    "                \n",
    "        # measure data loading time\n",
    "        data_time.update(time.time() - end)\n",
    "        \n",
    "        target = target.to(device)\n",
    "        input_var = Variable(input)\n",
    "        target_var = Variable(target)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # compute output\n",
    "        output = model(input_var) \n",
    "\n",
    "        def closure():\n",
    "            output = model(input_var) \n",
    "            loss = criterion(output, target_var)\n",
    "          \n",
    "            return loss\n",
    "            \n",
    "        \n",
    "        # measure accuracy and record loss\n",
    "        prec1, prec5 = accuracy(output.data, target, topk=(1, 5))\n",
    "        losses.update(closure().item(), input.size(0))\n",
    "        top1.update(prec1.item(), input.size(0))\n",
    "        top5.update(prec5.item(), input.size(0))\n",
    "       \n",
    "        closure().backward()\n",
    "        optimizer.step(closure)\n",
    "        \n",
    "        lr_scheduler.step(epoch)\n",
    "        \n",
    "        # measure elapsed time\n",
    "        batch_time.update(time.time() - end)\n",
    "        end = time.time()\n",
    "\n",
    "        if i % 1500 == 0:\n",
    "            print('Epoch: [{0}][{1}/{2}]\\t'\n",
    "                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n",
    "                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\\t'\n",
    "                  'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n",
    "                  'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\\t'\n",
    "                  'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(\n",
    "                epoch, i, len(train_loader), batch_time=batch_time,\n",
    "                data_time=data_time, loss=losses, top1=top1, top5=top5))\n",
    "    \n",
    "    print(' * Acc@1 {top1.avg:.3f} Acc@1 Error {top1_err:.3f}\\n'\n",
    "              ' * Acc@5 {top5.avg:.3f} Acc@5 Error {top5_err:.3f}'\n",
    "              .format(top1=top1, top1_err=100-top1.avg, top5=top5, top5_err=100-top5.avg))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0bCJXYwQICmW"
   },
   "source": [
    "### Accuracy on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kquGCWt6ICmX"
   },
   "outputs": [],
   "source": [
    "def validate(test_loader, model, criterion, epoch):\n",
    "   \n",
    "    # switch to evaluate mode\n",
    "    model.eval()\n",
    "    \n",
    "    print('Evaluating model on test data...\\n')\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        batch_time = AverageMeter()\n",
    "        losses = AverageMeter()\n",
    "        top1 = AverageMeter()\n",
    "        top5 = AverageMeter()\n",
    "\n",
    "        end = time.time()\n",
    "        for i, (input, target) in enumerate(test_loader):\n",
    "            target = target.to(device)\n",
    "            input = input.to(device)\n",
    "\n",
    "            # compute output\n",
    "            output = model(input)\n",
    "            loss = criterion(output, target)\n",
    "\n",
    "            # measure accuracy and record loss\n",
    "            prec1, prec5 = accuracy(output.data, target.data, topk=(1, 5))\n",
    "            losses.update(loss.data.item(), input.size(0))\n",
    "            top1.update(prec1.item(), input.size(0))\n",
    "            top5.update(prec5.item(), input.size(0))\n",
    "\n",
    "            # measure elapsed time\n",
    "            batch_time.update(time.time() - end)\n",
    "            end = time.time()\n",
    "\n",
    "            if i % 250 == 0:\n",
    "                print('Test: [{0}/{1}]\\t'\n",
    "                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\\t'\n",
    "                      'Loss {loss.val:.4f} ({loss.avg:.4f})\\t'\n",
    "                      'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\\t'\n",
    "                      'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(\n",
    "                       i, len(test_loader), batch_time=batch_time, loss=losses,\n",
    "                       top1=top1, top5=top5))\n",
    "                \n",
    "    \n",
    "        \n",
    "        print(' * Acc@1 {top1.avg:.3f} Acc@1 Error {top1_err:.3f}\\n'\n",
    "              ' * Acc@5 {top5.avg:.3f} Acc@5 Error {top5_err:.3f}'\n",
    "              .format(top1=top1, top1_err=100-top1.avg, top5=top5, top5_err=100-top5.avg))\n",
    "\n",
    "        return top1, top5, losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Vc-QoWHsICmb"
   },
   "source": [
    "### Train, Evaluate and Checkpoint Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 952994,
     "status": "error",
     "timestamp": 1578405015258,
     "user": {
      "displayName": "Enoch Tetteh",
      "photoUrl": "",
      "userId": "18429685187145851841"
     },
     "user_tz": -120
    },
    "id": "zKA7gbyfICmd",
    "outputId": "5284b8a2-b75e-4550-ab2e-8572954ef443"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "for epoch in range(start_epoch, num_epochs):\n",
    "    train(train_loader, model, criterion, optimizer, epoch)\n",
    "    top1, top5, losses = validate(test_loader, model, criterion, epoch)\n",
    "\n",
    "    if top1.avg > best_top1:\n",
    "        print('Saving checkpoint')\n",
    "        state = {\n",
    "          'model_state_dict': model.module.state_dict(),\n",
    "          'optimizer_state_dict': optimizer.state_dict(),\n",
    "          'epoch': epoch,\n",
    "          'loss': losses.avg, \n",
    "          'top1': top1.avg,\n",
    "          'top5': top5.avg}\n",
    "        if not os.path.isdir('checkpoint'):\n",
    "            os.mkdir('checkpoint')\n",
    "        torch.save(state, './checkpoint/seed0_ckpt.pth')\n",
    "\n",
    "        best_top1 = top1.avg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qlUGmdnvICmh"
   },
   "source": [
    "### Accuracy for each class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 231
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1629,
     "status": "error",
     "timestamp": 1578407435891,
     "user": {
      "displayName": "Enoch Tetteh",
      "photoUrl": "",
      "userId": "18429685187145851841"
     },
     "user_tz": -120
    },
    "id": "BnOd0SAGICmr",
    "outputId": "f154d673-41b0-410b-c295-29958ea3634e"
   },
   "outputs": [],
   "source": [
    "# class_correct = list(0. for i in range(100))\n",
    "# class_total = list(0. for i in range(100))\n",
    "# with torch.no_grad(): \n",
    "#     for data in test_loader:\n",
    "#         images, labels = data\n",
    "#         outputs = model(images)\n",
    "#         _, predicted = torch.max(outputs, 1)\n",
    "#         c = (predicted == labels).squeeze()\n",
    "#         for i in range(4):\n",
    "#             label = labels[i]\n",
    "#             class_correct[label] += c[i].item()\n",
    "#             class_total[label] += 1\n",
    "\n",
    "\n",
    "# for i in range(10):\n",
    "#     print('Accuracy of %5s : %2d %%' % (\n",
    "#         classes[i], 100 * class_correct[i] / class_total[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NyJled9cq615"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "image_classification_modified_resnext101_32x48d.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}