week04_training/homework.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "hse_dl_year": "2021-fall"
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qvW8-J6we6By",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Обучение нейросетей — оптимизация и регуляризация\n",
    "\n",
    "**Разработчик: Артем Бабенко**\n",
    "\n",
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mryab/dl-hse-ami/blob/master/week04_training/homework.ipynb)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1ecMva_Ge6B0",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "На этом семинаре будет необходимо\n",
    "1. реализовать Dropout-слой и проследить его влияние на обобщающую способность сети \n",
    "2. реализовать BatchNormalization-слой и пронаблюдать его влияние на скорость сходимости обучения."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wQZ-_wUwe6B0",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Dropout\n",
    "\n",
    "Как всегда будем экспериментировать на датасете MNIST. MNIST является стандартным бенчмарк-датасетом, и его можно подгрузить средствами pytorch."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "v4S5PFg5e6B1",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from IPython.display import clear_output\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "import torch.optim as optim\n",
    "from torch.utils.data.sampler import SubsetRandomSampler"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "5EuePwt3e6B5",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "input_size = 784\n",
    "num_classes = 10\n",
    "batch_size = 128\n",
    "\n",
    "train_dataset = dsets.MNIST(root='./MNIST/', \n",
    "                                   train=True, \n",
    "                                   transform=transforms.ToTensor(),\n",
    "                                   download=True)\n",
    "\n",
    "test_dataset = dsets.MNIST(root='./MNIST/', \n",
    "                                  train=False, \n",
    "                                  transform=transforms.ToTensor())\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                           batch_size=batch_size, \n",
    "                                           shuffle=False)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n",
    "                                          batch_size=batch_size, \n",
    "                                          shuffle=False)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nUFeuDtfe6B9",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Определим ряд стандартных функций с прошлых семинаров"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "a_rmaGJQe6B9",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "def train_epoch(model, optimizer, batchsize=32):\n",
    "    loss_log, acc_log = [], []\n",
    "    model.train()\n",
    "    for batch_num, (x_batch, y_batch) in enumerate(train_loader):\n",
    "        data = x_batch\n",
    "        target = y_batch\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        pred = torch.max(output, 1)[1]\n",
    "        acc = torch.eq(pred, y_batch).float().mean()\n",
    "        acc_log.append(acc)\n",
    "        \n",
    "        loss = F.nll_loss(output, target).cpu()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        loss = loss.item()\n",
    "        loss_log.append(loss)\n",
    "    return loss_log, acc_log    \n",
    "\n",
    "def test(model):\n",
    "    loss_log, acc_log = [], []\n",
    "    model.eval()\n",
    "    for batch_num, (x_batch, y_batch) in enumerate(test_loader):    \n",
    "        data = x_batch\n",
    "        target = y_batch\n",
    "\n",
    "        output = model(data)\n",
    "        loss = F.nll_loss(output, target).cpu()\n",
    "\n",
    "        pred = torch.max(output, 1)[1]\n",
    "        acc = torch.eq(pred, y_batch).float().mean()\n",
    "        acc_log.append(acc)\n",
    "        \n",
    "        loss = loss.item()\n",
    "        loss_log.append(loss)\n",
    "    return loss_log, acc_log\n",
    "\n",
    "def plot_history(train_history, val_history, title='loss'):\n",
    "    plt.figure()\n",
    "    plt.title('{}'.format(title))\n",
    "    plt.plot(train_history, label='train', zorder=1)\n",
    "    \n",
    "    points = np.array(val_history)\n",
    "    \n",
    "    plt.scatter(points[:, 0], points[:, 1], marker='+', s=180, c='orange', label='val', zorder=2)\n",
    "    plt.xlabel('train steps')\n",
    "    \n",
    "    plt.legend(loc='best')\n",
    "    plt.grid()\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "def train(model, opt, n_epochs):\n",
    "    train_log, train_acc_log = [], []\n",
    "    val_log, val_acc_log = [], []\n",
    "\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        print(\"Epoch {0} of {1}\".format(epoch, n_epochs))\n",
    "        train_loss, train_acc = train_epoch(model, opt, batchsize=batch_size)\n",
    "\n",
    "        val_loss, val_acc = test(model)\n",
    "\n",
    "        train_log.extend(train_loss)\n",
    "        train_acc_log.extend(train_acc)\n",
    "\n",
    "        steps = train_dataset.train_labels.shape[0] / batch_size\n",
    "        val_log.append((steps * (epoch + 1), np.mean(val_loss)))\n",
    "        val_acc_log.append((steps * (epoch + 1), np.mean(val_acc)))\n",
    "        \n",
    "        clear_output()\n",
    "        plot_history(train_log, val_log)    \n",
    "        plot_history(train_acc_log, val_acc_log, title='accuracy')   \n",
    "        \n",
    "        print(\"Epoch: {2}, val loss: {0}, val accuracy: {1}\".format(np.mean(val_loss), np.mean(val_acc), epoch))\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MjhiP5h-e6CA",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Создайте простейшую однослойную модель - однослойную полносвязную сеть и обучите ее с параметрами оптимизации, заданными ниже."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "TLB2iHNke6CB",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "class Flatten(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return x.view(x.size()[0], -1)\n",
    "    \n",
    "model = nn.Sequential(\n",
    "    #<your code>\n",
    "    )"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "z8ppSMDae6CE",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "opt = torch.optim.Adam(model.parameters(), lr=0.0005)\n",
    "train(model, opt, 10)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7smJP34Pe6CH",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Параметром обученной нейросети является матрица весов, в которой каждому классу соответствует один из 784-мерных столбцов. Визуализируйте обученные векторы для каждого из классов, сделав их двумерными изображениями 28-28. Для визуализации можно воспользоваться кодом для визуализации MNIST-картинок с предыдущих семинаров."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "WIwBt7cJe6CH",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "weights = #<your code>\n",
    "plt.figure(figsize=[10, 10])\n",
    "for i in range(10):\n",
    "    plt.subplot(5, 5, i + 1)\n",
    "    plt.title(\"Label: %i\" % i)\n",
    "    plt.imshow(weights[i].reshape([28, 28]), cmap='gray');"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TIzsqWU_e6CL",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Реализуйте Dropout-слой для полносвязной сети. Помните, что этот слой ведет себя по-разному во время обучения и во время применения. "
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "HaRFi9jqe6CL",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "class DropoutLayer(nn.Module):\n",
    "    def __init__(self, p):\n",
    "        super().__init__()\n",
    "        #<your code>\n",
    "\n",
    "    def forward(self, input):\n",
    "        if self.training:\n",
    "            #<your code>\n",
    "        else:\n",
    "            #<your code>"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gvq4PLN_e6CO",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Добавьте Dropout-слой в архитектуру сети, проведите оптимизацию с параметрами, заданными ранее, визуализируйте обученные веса. Есть ли разница между весами обученными с Dropout и без него? Параметр Dropout возьмите равным 0.7"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "YsfjKbTye6CO",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "modelDp = nn.Sequential(\n",
    "    #<your code>\n",
    "    )"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "KMlZHpXae6CR",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "opt = torch.optim.Adam(modelDp.parameters(), lr=0.0005)\n",
    "train(modelDp, opt, 10)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Xzzz9Tkje6CT",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "weights = #<your code>\n",
    "plt.figure(figsize=[10, 10])\n",
    "for i in range(10):\n",
    "    plt.subplot(5, 5, i + 1)\n",
    "    plt.title(\"Label: %i\" % i)\n",
    "    plt.imshow(weights[i].reshape([28, 28]), cmap='gray');"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "C5_G8wzQe6CW",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Обучите еще одну модель, в которой вместо Dropout-регуляризации используется L2-регуляризация с коэффициентом 0.05. (Параметр weight_decay в оптимизаторе). Визуализируйте веса и сравните с двумя предыдущими подходами."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ayzHCMx8e6CX",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "model = nn.Sequential(\n",
    "    Flatten(),\n",
    "    nn.Linear(input_size,num_classes),\n",
    "    nn.LogSoftmax(dim=-1)\n",
    "    )"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "pWYcCBZ7e6CZ",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "opt = torch.optim.Adam(model.parameters(), lr=0.0005, weight_decay=0.05)\n",
    "train(model, opt, 10)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "BJbA2mA3e6Cd",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "weights = #<your code>\n",
    "plt.figure(figsize=[10, 10])\n",
    "for i in range(10):\n",
    "    plt.subplot(5, 5, i + 1)\n",
    "    plt.title(\"Label: %i\" % i)\n",
    "    plt.imshow(weights[i].reshape([28, 28]), cmap='gray');"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nKtMonw3e6Cf",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Batch normalization\n",
    "\n",
    "Реализуйте BatchNormalization слой для полносвязной сети. В реализации достаточно только центрировать и разделить на корень из дисперсии, аффинную поправку (гамма и бета) в этом задании можно не реализовывать."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "LWvaRXmOe6Cg",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "class BnLayer(nn.Module):\n",
    "    def __init__(self, num_features):\n",
    "        super().__init__()\n",
    "        #<your code>\n",
    "\n",
    "    def forward(self, input):\n",
    "        if self.training:\n",
    "            #<your code>\n",
    "        else:\n",
    "            #<your code>\n",
    "        return #<your code>"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_tpNTUWVe6Ci",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Обучите трехслойную полносвязную сеть (размер скрытого слоя возьмите 100) с сигмоидами в качестве функций активации. "
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "xGNAd9Uae6Ci",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "model = nn.Sequential(\n",
    "    #<your code>\n",
    "    )"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "7qD2KJq6e6Ck",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "opt = torch.optim.RMSprop(model.parameters(), lr=0.01)\n",
    "train(model, opt, 3)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "M2IHZt_Se6Co",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Повторите обучение с теми же параметрами для сети с той же архитектурой, но с добавлением BatchNorm слоя (для всех трех скрытых слоев)."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "406kS0wFe6Co",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "modelBN = nn.Sequential(\n",
    "    #<your code>\n",
    "    )"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Uqt2bVKWe6Cq",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "source": [
    "opt = torch.optim.RMSprop(modelBN.parameters(), lr=0.01)\n",
    "train(modelBN, opt, 3)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "APw2lE1qe6Cs",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Сравните кривые обучения и сделайте вывод о влиянии BatchNorm на ход обучения."
   ]
  }
 ]
}