week03_architectures/homework.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZdIOldj_aag8",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Основные виды нейросетей (CNN и RNN)\n",
    "\n",
    "**Разработчик: Алексей Умнов**\n",
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mryab/dl-hse-ami/blob/master/week03_architectures/homework.ipynb)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PLYbee6p9xXZ",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Этот семинар будет состоять из двух частей: сначала мы позанимаемся реализацией сверточных и рекуррентных сетей, а потом поисследуем проблему затухающих и взрывающихся градиентов."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rAUuvVjg9xXc",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Сверточные сети\n",
    "\n",
    "Вернемся в очередной раз к датасету MNIST. Для начала загрузим данные и определим несколько полезных функций как на прошлом семинаре."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "w4fxgx-Ee2YN",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Load dependencies\n",
    "!wget --quiet --show-progress \"https://raw.githubusercontent.com/mryab/dl-hse-ami/master/week03_architectures/util.py\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "diIxeLEK9xXe",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "from IPython.display import clear_output\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "random.seed(42)\n",
    "np.random.seed(42)\n",
    "torch.manual_seed(42)\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed_all(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "epP3TUcA9xXk",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from util import load_mnist\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = load_mnist(flatten=True)\n",
    "\n",
    "plt.figure(figsize=[6, 6])\n",
    "for i in range(4):\n",
    "    plt.subplot(2, 2, i + 1)\n",
    "    plt.title(\"Label: %i\" % y_train[i])\n",
    "    plt.imshow(X_train[i].reshape([28, 28]), cmap='gray');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4o8KJ0Rh9xXq",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from util import iterate_minibatches\n",
    "\n",
    "def train_epoch(model, optimizer, batchsize=32):\n",
    "    loss_log, acc_log = [], []\n",
    "    model.train()\n",
    "    for x_batch, y_batch in iterate_minibatches(X_train, y_train, batchsize=batchsize, shuffle=True):\n",
    "        data = torch.from_numpy(x_batch.astype(np.float32))\n",
    "        target = torch.from_numpy(y_batch.astype(np.int64))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        \n",
    "        pred = torch.max(output, 1)[1].numpy()\n",
    "        acc = np.mean(pred == y_batch)\n",
    "        acc_log.append(acc)\n",
    "        \n",
    "        loss = F.nll_loss(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        loss = loss.item()\n",
    "        loss_log.append(loss)\n",
    "    return loss_log, acc_log\n",
    "\n",
    "def test(model):\n",
    "    loss_log, acc_log = [], []\n",
    "    model.eval()\n",
    "    for x_batch, y_batch in iterate_minibatches(X_val, y_val, batchsize=32, shuffle=True):\n",
    "        data = torch.from_numpy(x_batch.astype(np.float32))\n",
    "        target = torch.from_numpy(y_batch.astype(np.int64))\n",
    "\n",
    "        output = model(data)\n",
    "        loss = F.nll_loss(output, target)\n",
    "        \n",
    "        pred = torch.max(output, 1)[1].numpy()\n",
    "        acc = np.mean(pred == y_batch)\n",
    "        acc_log.append(acc)\n",
    "        \n",
    "        loss = loss.item()\n",
    "        loss_log.append(loss)\n",
    "    return loss_log, acc_log\n",
    "\n",
    "def plot_history(train_history, val_history, title='loss'):\n",
    "    plt.figure()\n",
    "    plt.title('{}'.format(title))\n",
    "    plt.plot(train_history, label='train', zorder=1)\n",
    "    \n",
    "    points = np.array(val_history)\n",
    "    \n",
    "    plt.scatter(points[:, 0], points[:, 1], marker='+', s=180, c='orange', label='val', zorder=2)\n",
    "    plt.xlabel('train steps')\n",
    "    \n",
    "    plt.legend(loc='best')\n",
    "    plt.grid()\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "def train(model, opt, n_epochs):\n",
    "    train_log, train_acc_log = [], []\n",
    "    val_log, val_acc_log = [], []\n",
    "\n",
    "    batchsize = 32\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        print(\"Epoch {} of {}\".format(epoch, n_epochs))\n",
    "        train_loss, train_acc = train_epoch(model, opt, batchsize=batchsize)\n",
    "\n",
    "        val_loss, val_acc = test(model)\n",
    "\n",
    "        train_log.extend(train_loss)\n",
    "        train_acc_log.extend(train_acc)\n",
    "\n",
    "        steps = len(X_train) / batchsize\n",
    "        val_log.append((steps * (epoch + 1), np.mean(val_loss)))\n",
    "        val_acc_log.append((steps * (epoch + 1), np.mean(val_acc)))\n",
    "\n",
    "        clear_output()\n",
    "        plot_history(train_log, val_log)    \n",
    "        plot_history(train_acc_log, val_acc_log, title='accuracy')\n",
    "        print(\"Epoch {} error = {:.2%}\".format(epoch, 1 - val_acc_log[-1][1]))\n",
    "            \n",
    "    print(\"Final error: {:.2%}\".format(1 - val_acc_log[-1][1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "E9gyKDDd9xXw",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "**Задание 1:** Реализуйте сверточную сеть, которая состоит из двух последовательных применений свертки, relu и max-пулинга, а потом полносвязного слоя. Подберите параметры так, чтобы на выходе последнего слоя размерность тензора была 4 x 4 x 16. В коде ниже используется обертка nn.Sequential, ознакомьтесь с ее интерфейсом.\n",
    "\n",
    "Добейтесь, чтобы ошибка классификации после обучения (см. ниже) была не выше 1.5%."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GcpwpeTE9xXy",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            # <your code here>\n",
    "        )\n",
    "        \n",
    "        self.classifier = nn.Linear(4 * 4 * 16, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        # <your code here>\n",
    "        return F.log_softmax(out, dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6jDp1wX99xX3",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Посчитаем количество обучаемых параметров сети (полносвязные сети с прошлого семинара имеют 30-40 тысяч параметров)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Rwu1hOND9xX6",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def count_parameters(model):\n",
    "    model_parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "    return sum([np.prod(p.size()) for p in model_parameters])\n",
    "\n",
    "model = ConvNet()\n",
    "print(\"Total number of trainable parameters:\", count_parameters(model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-65n280K9xX_",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "opt = torch.optim.RMSprop(model.parameters(), lr=0.001)\n",
    "train(model, opt, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PJFnA-sN9xYK",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Мы с легкостью получили качество классификаци лучше, чем было раньше с помощью полносвязных сетей. На самом деле для более честного сравнения нужно поисследовать обе архитектуры и подождать побольше итераций до сходимости, но в силу ограниченности вычислительных ресурсов мы это сделать не можем. Результаты из которых \"выжали максимум\" можно посмотреть например, на этой странице: http://yann.lecun.com/exdb/mnist/, и там видно, что качество сверточных сетей гораздо выше. А если работать с более сложными изоражениями (например, с ImageNet), то сверточные сети побеждают с большим отрывом.\n",
    "\n",
    "**Упражнение:** Вспомните материалы лекции и ответьте на вопросы ниже:\n",
    "* Почему сверточные сети обладают таким преимуществом именно для изображений?\n",
    "* Почему несмотря на малое количество параметров обучение сверточных сетей занимает так много времени?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hP6KOZ729xYP",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Рекуррентные сети\n",
    "\n",
    "Для рекуррентных сетей используем датасет с именами и будем определять из какого языка произошло данное имя. Для этого построим рекуррентную сеть, которая с именами на уровне символов. Для начала скачаем файлы и конвертируем их к удобному формату (можно не особо вникать в этот код)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "n5wPAmAy9xYR",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# На Windows придется скачать архив по ссылке (~3Mb) и распаковать самостоятельно\n",
    "!wget --quiet --show-progress https://download.pytorch.org/tutorial/data.zip\n",
    "!unzip -q ./data.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZKxrnmgW9xYY",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from io import open\n",
    "import glob\n",
    "\n",
    "def findFiles(path): return glob.glob(path)\n",
    "\n",
    "print(findFiles('data/names/*.txt'))\n",
    "\n",
    "import unicodedata\n",
    "import string\n",
    "\n",
    "all_letters = string.ascii_letters + \" .,;'\"\n",
    "n_letters = len(all_letters)\n",
    "\n",
    "# Turn a Unicode string to plain ASCII, thanks to http://stackoverflow.com/a/518232/2809427\n",
    "def unicodeToAscii(s):\n",
    "    return ''.join(\n",
    "        c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn'\n",
    "        and c in all_letters\n",
    "    )\n",
    "\n",
    "print(unicodeToAscii('Ślusàrski'))\n",
    "\n",
    "# Build the category_lines dictionary, a list of names per language\n",
    "category_lines = {}\n",
    "all_categories = []\n",
    "\n",
    "# Read a file and split into lines\n",
    "def readLines(filename):\n",
    "    lines = open(filename, encoding='utf-8').read().strip().split('\\n')\n",
    "    return [unicodeToAscii(line) for line in lines]\n",
    "\n",
    "for filename in findFiles('data/names/*.txt'):\n",
    "    category = filename.split('/')[-1].split('.')[0]\n",
    "    all_categories.append(category)\n",
    "    lines = readLines(filename)\n",
    "    category_lines[category] = lines\n",
    "\n",
    "n_categories = len(all_categories)\n",
    "\n",
    "def categoryFromOutput(output):\n",
    "    top_n, top_i = output.topk(1)\n",
    "    category_i = top_i[0][0]\n",
    "    return all_categories[category_i], category_i"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "319MdTEC9xYe",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Определим несколько удобных функций для конвертации букв и слов в тензоры.\n",
    "\n",
    "**Задание 2**: напишите последнюю функцию для конвертации слова в тензор."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "R3fUjiX_9xYg",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Find letter index from all_letters, e.g. \"a\" = 0\n",
    "def letterToIndex(letter):\n",
    "    return all_letters.find(letter)\n",
    "\n",
    "# Just for demonstration, turn a letter into a <1 x n_letters> Tensor\n",
    "def letterToTensor(letter):\n",
    "    tensor = torch.zeros(1, n_letters)\n",
    "    tensor[0][letterToIndex(letter)] = 1\n",
    "    return tensor\n",
    "\n",
    "# Turn a line into a <line_length x 1 x n_letters>,\n",
    "# or an array of one-hot letter vectors\n",
    "def lineToTensor(line):\n",
    "    # <your code here>\n",
    "\n",
    "print(letterToTensor('J'))\n",
    "print(lineToTensor('Jones').size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4bcXqGXx9xYl",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "**Задание 3:** Реализуйте однослойную рекуррентную сеть."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "whTbgC5R9xYn",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class RNNCell(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(RNNCell, self).__init__()\n",
    "        \n",
    "        self.hidden_size = hidden_size\n",
    "        # <your code here>\n",
    "        # <end>\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        # <your code here>\n",
    "        # <end>\n",
    "        return hidden\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, self.hidden_size)\n",
    "\n",
    "n_hidden = 128\n",
    "rnncell = RNNCell(n_letters, n_hidden)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6rnAXB009xYr",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Предсказание будем осуществлять при помощи линейного класссификатора поверх скрытых состояний сети."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "j6rWQ4_a9xYt",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "classifier = nn.Sequential(nn.Linear(n_hidden, n_categories), nn.LogSoftmax(dim=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B7KVpW449xYz",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Проверим, что все корректно работает: выходы классификаторы должны быть лог-вероятностями."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "r9fgCAXd9xY1",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "input = letterToTensor('A')\n",
    "hidden = torch.zeros(1, n_hidden)\n",
    "\n",
    "output = classifier(rnncell(input, hidden))\n",
    "print(output)\n",
    "print(torch.exp(output).sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ATHA1a8J9xY6",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "input = lineToTensor('Albert')\n",
    "hidden = torch.zeros(1, n_hidden)\n",
    "\n",
    "output = classifier(rnncell(input[0], hidden))\n",
    "print(output)\n",
    "print(torch.exp(output).sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HV0I9pd99xY-",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Для простоты в этот раз будем оптимизировать не по мини-батчам, а по отдельным примерам. Ниже несколько полезных функций для этого."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "97WUqv-O9xZB",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def randomChoice(l):\n",
    "    return l[random.randint(0, len(l) - 1)]\n",
    "\n",
    "def randomTrainingExample():\n",
    "    category = randomChoice(all_categories)\n",
    "    line = randomChoice(category_lines[category])\n",
    "    category_tensor = torch.tensor([all_categories.index(category)], dtype=torch.long)\n",
    "    line_tensor = lineToTensor(line)\n",
    "    return category, line, category_tensor, line_tensor\n",
    "\n",
    "for i in range(10):\n",
    "    category, line, category_tensor, line_tensor = randomTrainingExample()\n",
    "    print('category =', category, '/ line =', line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "o8KAO_JP9xZG",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "**Задание 4:** Реализуйте вычисление ответа в функции train. Если все сделано правильно, то точность на обучающей выборке должна быть не менее 70%."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BzuMsHyD9xZG",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "def train(category, category_tensor, line_tensor, optimizer):\n",
    "    hidden = rnncell.initHidden()\n",
    "\n",
    "    rnncell.zero_grad()\n",
    "    classifier.zero_grad()\n",
    "\n",
    "    # <your code here>\n",
    "    # use rnncell and classifier\n",
    "    # <end>\n",
    "\n",
    "    loss = F.nll_loss(output, category_tensor)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    acc = (categoryFromOutput(output)[0] == category)\n",
    "\n",
    "    return loss.item(), acc\n",
    "\n",
    "n_iters = 50000\n",
    "plot_every = 1000\n",
    "\n",
    "current_loss = 0\n",
    "all_losses = []\n",
    "current_acc = 0\n",
    "all_accs = []\n",
    "\n",
    "n_hidden = 128\n",
    "\n",
    "rnncell = RNNCell(n_letters, n_hidden)\n",
    "classifier = nn.Sequential(nn.Linear(n_hidden, n_categories), nn.LogSoftmax(dim=1))\n",
    "params = list(rnncell.parameters()) + list(classifier.parameters())\n",
    "opt = torch.optim.RMSprop(params, lr=0.001)\n",
    "for iter in tqdm(range(1, n_iters + 1)):\n",
    "    category, line, category_tensor, line_tensor = randomTrainingExample()\n",
    "    loss, acc = train(category, category_tensor, line_tensor, opt)\n",
    "    current_loss += loss\n",
    "    current_acc += acc\n",
    "\n",
    "    # Add current loss avg to list of losses\n",
    "    if iter % plot_every == 0:\n",
    "        all_losses.append(current_loss / plot_every)\n",
    "        current_loss = 0\n",
    "        all_accs.append(current_acc / plot_every)\n",
    "        current_acc = 0\n",
    "        \n",
    "plt.figure()\n",
    "plt.title(\"Loss\")\n",
    "plt.plot(all_losses)\n",
    "plt.grid()\n",
    "plt.show()\n",
    "\n",
    "plt.figure()\n",
    "plt.title(\"Accuracy\")\n",
    "plt.plot(all_accs)\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "81DCBYA69xZL",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Затухающие и взрывающиеся градиенты\n",
    "\n",
    "Эксперименты будем проводить опять на датасете MNIST, но будем работать с полносвязными сетями. В этом разделе мы не будем пытаться подобрать более удачную архитектуру, нам интересно только посмотреть на особенности обучения глубоких сетей."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lj2o7YMV9xZL",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from util import load_mnist\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = load_mnist(flatten=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gj-hnp_B9xZP",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Для экспериментов нам понадобится реализовать сеть, в которой можно легко менять количество слоев. Также эта сеть должна сохранять градиенты на всех слоях, чтобы потом мы могли посмотреть на их величины.\n",
    "\n",
    "**Задание 5:** допишите недостающую часть кода ниже."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "D_zBqPek9xZQ",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DeepDenseNet(nn.Module):\n",
    "    def __init__(self, n_layers, hidden_size, activation):\n",
    "        super().__init__()\n",
    "        self.activation = activation\n",
    "        \n",
    "        l0 = nn.Linear(X_train.shape[1], hidden_size)\n",
    "        self.weights = [l0.weight]\n",
    "        self.layers = [l0]\n",
    "        \n",
    "        # <your code here>\n",
    "        \n",
    "        self.seq = nn.Sequential(*self.layers)\n",
    "        \n",
    "        for l in self.weights:\n",
    "            l.retain_grad()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.seq(x)\n",
    "        return F.log_softmax(out, dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3vd1b00P9xZV",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Модифицируем наши функции обучения, чтобы они также рисовали графики изменения градиентов."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "k-4-vgt29xZY",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import scipy.sparse.linalg\n",
    "\n",
    "def train_epoch_grad(model, optimizer, batchsize=32):\n",
    "    loss_log, acc_log = [], []\n",
    "    grads = [[] for l in model.weights]\n",
    "    model.train()\n",
    "    for x_batch, y_batch in iterate_minibatches(X_train, y_train, batchsize=batchsize, shuffle=True):\n",
    "        # data preparation\n",
    "        data = torch.from_numpy(x_batch.astype(np.float32))\n",
    "        target = torch.from_numpy(y_batch.astype(np.int64))\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        \n",
    "        pred = torch.max(output, 1)[1].numpy()\n",
    "        acc = np.mean(pred == y_batch)\n",
    "        acc_log.append(acc)\n",
    "        \n",
    "        loss = F.nll_loss(output, target)\n",
    "        # compute gradients\n",
    "        loss.backward()\n",
    "        # make a step\n",
    "        optimizer.step()\n",
    "        loss = loss.item()\n",
    "        loss_log.append(loss)\n",
    "        \n",
    "        for g, l in zip(grads, model.weights):\n",
    "            g.append(np.linalg.norm(l.grad.numpy()))\n",
    "    return loss_log, acc_log, grads\n",
    "\n",
    "\n",
    "def train_grad(model, opt, n_epochs):\n",
    "    train_log, train_acc_log = [], []\n",
    "    val_log, val_acc_log = [], []\n",
    "    grads_log = None\n",
    "\n",
    "    batchsize = 32\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        print(\"Epoch {} of {}\".format(epoch, n_epochs))\n",
    "        train_loss, train_acc, grads = train_epoch_grad(model, opt, batchsize=batchsize)\n",
    "        if grads_log is None:\n",
    "            grads_log = grads\n",
    "        else:\n",
    "            for a, b in zip(grads_log, grads):\n",
    "                a.extend(b)\n",
    "\n",
    "        val_loss, val_acc = test(model)\n",
    "\n",
    "        train_log.extend(train_loss)\n",
    "        train_acc_log.extend(train_acc)\n",
    "\n",
    "        steps = len(X_train) / batchsize\n",
    "        val_log.append((steps * (epoch + 1), np.mean(val_loss)))\n",
    "        val_acc_log.append((steps * (epoch + 1), np.mean(val_acc)))\n",
    "\n",
    "        # display all metrics\n",
    "        clear_output()\n",
    "        plot_history(train_log, val_log)    \n",
    "        plot_history(train_acc_log, val_acc_log, title='accuracy')    \n",
    "\n",
    "        plt.figure()\n",
    "        all_vals = []\n",
    "        for i, g in enumerate(grads_log):\n",
    "            w = np.ones(100)\n",
    "            w /= w.sum()\n",
    "            vals = np.convolve(w, g, mode='valid')\n",
    "            plt.semilogy(vals, label=str(i+1), color=plt.cm.coolwarm((i / len(grads_log))))\n",
    "            all_vals.extend(vals)\n",
    "        plt.legend(loc='best')\n",
    "        plt.grid()\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "J5fobtIJ9xZc",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "**Задание 6:**\n",
    "* Обучите сети глубины 10 и больше с сигмоидой в качестве активации. Исследуйте, как глубина влияет на качество обучения и поведение градиентов на далеких от выхода слоях.\n",
    "* Теперь замените активацию на ReLU и посмотрите, что получится."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ASylWcPc9xZd",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I38UAdgR9xZt",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Теперь попробуем добавить в сеть skip-connections (по примеру ResNet) вместо замены сигмоиды на relu и посмотрим, что получится. Запихнуть все слои в nn.Sequential и просто их применить теперь не получится - вместо этого мы их применим вручную. Но положить их в отдельный модуль nn.ModuleList все равно нужно, иначе torch не сможет их найти и оптимизировать.\n",
    "\n",
    "**Задание 7:** допишите недостающую часть кода ниже."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3UZ3MTB-9xZv",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class DeepDenseResNet(nn.Module):\n",
    "    def __init__(self, n_layers, hidden_size, activation):\n",
    "        super().__init__()\n",
    "        self.activation = activation\n",
    "        \n",
    "        l0 = nn.Linear(X_train.shape[1], hidden_size)\n",
    "        self.weights = [l0.weight]\n",
    "        self.layers = [l0]\n",
    "        \n",
    "        for i in range(1, n_layers - 1):\n",
    "            l = nn.Linear(hidden_size, hidden_size)\n",
    "            self.layers.append(l)\n",
    "            self.weights.append(l.weight)\n",
    "            \n",
    "        l = nn.Linear(hidden_size, 10)\n",
    "        self.layers.append(l)\n",
    "        self.weights.append(l.weight)\n",
    "        \n",
    "        self.seq = nn.Sequential(*self.layers)\n",
    "        \n",
    "        for l in self.weights:\n",
    "            l.retain_grad()\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \n",
    "        # <your code here>\n",
    "        \n",
    "        return F.log_softmax(x, dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y08rBtNH9xZz",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Убедимся, что такая сеть отлично учится даже на большом числе слоев."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "878WBPW19xZ2",
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model = DeepDenseResNet(n_layers=20, hidden_size=10, activation=nn.Sigmoid)\n",
    "opt = torch.optim.RMSprop(model.parameters(), lr=0.001)\n",
    "\n",
    "train_grad(model, opt, 10)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "DL21-fall-shw3.ipynb",
   "provenance": []
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}