4.b-implementation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementation of neural network training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will be using what we have learned to train a neural network.\n",
    "\n",
    "In particular, we will be using the adam update rule, considering numerical stability, and saving best set of parameters."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the dataset\n",
    "\n",
    "We will be using the [iris data set](https://archive.ics.uci.edu/ml/datasets/iris)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = os.path.join('data', 'iris.data')\n",
    "dataset = pd.read_csv(data_path, names=['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)', 'class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)  # Size of dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.3</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>7.0</td>\n",
       "      <td>3.2</td>\n",
       "      <td>4.7</td>\n",
       "      <td>1.4</td>\n",
       "      <td>Iris-versicolor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)  \\\n",
       "49                5.0               3.3                1.4               0.2   \n",
       "50                7.0               3.2                4.7               1.4   \n",
       "\n",
       "              class  \n",
       "49      Iris-setosa  \n",
       "50  Iris-versicolor  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[49:51]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The \"class\" column is the type of flower.\n",
    "\n",
    "The other four column are the length and width of sepal and petal, these are the four features given in the dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This time around we will train on the full data and try to classify all three classes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input\n",
    "The input will be vector of 4 scalars, the length and width of sepal and petal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = {}\n",
    "fields = ['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']\n",
    "data['input'] = dataset.loc[:, fields].to_numpy(dtype=np.float_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5. , 3.3, 1.4, 0.2],\n",
       "       [7. , 3.2, 4.7, 1.4]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A sample of input\n",
    "data['input'][49:51]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Target\n",
    "The target will be a one-hot vector representing the class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_class = {0: \"Iris-setosa\", 1: \"Iris-versicolor\", 2: \"Iris-virginica\"}\n",
    "to_index = {\"Iris-setosa\": 0, \"Iris-versicolor\": 1, \"Iris-virginica\": 2}\n",
    "\n",
    "def to_one_hot(num_class, indexes):\n",
    "    \"\"\"Outputs a matrix of one hot vectors.\"\"\"\n",
    "    # Create vectors of zeros.\n",
    "    n = len(indexes)\n",
    "    one_hot = np.zeros((n, num_class), dtype=np.float_)\n",
    "    # Set correct class to 1.\n",
    "    one_hot[np.arange(n), indexes] = 1\n",
    "    return one_hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['target'] = to_one_hot(3, [to_index[t] for t in dataset.loc[:,'class']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1., 0., 0.],\n",
       "       [0., 1., 0.]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A sample of target\n",
    "data['target'][49:51]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting and shuffling the data\n",
    "\n",
    "We need to split the dataset into training data, validation data, and test data.\n",
    "\n",
    "Since there is only 150 samples in the dataset, it is important to ensure there is a even distribution of classes in each type of data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_ratio = 1/5\n",
    "test_ratio = 1/5\n",
    "# rest for training data\n",
    "\n",
    "num_per_class = 50\n",
    "v_n = int(num_per_class * valid_ratio)  # Number of valid per class\n",
    "t_n = int(num_per_class * test_ratio)\n",
    "\n",
    "valid_idx = np.r_[0:v_n, 50:50 + v_n, 100:100 + v_n]\n",
    "test_idx = np.r_[v_n:v_n + t_n, 50 + v_n:50 + v_n + t_n, 100 + v_n:100 + v_n + t_n]\n",
    "training_idx = np.r_[v_n:v_n + t_n, 50 + v_n:50 + v_n + t_n, 100 + v_n:100 + v_n + t_n]\n",
    "\n",
    "valid = {}\n",
    "test = {}\n",
    "training = {}\n",
    "\n",
    "valid['input'] = data['input'][valid_idx]\n",
    "valid['target'] = data['target'][valid_idx]\n",
    "test['input'] = data['input'][test_idx]\n",
    "test['target'] = data['target'][test_idx]\n",
    "training['input'] = data['input'][training_idx]\n",
    "training['target'] = data['target'][training_idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Shuffle the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "training['input'], training['target'] = shuffle(training['input'], training['target'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss function\n",
    "We will use cross entropy loss as loss function. The softmax function and loss function is modified for numerical stability as explained in the previous lesson.\n",
    "\n",
    "The cross entropy loss is modified to prevent calculating the log of 0, which is infinity and will overflow. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    e = np.exp(x - np.max(x))\n",
    "    return e / np.sum(e)\n",
    "\n",
    "class CrossEntropyLoss:\n",
    "    @staticmethod\n",
    "    def forward(x, t):\n",
    "        x = softmax(x)\n",
    "        x = np.maximum(1e-8, x)  # Prevent x = 0\n",
    "        return -np.log(x) @ t\n",
    "        \n",
    "    @staticmethod\n",
    "    def backward(x, t):\n",
    "        x = softmax(x)\n",
    "        return x - t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activation function\n",
    "\n",
    "We will use ReLU as activation function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReLU:\n",
    "    \n",
    "    @staticmethod\n",
    "    def forward(x):\n",
    "        return np.maximum(0, x)\n",
    "    \n",
    "    @staticmethod\n",
    "    def backward(x):\n",
    "        x[x <= 0] = 0\n",
    "        x[x > 0] = 1\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update rule\n",
    "\n",
    "We will be using the Adam optimizer. The implementation is modified to take Python dictionary of params and update them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Adam:\n",
    "    \n",
    "    def __init__(self, alpha, beta1, beta2, epsilon):\n",
    "        self.a = alpha\n",
    "        self.b1 = beta1\n",
    "        self.b2 = beta2\n",
    "        self.eps = epsilon\n",
    "        self.t = 0\n",
    "    \n",
    "    def set_param(self, params):\n",
    "        \"\"\"Create the storage for m and n values for a given parameter set.\"\"\"\n",
    "        self.m = {}\n",
    "        self.n = {}\n",
    "        for key, lst in params.items():\n",
    "            m_lst = []\n",
    "            n_lst = []\n",
    "            for param in lst:\n",
    "                m_lst.append(np.zeros_like(param))\n",
    "                n_lst.append(np.zeros_like(param))\n",
    "            self.m[key] = m_lst\n",
    "            self.n[key] = n_lst\n",
    "    \n",
    "    def __call__(self, params, grads):\n",
    "        self.t += 1\n",
    "        for key in params:\n",
    "            num = len(params[key])\n",
    "            for i in range(num):\n",
    "                self.m[key][i] = self.b1 * self.m[key][i] + (1 - self.b1) * grads[key][i]\n",
    "                self.n[key][i] = self.b1 * self.n[key][i] + (1 - self.b1) * (grads[key][i] ** 2)\n",
    "                m_hat = self.m[key][i] / (1 - self.b1 ** self.t)\n",
    "                n_hat = self.n[key][i] / (1 - self.b2 ** self.t)\n",
    "                params[key][i] -= self.a * m_hat / (np.sqrt(n_hat) + self.eps)\n",
    "        return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modular network implementation\n",
    "\n",
    "The backpropagation implementation in previous lesson only works for networks which solely consist of fully connected feed forward layers with same activation function.\n",
    "\n",
    "Many common network include multiple type of layers and different types of activation function.\n",
    "\n",
    "The common frameworks to train neural network offers a module approach."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rather than using the delta short hand, we will be using the following formula:\n",
    "\n",
    "\\begin{align}\n",
    "& \\frac{\\partial E}{\\partial w_{ij}} = \\frac{\\partial E}{\\partial o_j} \\cdot \\frac{\\partial h_j(s_j)}{\\partial s_j} \\cdot o_i \\\\\n",
    "& \\frac{\\partial E}{\\partial o_j} = \\begin{cases} \\frac{\\partial E(t_j, o_j)}{\\partial o_j} & \\text{if $j$ is an output neuron} \\\\ \\sum_{l \\in L} \\frac{\\partial E}{\\partial o_l} \\cdot \\frac{\\partial h_l(s_l)}{\\partial s_l} \\cdot w_{jl} & \\text{otherwise}\\end{cases}\n",
    "\\end{align}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementation of the fully connected feed forward layer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FullyConnected:\n",
    "    \n",
    "    def __init__(self, input_size, output_size, activation):\n",
    "        self.weight = np.random.rand(input_size, output_size)\n",
    "        self.bias = np.zeros(output_size)\n",
    "        self.activation = activation\n",
    "        self.x_cache = None\n",
    "    \n",
    "    def register_param(self):\n",
    "        \"\"\"Return all tunable parameters of this layer\"\"\"\n",
    "        return [self.weight, self.bias]\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \"\"\"Calculate layer output with given input.\"\"\"\n",
    "        # Store network input in cache.\n",
    "        self.x_cache = x\n",
    "       \n",
    "        x = (x @ self.weight) + self.bias\n",
    "        x = self.activation.forward(x)\n",
    "        return x\n",
    "    \n",
    "    def backward(self, output_der):\n",
    "        \"\"\"Calculate derivative of layer weights and biases with given layer output derivative\"\"\"\n",
    "        s_j_der = self.activation.backward(self.x_cache @ self.weight)\n",
    "        weight_grad = np.outer(self.x_cache, output_der * s_j_der)\n",
    "        bias_grad = output_der * s_j_der\n",
    "        return [weight_grad, bias_grad]\n",
    "    \n",
    "    def input_der(self, output_der):\n",
    "        \"\"\"Helper function to calculate layer input derivative to be used for the previous layer\"\"\"\n",
    "        s_j_der = self.activation.backward(self.x_cache @ self.weight)\n",
    "        return self.weight @ (output_der * s_j_der)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `forward` function takes the layer inputs in vector form and produce layer outputs in vector form.\n",
    "\n",
    "The `backward` function takes the partial derivative of the loss with respect to the layer output ($\\frac{\\partial E}{\\partial o_j}$ of the whole layer) and produce the partial derivative of the loss with respect to the layer weight ($\\frac{\\partial E}{\\partial w_{ij}}$ of whole layer)\n",
    "\n",
    "The `input_der` function takes the partial derivative of the loss with respect to the layer output ($\\frac{\\partial E}{\\partial o_j}$ of the whole layer) and produce the partial derivative of the loss with respect to the layer input. This is used as the partial derivative of the loss with respect to the layer output for the previous layer ($\\frac{\\partial E}{\\partial o_j}$ of the whole layer)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Network structure\n",
    "\n",
    "The input layer has 4 units, since the input is 4 scalars.\n",
    "\n",
    "There are 2 hidden layers of 100 units.\n",
    "\n",
    "The output layer has 3 units, since the output is 3 classes.\n",
    "\n",
    "The activation function is ReLU for all layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class IrisNetwork:\n",
    "    \n",
    "    def __init__(self, loss, optimizer):\n",
    "        self.loss = loss\n",
    "        self.optim = optimizer\n",
    "        self.layers = [FullyConnected(4, 100, ReLU), \n",
    "                       FullyConnected(100, 100, ReLU),\n",
    "                       FullyConnected(100, 3, ReLU)]\n",
    "        \n",
    "        # Register parameters from each layer\n",
    "        self.params = {}\n",
    "        for i, layer in enumerate(self.layers):\n",
    "            self.params[i] = layer.register_param()\n",
    "        self.optim.set_param(self.params)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer.forward(x)\n",
    "        return x\n",
    "    \n",
    "    def backward(self, x, t):\n",
    "        self.grads = {}\n",
    "        \n",
    "        output_der = self.loss.backward(x, t)\n",
    "        for i, layer in reversed(list(enumerate(self.layers))):\n",
    "            der_param = layer.backward(output_der)\n",
    "            self.grads[i] = der_param\n",
    "            # Calculate output derivative for previous layer.\n",
    "            output_der = layer.input_der(output_der)\n",
    "        return\n",
    "    \n",
    "    def update(self):\n",
    "        self.optim(self.params, self.grads)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training\n",
    "\n",
    "Train the model, and save best model on validation data during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "adam = Adam(0.01, 0.9, 0.999, 1e-8)\n",
    "loss = CrossEntropyLoss\n",
    "\n",
    "# Initialize the network\n",
    "iris_network = IrisNetwork(loss, adam)\n",
    "\n",
    "epoch = 50  # How many time to train the network with the whole training data.\n",
    "num_e = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "At     5 epoch,\tvalidation accuracy: 66.67\tLoss: 327.7365285997662\n",
      "At    10 epoch,\tvalidation accuracy: 66.67\tLoss: 293.23129308878424\n",
      "At    15 epoch,\tvalidation accuracy: 66.67\tLoss: 245.31443423714842\n",
      "At    20 epoch,\tvalidation accuracy: 100.00\tLoss: 170.7149215299162\n",
      "At    25 epoch,\tvalidation accuracy: 100.00\tLoss: 109.98302591645019\n",
      "At    30 epoch,\tvalidation accuracy: 100.00\tLoss: 51.77957669560662\n",
      "At    35 epoch,\tvalidation accuracy: 100.00\tLoss: 58.97381317610746\n",
      "At    40 epoch,\tvalidation accuracy: 100.00\tLoss: 6.015790340378242\n",
      "At    45 epoch,\tvalidation accuracy: 100.00\tLoss: 15.82838167519875\n",
      "At    50 epoch,\tvalidation accuracy: 100.00\tLoss: 5.216048953594284\n"
     ]
    }
   ],
   "source": [
    "valid_accu = 0\n",
    "training_loss = 0\n",
    "\n",
    "for e in range(epoch):     \n",
    "    # Test with validation data\n",
    "    correct_count = 0\n",
    "    total_count = 0\n",
    "    for inputs, targets in zip(valid['input'], valid['target']):\n",
    "        outputs = iris_network.forward(inputs)\n",
    "\n",
    "        total_count += 1\n",
    "        if np.argmax(outputs) == np.argmax(targets):\n",
    "            correct_count += 1\n",
    "\n",
    "    accu = correct_count / total_count\n",
    "    \n",
    "    # Save model if current best\n",
    "    if accu >= valid_accu:\n",
    "        valid_accu = accu\n",
    "        best_params = iris_network.params\n",
    "    \n",
    "    # Train network with training data\n",
    "    for inputs, targets in zip(training['input'], training['target']):\n",
    "        outputs = iris_network.forward(inputs)\n",
    "        iris_network.backward(outputs, targets)\n",
    "        iris_network.update()\n",
    "        \n",
    "        training_loss += loss.forward(outputs, targets) / num_e\n",
    "\n",
    "    # Print training info every set amount of epoch\n",
    "    if not (e + 1) % num_e:\n",
    "        print(f\"At {e + 1: 5} epoch,\\tvalidation accuracy: {valid_accu * 100:.2f}\\tLoss: {training_loss}\")\n",
    "        training_loss = 0\n",
    "        \n",
    "# Load network with best parameters\n",
    "iris_network.params = best_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network accuracy: 100.0%\n"
     ]
    }
   ],
   "source": [
    "# Test the network:\n",
    "correct_count = 0\n",
    "total_count = 0\n",
    "for inputs, targets in zip(test['input'], test['target']):\n",
    "    outputs = iris_network.forward(inputs)\n",
    "\n",
    "    total_count += 1\n",
    "    if np.argmax(outputs) == np.argmax(targets):\n",
    "        correct_count += 1\n",
    "\n",
    "accu = correct_count / total_count\n",
    "\n",
    "print(f\"Network accuracy: {accu * 100}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normally the difference between the validation accuracy and test accuracy should be small, but since we are using a small dataset, it might be larger.\n",
    "\n",
    "Either way, the network should be trained faster and performing better on a harder task (classification of three classes compared to two classes)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tools and library for training networks\n",
    "\n",
    "The two most common python libraries for training neural network are: PyTorch and TensorFlow.\n",
    "\n",
    "Both framework work similar to our modular implementation. However, nowadays both framework supports automatic differentiation, which means you don't have to implement the derivative functions if using predefined modules by the framework (or if your module is solely consist of predefined modules).\n",
    "\n",
    "Usually your module is going to be solely consist of predefined modules, since basic operations such as addition, multiplication, etc. are already predefined. Therefore to use these frameworks, you only need to define the structure of the network, loss function, learning rate, and optimizer/update rule. The rest will be taken care of by the framework.\n",
    "\n",
    "However, it is still important to understand how the framework works. This knowledge can help in debugging, and improving models. Especially, if you are developing a new module."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Practice"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 1\n",
    "\n",
    "Add gradient clipping to the above neural network implementation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# code here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise 2\n",
    "\n",
    "Train the network with gradient clipping. Find a good set of hyperparameters (learning rate, etc.)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# code here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Solution to exercise 1.\n",
    "\n",
    "def gradient_clipping(g, T):\n",
    "    g_norm = np.linalg.norm(g)\n",
    "    if g_norm > T:\n",
    "        g = T * g / g_norm\n",
    "    return g\n",
    "\n",
    "class GradClipNetwork:\n",
    "    \n",
    "    def __init__(self, loss, optimizer, t):\n",
    "        self.loss = loss\n",
    "        self.optim = optimizer\n",
    "        self.t = t\n",
    "        self.layers = [FullyConnected(4, 100, ReLU), \n",
    "                       FullyConnected(100, 100, ReLU),\n",
    "                       FullyConnected(100, 3, ReLU)]\n",
    "        \n",
    "        # Register parameters from each layer\n",
    "        self.params = {}\n",
    "        for i, layer in enumerate(self.layers):\n",
    "            self.params[i] = layer.register_param()\n",
    "        self.optim.set_param(self.params)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        for layer in self.layers:\n",
    "            x = layer.forward(x)\n",
    "        return x\n",
    "    \n",
    "    def backward(self, x, t):\n",
    "        self.grads = {}\n",
    "        \n",
    "        output_der = self.loss.backward(x, t)\n",
    "        for i, layer in reversed(list(enumerate(self.layers))):\n",
    "            der_param = layer.backward(output_der)\n",
    "            self.grads[i] = der_param\n",
    "            # Calculate output derivative for previous layer.\n",
    "            output_der = layer.input_der(output_der)\n",
    "        return\n",
    "    \n",
    "    def update(self):\n",
    "        # Gradient clipping\n",
    "        for key in self.grads:\n",
    "            n = len(self.grads[key])\n",
    "            for i in range(n):\n",
    "                self.grads[key][i] =  gradient_clipping(self.grads[key][i], self.t)\n",
    "        self.optim(self.params, self.grads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "At     5 epoch,\tvalidation accuracy: 76.67\tLoss: 246.1727970135372\n",
      "At    10 epoch,\tvalidation accuracy: 100.00\tLoss: 27.233251985620285\n",
      "At    15 epoch,\tvalidation accuracy: 100.00\tLoss: 29.36586529646044\n",
      "At    20 epoch,\tvalidation accuracy: 100.00\tLoss: 27.27442856229027\n",
      "At    25 epoch,\tvalidation accuracy: 100.00\tLoss: 27.13470307505745\n",
      "At    30 epoch,\tvalidation accuracy: 100.00\tLoss: 14.491447500240628\n",
      "At    35 epoch,\tvalidation accuracy: 100.00\tLoss: 12.543871939348575\n",
      "At    40 epoch,\tvalidation accuracy: 100.00\tLoss: 14.727432621161494\n",
      "At    45 epoch,\tvalidation accuracy: 100.00\tLoss: 15.337508327065628\n",
      "At    50 epoch,\tvalidation accuracy: 100.00\tLoss: 7.441114048989004\n"
     ]
    }
   ],
   "source": [
    "# Solution to exercise 2\n",
    "\n",
    "adam = Adam(0.05, 0.9, 0.999, 1e-8)\n",
    "loss = CrossEntropyLoss\n",
    "\n",
    "# Initialize the network\n",
    "grad_clip_network = GradClipNetwork(loss, adam, 1)\n",
    "\n",
    "epoch = 50\n",
    "num_e = 5\n",
    "\n",
    "\n",
    "valid_accu = 0\n",
    "training_loss = 0\n",
    "\n",
    "for e in range(epoch):     \n",
    "    # Test with validation data\n",
    "    correct_count = 0\n",
    "    total_count = 0\n",
    "    for inputs, targets in zip(valid['input'], valid['target']):\n",
    "        outputs = grad_clip_network.forward(inputs)\n",
    "\n",
    "        total_count += 1\n",
    "        if np.argmax(outputs) == np.argmax(targets):\n",
    "            correct_count += 1\n",
    "\n",
    "    accu = correct_count / total_count\n",
    "    \n",
    "    # Save model if current best\n",
    "    if accu >= valid_accu:\n",
    "        valid_accu = accu\n",
    "        best_params = grad_clip_network.params\n",
    "    \n",
    "    # Train network with training data\n",
    "    for inputs, targets in zip(training['input'], training['target']):\n",
    "        outputs = grad_clip_network.forward(inputs)\n",
    "        grad_clip_network.backward(outputs, targets)\n",
    "        grad_clip_network.update()\n",
    "        \n",
    "        training_loss += loss.forward(outputs, targets) / num_e\n",
    "\n",
    "    # Print training info every set amount of epoch\n",
    "    if not (e + 1) % num_e:\n",
    "        print(f\"At {e + 1: 5} epoch,\\tvalidation accuracy: {valid_accu * 100:.2f}\\tLoss: {training_loss}\")\n",
    "        training_loss = 0\n",
    "        \n",
    "# Load network with best parameters\n",
    "grad_clip_network.params = best_params"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}