W2_Assignment_Multi_Layer_Perceptrons_(MLP).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment_Multi-Layer Perceptrons (MLP).ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPklZ/YvQu7hg6u//xJG4wx",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/liudiepie/Introduction_to_Machine_Learning/blob/main/Assignment_Multi_Layer_Perceptrons_(MLP).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k14Qf4WPArwd"
      },
      "source": [
        "# Multi-Layer Perceptrons (MLP)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T9UW6Z8JAxj_",
        "outputId": "331610ca-6065-4076-9961-4a32fc8fe2c1"
      },
      "source": [
        "import torch\n",
        "\n",
        "x = torch.rand(5, 3)*2 - 1\n",
        "x_relu_max = torch.max(torch.zeros_like(x),x)\n",
        "\n",
        "print(\"x: {}\".format(x))\n",
        "print(\"x after ReLU with max: {}\".format(x_relu_max))"
      ],
      "execution_count": 121,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "x: tensor([[ 0.1779, -0.5660, -0.7355],\n",
            "        [-0.2317, -0.7305,  0.1393],\n",
            "        [-0.2435,  0.6126,  0.8400],\n",
            "        [-0.6209, -0.7020,  0.6116],\n",
            "        [ 0.6465,  0.1652, -0.5081]])\n",
            "x after ReLU with max: tensor([[0.1779, 0.0000, 0.0000],\n",
            "        [0.0000, 0.0000, 0.1393],\n",
            "        [0.0000, 0.6126, 0.8400],\n",
            "        [0.0000, 0.0000, 0.6116],\n",
            "        [0.6465, 0.1652, 0.0000]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PMeg39XZDf-s",
        "outputId": "8dcdc50a-3c31-42e6-dae5-88b74b7c6d9d"
      },
      "source": [
        "import torch.nn.functional as F\n",
        "\n",
        "x_relu_F = F.relu(x)\n",
        "\n",
        "print(\"x after ReLU with nn.funtional: {}\".format(x_relu_F))"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "x after ReLU with nn.funtional: tensor([[0.1779, 0.0000, 0.0000],\n",
            "        [0.0000, 0.0000, 0.1393],\n",
            "        [0.0000, 0.6126, 0.8400],\n",
            "        [0.0000, 0.0000, 0.6116],\n",
            "        [0.6465, 0.1652, 0.0000]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q09aP78IHzs6"
      },
      "source": [
        "import numpy as np\n",
        "from torch import nn\n",
        "import torch\n",
        "from torch.autograd import Variable\n",
        "import torch.nn.functional as F\n",
        "from torchvision import datasets, transforms\n",
        "from tqdm import tqdm\n",
        "from tqdm import tqdm_notebook\n"
      ],
      "execution_count": 123,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rb6iC6AVIGD9"
      },
      "source": [
        "# Load the data\n",
        "mnist_train = datasets.MNIST(root=\"./datasets\",train = True, transform = transforms.ToTensor(), download = True)\n",
        "mnist_test = datasets.MNIST(root=\"./datasets\",train = False, transform = transforms.ToTensor(), download = True)\n",
        "train_loader = torch.utils.data.DataLoader(mnist_train, batch_size=100, shuffle = True)\n",
        "test_loader = torch.utils.data.DataLoader(mnist_test, batch_size=100, shuffle = False)"
      ],
      "execution_count": 124,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qJiYnOCGNzPj"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nXWo_Qy0FDi9"
      },
      "source": [
        "# Define the MLP function\n",
        "class MLP(nn.Module):\n",
        "     def __init__(self):\n",
        "         super(MLP, self).__init__()\n",
        "         self.fc1 = nn.Linear(28*28, 500)\n",
        "         self.fc2 = nn.Linear(500, 10)\n",
        "         self.layers = nn.Sequential(nn.Linear(28*28, 500), nn.ReLU(), nn.Linear(500, 10))\n",
        "\n",
        "         \n",
        "     def forward(self, x):\n",
        "         x = F.relu(self.fc1(x))\n",
        "         x = self.fc2(x)\n",
        "         return x\n",
        " \n",
        "mlp = MLP()"
      ],
      "execution_count": 125,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0K4GNWnjOH80"
      },
      "source": [
        "# Optimizer\n",
        "optimizer = torch.optim.SGD(mlp.parameters(), lr=0.1)"
      ],
      "execution_count": 135,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4bK-umnNOPDN",
        "outputId": "59636e23-afe9-4319-b989-611414cca147"
      },
      "source": [
        "# Iterate through train set minibatchs\n",
        "for images, labels in tqdm(train_loader):\n",
        "  # Zero out the gradients\n",
        "  optimizer.zero_grad()\n",
        "  mlp.train()\n",
        "  images = Variable(images)\n",
        "  labels = Variable(labels)\n",
        "  # Forward pass\n",
        "  x = images.view(-1, 28*28)\n",
        "  out = mlp(x)\n",
        "  cross_entropy = F.cross_entropy(out, labels)\n",
        "  # Backward pass\n",
        "  cross_entropy.backward()\n",
        "  optimizer.step()"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 600/600 [00:11<00:00, 52.07it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gSPcAGXcsuIx"
      },
      "source": [
        "### Testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Arb3xX3Nsvsj",
        "outputId": "0f39b545-fab2-4fab-9b94-16e9d108d240"
      },
      "source": [
        "correct = 0\n",
        "total = len(mnist_test)\n",
        "\n",
        "# Iterate through test set minibatchs\n",
        "with torch.no_grad():\n",
        "  for images, labels in tqdm(test_loader):\n",
        "    # Zero out the gradients\n",
        "    optimizer.zero_grad()\n",
        "    mlp.train()\n",
        "    images = Variable(images)\n",
        "    labels = Variable(labels)\n",
        "    # Forward pass\n",
        "    x = images.view(-1, 28*28)\n",
        "    out = mlp(x)\n",
        "\n",
        "    predictions = torch.argmax(out, dim = 1)\n",
        "    correct += torch.sum((predictions == labels).float())\n",
        "\n",
        "print('Test accuracy: {}'.format(correct/total))"
      ],
      "execution_count": 137,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 100/100 [00:01<00:00, 62.20it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test accuracy: 0.9236999750137329\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}