W4_Assignment_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment_NLP.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMSDch5DDCEA6wynizg11eg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/liudiepie/Introduction_to_Machine_Learning/blob/main/Assignment_NLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IMXZBMlKx8MZ"
      },
      "source": [
        "# Natural Language Processing(NLP)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jMrfMTIyyFDe"
      },
      "source": [
        "### Loading AG News with Torchtext"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "olwiLNVPIUW-",
        "outputId": "61f6866a-08fb-4e10-fb0d-819218769e71"
      },
      "source": [
        "!pip install torchtext==0.8.1"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting torchtext==0.8.1\n",
            "  Downloading torchtext-0.8.1-cp37-cp37m-manylinux1_x86_64.whl (7.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.0 MB 6.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchtext==0.8.1) (2.23.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from torchtext==0.8.1) (4.41.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchtext==0.8.1) (1.19.5)\n",
            "Collecting torch==1.7.1\n",
            "  Downloading torch-1.7.1-cp37-cp37m-manylinux1_x86_64.whl (776.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 776.8 MB 17 kB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.7.1->torchtext==0.8.1) (3.7.4.3)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.8.1) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.8.1) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.8.1) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.8.1) (2021.5.30)\n",
            "Installing collected packages: torch, torchtext\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.9.0+cu102\n",
            "    Uninstalling torch-1.9.0+cu102:\n",
            "      Successfully uninstalled torch-1.9.0+cu102\n",
            "  Attempting uninstall: torchtext\n",
            "    Found existing installation: torchtext 0.10.0\n",
            "    Uninstalling torchtext-0.10.0:\n",
            "      Successfully uninstalled torchtext-0.10.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchvision 0.10.0+cu102 requires torch==1.9.0, but you have torch 1.7.1 which is incompatible.\u001b[0m\n",
            "Successfully installed torch-1.7.1 torchtext-0.8.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ik6TOnDyCNJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b7343518-837e-45dc-bd13-ae1d8a314191"
      },
      "source": [
        "import torchtext\n",
        "agnews_train, agnews_test = torchtext.datasets.text_classification.DATASETS[\"AG_NEWS\"](root=\"./datasets\")"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ag_news_csv.tar.gz: 11.8MB [00:00, 37.8MB/s]\n",
            "120000lines [00:04, 29884.52lines/s]\n",
            "120000lines [00:08, 13850.61lines/s]\n",
            "7600lines [00:00, 13136.18lines/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 120
        },
        "id": "_vDiAKgRzZTM",
        "outputId": "abfdfb85-819f-4fc5-f2d5-c638c2773ba9"
      },
      "source": [
        "#this code for loading datasets from own computer\n",
        "\"\"\"\n",
        "import torchtext\n",
        "\n",
        "ngrams = 1\n",
        "train_csv_path = './datasets/ag_news_csv/train.csv'\n",
        "test_csv_path = './datasets/ag_news_csv/test.csv'\n",
        "vocab = torchtext.vocab.build_vocab_from_iterator(\n",
        "    torchtext.legacy.datasets.text_classification._csv_iterator(train_csv_path, ngrams))\n",
        "train_data, train_labels = torchtext.legacy.datasets.text_classification._create_data_from_iterator(\n",
        "        vocab, torchtext.legacy.datasets.text_classification._csv_iterator(train_csv_path, ngrams, yield_cls=True), False)\n",
        "test_data, test_labels = torchtext.legacy.datasets.text_classification._create_data_from_iterator(\n",
        "        vocab, torchtext.legacy.datasets.text_classification._csv_iterator(test_csv_path, ngrams, yield_cls=True), False)\n",
        "if len(train_labels ^ test_labels) > 0:\n",
        "    raise ValueError(\"Training and test labels don't match\")\n",
        "agnews_train = torchtext.legacy.datasets.TextClassificationDataset(vocab, train_data, train_labels)\n",
        "agnews_test = torchtext.legacy.datasets.TextClassificationDataset(vocab, test_data, test_labels)\n",
        "\"\"\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\nimport torchtext\\n\\nngrams = 1\\ntrain_csv_path = \\'./datasets/ag_news_csv/train.csv\\'\\ntest_csv_path = \\'./datasets/ag_news_csv/test.csv\\'\\nvocab = torchtext.vocab.build_vocab_from_iterator(\\n    torchtext.legacy.datasets.text_classification._csv_iterator(train_csv_path, ngrams))\\ntrain_data, train_labels = torchtext.legacy.datasets.text_classification._create_data_from_iterator(\\n        vocab, torchtext.legacy.datasets.text_classification._csv_iterator(train_csv_path, ngrams, yield_cls=True), False)\\ntest_data, test_labels = torchtext.legacy.datasets.text_classification._create_data_from_iterator(\\n        vocab, torchtext.legacy.datasets.text_classification._csv_iterator(test_csv_path, ngrams, yield_cls=True), False)\\nif len(train_labels ^ test_labels) > 0:\\n    raise ValueError(\"Training and test labels don\\'t match\")\\nagnews_train = torchtext.legacy.datasets.TextClassificationDataset(vocab, train_data, train_labels)\\nagnews_test = torchtext.legacy.datasets.TextClassificationDataset(vocab, test_data, test_labels)\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ryogkoNQLjWM",
        "outputId": "0a8f6092-253a-442c-b55f-6a864d8f8fc2"
      },
      "source": [
        "print(agnews_train[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(2, tensor([  432,   426,     2,  1606, 14839,   114,    67,     3,   849,    14,\n",
            "           28,    15,    28,    16, 50726,     4,   432,   375,    17,    10,\n",
            "        67508,     7, 52259,     4,    43,  4010,   784,   326,     2]))\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tJMV5QcrLl9P",
        "outputId": "ceac0a85-e1ae-4089-ef58-a15fd134d88d"
      },
      "source": [
        "print(\"Length of the first text example: {}\".format(len(agnews_train[0][1])))\n",
        "print(\"Length of the second text example: {}\".format(len(agnews_train[1][1])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Length of the first text example: 29\n",
            "Length of the second text example: 42\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SNvKg71DLpK8",
        "outputId": "b666ea9a-507d-4bb8-c150-e5aa46bc9e95"
      },
      "source": [
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "padded_exs = pad_sequence([agnews_train[0][1], agnews_train[1][1]])\n",
        "print(\"First sequence padded: {}\".format(padded_exs[:,0]))\n",
        "print(\"First sequence length: {}\".format(len(padded_exs[:,0])))\n",
        "print(\"Second sequence padded: {}\".format(padded_exs[:,1]))\n",
        "print(\"Second sequence length: {}\".format(len(padded_exs[:,1])))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "First sequence padded: tensor([  432,   426,     2,  1606, 14839,   114,    67,     3,   849,    14,\n",
            "           28,    15,    28,    16, 50726,     4,   432,   375,    17,    10,\n",
            "        67508,     7, 52259,     4,    43,  4010,   784,   326,     2,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0])\n",
            "First sequence length: 42\n",
            "Second sequence padded: tensor([15875,  1073,   855,  1311,  4251,    14,    28,    15,    28,    16,\n",
            "          930,   798,   321, 15875,    99,     4, 27658,    29,     6,  4460,\n",
            "           12,   565, 52791,     9, 80618,  2126,     8,     3,   526,   242,\n",
            "            4,    29,  3891, 82815,  6575,    11,   207,   360,     7,     3,\n",
            "          127,     2])\n",
            "Second sequence length: 42\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m6P9pTfsLu4b"
      },
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "\n",
        "def collator(batch):\n",
        "    labels = torch.tensor([example[0] for example in batch])\n",
        "    sentences = [example[1] for example in batch]\n",
        "    data = pad_sequence(sentences)\n",
        "    \n",
        "    return [data, labels]"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T2xixUB5Lwzg"
      },
      "source": [
        "BATCH_SIZE = 128\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(agnews_train, batch_size=BATCH_SIZE, shuffle=True, collate_fn=collator)\n",
        "test_loader = torch.utils.data.DataLoader(agnews_test, batch_size=BATCH_SIZE, shuffle=False, collate_fn=collator)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h1_Bt7trL6ZY"
      },
      "source": [
        "### Simple Word Embedding Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wXm6ZYVJL-MQ"
      },
      "source": [
        "VOCAB_SIZE = len(agnews_train.get_vocab())\n",
        "EMBED_DIM = 100\n",
        "HIDDEN_DIM = 64\n",
        "NUM_OUTPUTS = len(agnews_train.get_labels())\n",
        "NUM_EPOCHS = 3"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gm4jgRyeMAp6"
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "class SWEM(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_size, hidden_dim, num_outputs):\n",
        "        super().__init__()\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        \n",
        "        self.fc1 = nn.Linear(embedding_size, hidden_dim)\n",
        "        self.fc2 = nn.Linear(hidden_dim, num_outputs)\n",
        "\n",
        "    def forward(self, x):\n",
        "        embed = self.embedding(x)\n",
        "        embed_mean = torch.mean(embed, dim=0)\n",
        "        \n",
        "        h = self.fc1(embed_mean)\n",
        "        h = F.relu(h)\n",
        "        h = self.fc2(h)\n",
        "        return h"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 490
        },
        "id": "LQDExPrKNdzM",
        "outputId": "67db97ed-29b3-41e8-b73c-26900d8015e3"
      },
      "source": [
        "## Training\n",
        "# Instantiate model\n",
        "model = SWEM(VOCAB_SIZE, EMBED_DIM, HIDDEN_DIM, NUM_OUTPUTS)\n",
        "\n",
        "# Binary Cross Entropy Loss and Adam Optimizer\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
        "\n",
        "# Iterate through train set minibatchs \n",
        "for epoch in range(250):\n",
        "    correct = 0\n",
        "    num_examples = 0\n",
        "    for inputs, labels in train_loader:\n",
        "        # Zero out the gradients\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        loss = criterion(y, labels)\n",
        "        \n",
        "        # Backward pass\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_examples += labels.size(0)\n",
        "    \n",
        "    # Print training progress\n",
        "    if epoch % 10 == 0:\n",
        "        acc = correct/num_examples\n",
        "        print(\"Epoch: {0} \\t Train Loss: {1} \\t Train Acc: {2}\".format(epoch, loss, acc))\n",
        "\n",
        "## Testing\n",
        "correct = 0\n",
        "num_test = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    # Iterate through test set minibatchs \n",
        "    for inputs, labels in test_loader:\n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_examples += labels.size(0)\n",
        "    \n",
        "print('Test accuracy: {}'.format(correct/num_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0 \t Train Loss: 0.4672899544239044 \t Train Acc: 0.7307166666666667\n",
            "Epoch: 10 \t Train Loss: 0.025780154392123222 \t Train Acc: 0.9743666666666667\n",
            "Epoch: 20 \t Train Loss: 0.0004224234726279974 \t Train Acc: 0.99125\n",
            "Epoch: 30 \t Train Loss: 0.031344421207904816 \t Train Acc: 0.9951416666666667\n",
            "Epoch: 40 \t Train Loss: 6.0853806644445285e-05 \t Train Acc: 0.9965\n",
            "Epoch: 50 \t Train Loss: 0.0005616910057142377 \t Train Acc: 0.9977083333333333\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-a11421cceb3c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;31m# Backward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    117\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'weight_decay'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m                    )\n\u001b[1;32m    121\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/functional.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[1;32m     92\u001b[0m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UXBBcyfrZD24"
      },
      "source": [
        "### RNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WBOPJF_9ZGmp"
      },
      "source": [
        "class RNN(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_size, hidden_dim, num_outputs):\n",
        "        super().__init__()\n",
        "        self.vocab_size=vocab_size\n",
        "        self.embedding_size=embedding_size\n",
        "        self.hidden_dim=hidden_dim\n",
        "        self.num_outputs=num_outputs\n",
        "        \n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        self.rnn=nn.RNN(embedding_size,hidden_dim,num_layers=1,batch_first=True)\n",
        "        self.lin = nn.Linear(hidden_dim, num_outputs)\n",
        "\n",
        "    def forward(self, x):\n",
        "        embed = self.embedding(x)\n",
        "        h0 = torch.zeros(1, embed.size(0), self.hidden_dim)\n",
        "        out, _ = self.rnn(embed,h0)\n",
        "        h = self.lin(out[-1,:])\n",
        "        return h"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422
        },
        "id": "D-orgZriZJ7_",
        "outputId": "38d0874c-904e-4e01-a295-2b40fb348dd0"
      },
      "source": [
        "## Training\n",
        "# Instantiate model\n",
        "model = RNN(VOCAB_SIZE, EMBED_DIM, HIDDEN_DIM, NUM_OUTPUTS)\n",
        "\n",
        "# Binary Cross Entropy Loss and Adam Optimizer\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
        "\n",
        "# Iterate through train set minibatchs \n",
        "for epoch in range(10):\n",
        "    correct = 0\n",
        "    num_examples = 0\n",
        "    for inputs, labels in train_loader:\n",
        "\n",
        "        # Zero out the gradients\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        loss = criterion(y, labels)\n",
        "        \n",
        "        # Backward pass\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_examples += labels.size(0)\n",
        "    \n",
        "    # Print training progress\n",
        "    if epoch :\n",
        "        acc = correct/num_examples\n",
        "        print(\"Epoch: {0} \\t Train Loss: {1} \\t Train Acc: {2}\".format(epoch, loss, acc))\n",
        "\n",
        "## Testing\n",
        "correct = 0\n",
        "num_test = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    # Iterate through test set minibatchs \n",
        "    for inputs, labels in test_loader:\n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_examples += labels.size(0)\n",
        "    \n",
        "print('Test accuracy: {}'.format(correct/num_test))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 \t Train Loss: 1.3854719400405884 \t Train Acc: 0.25193333333333334\n",
            "Epoch: 2 \t Train Loss: 1.3821303844451904 \t Train Acc: 0.2522083333333333\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-24-2f7c7d4833ad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;31m# Backward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    117\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'weight_decay'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m                    )\n\u001b[1;32m    121\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/functional.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[1;32m     92\u001b[0m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mmax_exp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m             \u001b[0mdenom\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mmath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias_correction2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mstep_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlr\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mbias_correction1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qOBNiwrIUUB5"
      },
      "source": [
        "### LSTM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K28pxpobUZok"
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "\n",
        "class LSTM(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_size, hidden_dim, num_outputs):\n",
        "        super().__init__()\n",
        "        self.vocab_size=vocab_size\n",
        "        self.embedding_size=embedding_size\n",
        "        self.hidden_dim=hidden_dim\n",
        "        self.num_outputs=num_outputs\n",
        "        \n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        self.lstm=nn.LSTM(embedding_size,hidden_dim,num_layers=1,batch_first=True)\n",
        "        self.lin = nn.Linear(hidden_dim, num_outputs)\n",
        "\n",
        "    def forward(self, x):\n",
        "        embed = self.embedding(x)\n",
        "        h0 = torch.zeros(1, embed.size(0), self.hidden_dim)\n",
        "        c0 = torch.zeros(1, embed.size(0), self.hidden_dim)\n",
        "        out, hidden = self.lstm(embed,(h0,c0))\n",
        "        h = self.lin(out[-1,:])\n",
        "        return h"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422
        },
        "id": "w7hKNxSsWUkV",
        "outputId": "ae1400f9-a851-4257-bf2c-58b4d42c0596"
      },
      "source": [
        "## Training\n",
        "# Instantiate model\n",
        "model = LSTM(VOCAB_SIZE, EMBED_DIM, HIDDEN_DIM, NUM_OUTPUTS)\n",
        "\n",
        "# Binary Cross Entropy Loss and Adam Optimizer\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
        "\n",
        "# Iterate through train set minibatchs \n",
        "for epoch in range(20):\n",
        "    correct = 0\n",
        "    num_examples = 0\n",
        "    for inputs, labels in train_loader:\n",
        "        inputs, labels = Variable(inputs), Variable(labels)\n",
        "        # Zero out the gradients\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        loss = criterion(y, labels)\n",
        "        \n",
        "        # Backward pass\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_examples += labels.size(0)\n",
        "    \n",
        "    # Print training progress\n",
        "    if epoch :\n",
        "        acc = correct/num_examples\n",
        "        print(\"Epoch: {0} \\t Train Loss: {1} \\t Train Acc: {2}\".format(epoch, loss, acc))\n",
        "\n",
        "## Testing\n",
        "correct = 0\n",
        "num_test = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    # Iterate through test set minibatchs \n",
        "    for inputs, labels in test_loader:\n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_test += labels.size(0)\n",
        "    \n",
        "print('Test accuracy: {}'.format(correct/num_test))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 \t Train Loss: 1.3859484195709229 \t Train Acc: 0.2506583333333333\n",
            "Epoch: 2 \t Train Loss: 1.3923836946487427 \t Train Acc: 0.252825\n",
            "Epoch: 3 \t Train Loss: 1.3664660453796387 \t Train Acc: 0.252\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-26-a7618011d81a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;31m# Forward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-25-ed92f588d093>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mh0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_dim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mc0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden_dim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m         \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membed\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mc0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    725\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    726\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 727\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    728\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    729\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    580\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m             result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,\n\u001b[0;32m--> 582\u001b[0;31m                               self.dropout, self.training, self.bidirectional, self.batch_first)\n\u001b[0m\u001b[1;32m    583\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    584\u001b[0m             result = _VF.lstm(input, batch_sizes, hx, self._flat_weights, self.bias,\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L8j9Lz44kTB4"
      },
      "source": [
        "### GRU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3EXsvY1lkScp"
      },
      "source": [
        "class GRU(nn.Module):\n",
        "    def __init__(self, vocab_size, embedding_size, hidden_dim, num_outputs):\n",
        "        super().__init__()\n",
        "        self.vocab_size=vocab_size\n",
        "        self.embedding_size=embedding_size\n",
        "        self.hidden_dim=hidden_dim\n",
        "        self.num_outputs=num_outputs\n",
        "        \n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        self.gru=nn.GRU(embedding_size,hidden_dim,num_layers=1,batch_first=True)\n",
        "        self.lin = nn.Linear(hidden_dim, num_outputs)\n",
        "\n",
        "    def forward(self, x):\n",
        "        embed = self.embedding(x)\n",
        "        h0 = torch.zeros(1, embed.size(0), self.hidden_dim)\n",
        "        out, _ = self.gru(embed,h0)\n",
        "        h = self.lin(out[-1,:])\n",
        "        return h"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 348
        },
        "id": "-tftvjBckc-9",
        "outputId": "6fdf0d12-aeaa-4a6e-8afa-dd2324c979b8"
      },
      "source": [
        "## Training\n",
        "# Instantiate model\n",
        "model = RNN(VOCAB_SIZE, EMBED_DIM, HIDDEN_DIM, NUM_OUTPUTS)\n",
        "\n",
        "# Binary Cross Entropy Loss and Adam Optimizer\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
        "\n",
        "# Iterate through train set minibatchs \n",
        "for epoch in range(10):\n",
        "    correct = 0\n",
        "    num_examples = 0\n",
        "    for inputs, labels in train_loader:\n",
        "\n",
        "        # Zero out the gradients\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        loss = criterion(y, labels)\n",
        "        \n",
        "        # Backward pass\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_examples += labels.size(0)\n",
        "    \n",
        "    # Print training progress\n",
        "    if epoch :\n",
        "        acc = correct/num_examples\n",
        "        print(\"Epoch: {0} \\t Train Loss: {1} \\t Train Acc: {2}\".format(epoch, loss, acc))\n",
        "\n",
        "## Testing\n",
        "correct = 0\n",
        "num_test = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    # Iterate through test set minibatchs \n",
        "    for inputs, labels in test_loader:\n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        \n",
        "        predictions = y.argmax(dim=1)\n",
        "        correct += (predictions == labels).sum().item()\n",
        "        num_test += labels.size(0)\n",
        "    \n",
        "print('Test accuracy: {}'.format(correct/num_test))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 \t Train Loss: 1.387436032295227 \t Train Acc: 0.251475\n",
            "Epoch: 2 \t Train Loss: 1.393903136253357 \t Train Acc: 0.2521083333333333\n",
            "Epoch: 3 \t Train Loss: 1.3921102285385132 \t Train Acc: 0.2528\n",
            "Epoch: 4 \t Train Loss: 1.3808529376983643 \t Train Acc: 0.25226666666666664\n",
            "Epoch: 5 \t Train Loss: 1.37789785861969 \t Train Acc: 0.2522333333333333\n",
            "Epoch: 6 \t Train Loss: 1.382761001586914 \t Train Acc: 0.25003333333333333\n",
            "Epoch: 7 \t Train Loss: 1.3957297801971436 \t Train Acc: 0.251475\n",
            "Epoch: 8 \t Train Loss: 1.3731886148452759 \t Train Acc: 0.2518666666666667\n",
            "Epoch: 9 \t Train Loss: 1.3426324129104614 \t Train Acc: 0.2530833333333333\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "ZeroDivisionError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-28-2f7c7d4833ad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0mnum_examples\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Test accuracy: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorrect\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mnum_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YLg7oOjXmcdw",
        "outputId": "0f578587-bc92-47b5-c476-7c70b1d8a2be"
      },
      "source": [
        "# Access all weights of a model\n",
        "params = []\n",
        "for param in model.parameters():\n",
        "    params.append(param.view(-1))\n",
        "params = torch.cat(params)\n",
        "print(params.shape)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([9592084])\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}