W4_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyN5WuY6qtvIbcY8GC5WFxN5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/liudiepie/Introduction_to_Machine_Learning/blob/main/NLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xG8Drw-iKkPq"
      },
      "source": [
        "# Introduction to Natural Language Processing (NLP) in PyTorch"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LKvBttw4KyUS"
      },
      "source": [
        "### Word Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "waKfzCRUXkEp",
        "outputId": "0091fd0c-cd95-4b65-e3f0-3ddd9ca43ad7"
      },
      "source": [
        "# load h5py into colab\n",
        "!pip install h5py"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (3.1.0)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py) (1.5.2)\n",
            "Requirement already satisfied: numpy>=1.14.5 in /usr/local/lib/python3.7/dist-packages (from h5py) (1.19.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j60tyWzgK04G",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aa3d7ba8-42ed-49db-97bc-3c90fb3d068c"
      },
      "source": [
        "# Download word vectors\n",
        "from urllib.request import urlretrieve\n",
        "import os\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TlNSXduFik29",
        "outputId": "1f8195cd-fa32-49d8-97bf-e54216b210ab"
      },
      "source": [
        "# Load the file and pull out words and embeddings\n",
        "# I save the mini.h5 in my drive. Download the mini.h5 and change the path based on your file\n",
        "import h5py\n",
        "\n",
        "with h5py.File('/content/drive/MyDrive/mini.h5', 'r') as f:\n",
        "    all_words = [word.decode('utf-8') for word in f['mat']['axis1'][:]]\n",
        "    all_embeddings = f['mat']['block0_values'][:]\n",
        "    \n",
        "print(\"all_words dimensions: {}\".format(len(all_words)))\n",
        "print(\"all_embeddings dimensions: {}\".format(all_embeddings.shape))\n",
        "\n",
        "print(\"Random example word: {}\".format(all_words[1337]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "all_words dimensions: 362891\n",
            "all_embeddings dimensions: (362891, 300)\n",
            "Random example word: /c/de/aufmachung\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NmzAdunbmQ6b",
        "outputId": "0944eedb-1bd2-485d-fa8f-acb930456441"
      },
      "source": [
        "# Restrict our vocabulary to just the English words\n",
        "english_words = [word[6:] for word in all_words if word.startswith('/c/en/')]\n",
        "english_word_indices = [i for i, word in enumerate(all_words) if word.startswith('/c/en/')]\n",
        "english_embeddings = all_embeddings[english_word_indices]\n",
        "\n",
        "print(\"Number of English words in all_words: {0}\".format(len(english_words)))\n",
        "print(\"english_embeddings dimensions: {0}\".format(english_embeddings.shape))\n",
        "print(english_words[1337])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of English words in all_words: 150875\n",
            "english_embeddings dimensions: (150875, 300)\n",
            "activated_carbon\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U1-bZW6unRu6"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "norms = np.linalg.norm(english_embeddings, axis=1)\n",
        "normalized_embeddings = english_embeddings.astype('float32') / norms.astype('float32').reshape([-1, 1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5cqLTH-n13S"
      },
      "source": [
        "index = {word: i for i, word in enumerate(english_words)}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Lzj_cEFnqMy",
        "outputId": "bd5c4319-7097-4aa7-f8bc-ebcad07eb983"
      },
      "source": [
        "def similarity_score(w1, w2):\n",
        "    score = np.dot(normalized_embeddings[index[w1], :], normalized_embeddings[index[w2], :])\n",
        "    return score\n",
        "\n",
        "# A word is as similar with itself as possible:\n",
        "print('cat\\tcat\\t', similarity_score('cat', 'cat'))\n",
        "\n",
        "# Closely related words still get high scores:\n",
        "print('cat\\tfeline\\t', similarity_score('cat', 'feline'))\n",
        "print('cat\\tdog\\t', similarity_score('cat', 'dog'))\n",
        "\n",
        "# Unrelated words, not so much\n",
        "print('cat\\tmoo\\t', similarity_score('cat', 'moo'))\n",
        "print('cat\\tfreeze\\t', similarity_score('cat', 'freeze'))\n",
        "\n",
        "# Antonyms are still considered related, sometimes more so than synonyms\n",
        "print('antonym\\topposite\\t', similarity_score('antonym', 'opposite'))\n",
        "print('antonym\\tsynonym\\t', similarity_score('antonym', 'synonym'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cat\tcat\t 1.0000001\n",
            "cat\tfeline\t 0.8199548\n",
            "cat\tdog\t 0.590724\n",
            "cat\tmoo\t 0.0039538303\n",
            "cat\tfreeze\t -0.030225191\n",
            "antonym\topposite\t 0.3941065\n",
            "antonym\tsynonym\t 0.46883982\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3U1DuxZsoDSf"
      },
      "source": [
        "def closest_to_vector(v, n):\n",
        "    all_scores = np.dot(normalized_embeddings, v)\n",
        "    best_words = list(map(lambda i: english_words[i], reversed(np.argsort(all_scores))))\n",
        "    return best_words[:n]\n",
        "\n",
        "def most_similar(w, n):\n",
        "    return closest_to_vector(normalized_embeddings[index[w], :], n)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hDOSGt4KoGis",
        "outputId": "65964303-d7bf-42ed-87ca-b86b8ac63216"
      },
      "source": [
        "print(most_similar('cat', 10))\n",
        "print(most_similar('dog', 10))\n",
        "print(most_similar('duke', 10))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['cat', 'humane_society', 'kitten', 'feline', 'colocolo', 'cats', 'kitty', 'maine_coon', 'housecat', 'sharp_teeth']\n",
            "['dog', 'dogs', 'wire_haired_dachshund', 'doggy_paddle', 'lhasa_apso', 'good_friend', 'puppy_dog', 'bichon_frise', 'woof_woof', 'golden_retrievers']\n",
            "['duke', 'dukes', 'duchess', 'duchesses', 'ducal', 'dukedom', 'duchy', 'voivode', 'princes', 'prince']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1qMppBzUoKJ2",
        "outputId": "af86a213-7b1f-4adb-ee2e-262049ee2cca"
      },
      "source": [
        "def solve_analogy(a1, b1, a2):\n",
        "    b2 = normalized_embeddings[index[b1], :] - normalized_embeddings[index[a1], :] + normalized_embeddings[index[a2], :]\n",
        "    return closest_to_vector(b2, 1)\n",
        "\n",
        "print(solve_analogy(\"man\", \"brother\", \"woman\"))\n",
        "print(solve_analogy(\"man\", \"husband\", \"woman\"))\n",
        "print(solve_analogy(\"spain\", \"madrid\", \"france\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['sister']\n",
            "['wife']\n",
            "['paris']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FNKAPH0moUnE"
      },
      "source": [
        "### Using word embeddings in deep models\n",
        "If you don't have it already, download the movie-simple.txt file. Each line of that file contains\n",
        "\n",
        "1. the numeral 0 (for negative) or the numeral 1 (for positive), followed by\n",
        "2. a tab (the whitespace character), and then\n",
        "3. the review itself."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-klodzr3par2"
      },
      "source": [
        "import string\n",
        "remove_punct=str.maketrans('','',string.punctuation)\n",
        "\n",
        "# This function converts a line of our data file into\n",
        "# a tuple (x, y), where x is 300-dimensional representation\n",
        "# of the words in a review, and y is its label.\n",
        "def convert_line_to_example(line):\n",
        "    # Pull out the first character: that's our label (0 or 1)\n",
        "    y = int(line[0])\n",
        "    \n",
        "    # Split the line into words using Python's split() function\n",
        "    words = line[2:].translate(remove_punct).lower().split()\n",
        "    \n",
        "    # Look up the embeddings of each word, ignoring words not\n",
        "    # in our pretrained vocabulary.\n",
        "    embeddings = [normalized_embeddings[index[w]] for w in words\n",
        "                  if w in index]\n",
        "    \n",
        "    # Take the mean of the embeddings\n",
        "    x = np.mean(np.vstack(embeddings), axis=0)\n",
        "    return x, y\n",
        "\n",
        "# Apply the function to each line in the file.\n",
        "xs = []\n",
        "ys = []\n",
        "with open(\"/content/drive/MyDrive/movie-simple.txt\", \"r\", encoding='utf-8', errors='ignore') as f:\n",
        "    for l in f.readlines():\n",
        "        x, y = convert_line_to_example(l)\n",
        "        xs.append(x)\n",
        "        ys.append(y)\n",
        "\n",
        "# Concatenate all examples into a numpy array\n",
        "xs = np.vstack(xs)\n",
        "ys = np.vstack(ys)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pzbPPNNIpblb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e7cd11ce-8fcc-41bc-dfe0-e17030474df2"
      },
      "source": [
        "print(\"Shape of inputs: {}\".format(xs.shape))\n",
        "print(\"Shape of labels: {}\".format(ys.shape))\n",
        "\n",
        "num_examples = xs.shape[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of inputs: (1411, 300)\n",
            "Shape of labels: (1411, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_AWBSXNVrEKu",
        "outputId": "f2886614-f1d7-4ebd-8793-ebb03aead3d1"
      },
      "source": [
        "print(\"First 20 labels before shuffling: {0}\".format(ys[:20, 0]))\n",
        "\n",
        "shuffle_idx = np.random.permutation(num_examples)\n",
        "xs = xs[shuffle_idx, :]\n",
        "ys = ys[shuffle_idx, :]\n",
        "\n",
        "print(\"First 20 labels after shuffling: {0}\".format(ys[:20, 0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "First 20 labels before shuffling: [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
            "First 20 labels after shuffling: [1 1 1 0 1 1 0 0 0 1 0 0 0 1 0 1 1 1 1 1]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Ygp986erkqs"
      },
      "source": [
        "import torch\n",
        "\n",
        "num_train = 4*num_examples // 5\n",
        "\n",
        "x_train = torch.tensor(xs[:num_train])\n",
        "y_train = torch.tensor(ys[:num_train], dtype=torch.float32)\n",
        "\n",
        "x_test = torch.tensor(xs[num_train:])\n",
        "y_test = torch.tensor(ys[num_train:], dtype=torch.float32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-qLJYy2rmqc"
      },
      "source": [
        "reviews_train = torch.utils.data.TensorDataset(x_train, y_train)\n",
        "reviews_test = torch.utils.data.TensorDataset(x_test, y_test)\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(reviews_train, batch_size=100, shuffle=True)\n",
        "test_loader = torch.utils.data.DataLoader(reviews_test, batch_size=100, shuffle=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JfY7pbg3roZB"
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g2ks-OsIrq_T"
      },
      "source": [
        "class SWEM(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.fc1 = nn.Linear(300, 64)\n",
        "        self.fc2 = nn.Linear(64, 1)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.fc1(x)\n",
        "        x = F.relu(x)\n",
        "        x = self.fc2(x)\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-EgvshRSrufZ",
        "outputId": "afa19305-3378-45fc-dd67-cf940ecb7afb"
      },
      "source": [
        "## Training\n",
        "# Instantiate model\n",
        "model = SWEM()\n",
        "\n",
        "# Binary cross-entropy (BCE) Loss and Adam Optimizer\n",
        "criterion = nn.BCEWithLogitsLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
        "\n",
        "# Iterate through train set minibatchs \n",
        "for epoch in range(250):\n",
        "    correct = 0\n",
        "    num_examples = 0\n",
        "    for inputs, labels in train_loader:\n",
        "        # Zero out the gradients\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        loss = criterion(y, labels)\n",
        "        \n",
        "        # Backward pass\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        predictions = torch.round(torch.sigmoid(y))\n",
        "        correct += torch.sum((predictions == labels).float())\n",
        "        num_examples += len(inputs)\n",
        "    \n",
        "    # Print training progress\n",
        "    if epoch % 25 == 0:\n",
        "        acc = correct/num_examples\n",
        "        print(\"Epoch: {0} \\t Train Loss: {1} \\t Train Acc: {2}\".format(epoch, loss, acc))\n",
        "\n",
        "## Testing\n",
        "correct = 0\n",
        "num_test = 0\n",
        "\n",
        "with torch.no_grad():\n",
        "    # Iterate through test set minibatchs \n",
        "    for inputs, labels in test_loader:\n",
        "        # Forward pass\n",
        "        y = model(inputs)\n",
        "        \n",
        "        predictions = torch.round(torch.sigmoid(y))\n",
        "        correct += torch.sum((predictions == labels).float())\n",
        "        num_test += len(inputs)\n",
        "    \n",
        "print('Test accuracy: {}'.format(correct/num_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0 \t Train Loss: 0.6903021931648254 \t Train Acc: 0.5407801270484924\n",
            "Epoch: 25 \t Train Loss: 0.2671360373497009 \t Train Acc: 0.9503546357154846\n",
            "Epoch: 50 \t Train Loss: 0.12922227382659912 \t Train Acc: 0.9671986103057861\n",
            "Epoch: 75 \t Train Loss: 0.05315467342734337 \t Train Acc: 0.9778369069099426\n",
            "Epoch: 100 \t Train Loss: 0.10583224147558212 \t Train Acc: 0.9769503474235535\n",
            "Epoch: 125 \t Train Loss: 0.019270751625299454 \t Train Acc: 0.9822695255279541\n",
            "Epoch: 150 \t Train Loss: 0.03239167854189873 \t Train Acc: 0.9867021441459656\n",
            "Epoch: 175 \t Train Loss: 0.041654136031866074 \t Train Acc: 0.9902482032775879\n",
            "Epoch: 200 \t Train Loss: 0.009401567280292511 \t Train Acc: 0.991134762763977\n",
            "Epoch: 225 \t Train Loss: 0.014387448318302631 \t Train Acc: 0.9920212626457214\n",
            "Test accuracy: 0.9540635943412781\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H0Y7_Z19r4IV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7dcde594-d1c7-406c-975b-37fec30d5a3e"
      },
      "source": [
        "# Check some words\n",
        "words_to_test = [\"exciting\", \"hated\", \"boring\", \"loved\"]\n",
        "\n",
        "for word in words_to_test:\n",
        "    x = torch.tensor(normalized_embeddings[index[word]].reshape(1, 300))\n",
        "    print(\"Sentiment of the word '{0}': {1}\".format(word, torch.sigmoid(model(x))))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentiment of the word 'exciting': tensor([[1.]], grad_fn=<SigmoidBackward>)\n",
            "Sentiment of the word 'hated': tensor([[6.0613e-19]], grad_fn=<SigmoidBackward>)\n",
            "Sentiment of the word 'boring': tensor([[5.5512e-15]], grad_fn=<SigmoidBackward>)\n",
            "Sentiment of the word 'loved': tensor([[1.]], grad_fn=<SigmoidBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}