W7-3 Seasons wordvecs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "vulnerable-pursuit",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from numpy import dot\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "exceptional-assets",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def norm_vec(v):\n",
    "    return v / np.linalg.norm(v)\n",
    "np.set_printoptions(precision=3)\n",
    "\n",
    "def normalize_rows(x):\n",
    "    return x/np.linalg.norm(x, ord=2, axis=1, keepdims=True)\n",
    "\n",
    "def normalize_columns(x):\n",
    "    return x/np.linalg.norm(x, ord=2, axis=0, keepdims=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "heavy-center",
   "metadata": {},
   "source": [
    "# Vectors for *words*\n",
    "\n",
    "We're going to prepare a version of the corpus here where we treat each student turn of talk (utterance) as a document.\n",
    "\n",
    "I'm not going to normalize the vectors this time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "entire-discharge",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utilities import *\n",
    "from seasons_module import load_seasons_corpus_as_utterances\n",
    "from seasons_module import load_seasons_comparison_files\n",
    "\n",
    "import numpy as np\n",
    "def norm_vec(vec):\n",
    "    mag = np.dot(vec, vec)\n",
    "    if mag == 0:\n",
    "        return vec\n",
    "    else:\n",
    "        return(vec / np.sqrt(mag))\n",
    "    \n",
    "def pure_tf(tf, df, cf, N):\n",
    "    return tf\n",
    "\n",
    "def tf(tf):\n",
    "    if tf == 0:\n",
    "        result = 0\n",
    "    else:\n",
    "        result = (1 + np.log(tf))\n",
    "    return result\n",
    "\n",
    "def weighted_word(the_text, word):\n",
    "    return tf(the_text.count(word))\n",
    "\n",
    "def compute_normed_doc_vector(word_list, vocab):\n",
    "    return norm_vec(np.array([weighted_word(word_list, word) for word in vocab]))\n",
    "\n",
    "def compute_doc_vector(word_list, vocab):\n",
    "    return np.array([weighted_word(word_list, word) for word in vocab])\n",
    "\n",
    "def prepare_seasons_corpus_by_utterance(vocab_size=50, norm_vecs=False):\n",
    "    seasons_corpus = load_seasons_corpus_as_utterances()\n",
    "    f = open(\"lists/seasons_stop_list.txt\")\n",
    "    stop_list = set(f.read().split(\"\\n\"))\n",
    "    word_fdist = nltk.FreqDist()\n",
    "    for fname in seasons_corpus.keys():\n",
    "        for utterance in seasons_corpus[fname][0]:\n",
    "            pruned_transcript_words = [w for w in utterance if w not in stop_list]\n",
    "            word_fdist.update(pruned_transcript_words)\n",
    "    new_vocab = [w[0] for w in word_fdist.most_common(vocab_size) if w not in stop_list]\n",
    "    \n",
    "    # compute the document vector for each document\n",
    "    doc_vectors = []\n",
    "    for fname in seasons_corpus.keys():\n",
    "        for utterance in seasons_corpus[fname][0]:\n",
    "            if norm_vecs:\n",
    "                doc_vectors.append(compute_normed_doc_vector(utterance, new_vocab))\n",
    "            else:\n",
    "                doc_vectors.append(compute_doc_vector(utterance, new_vocab))\n",
    "    return doc_vectors, new_vocab\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "specified-coordinator",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_vectors, vocab = prepare_seasons_corpus_by_utterance()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "declared-rough",
   "metadata": {},
   "outputs": [],
   "source": [
    "vp = doc_vectors[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "protective-product",
   "metadata": {},
   "source": [
    "Next we put all of these vectors into a big array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "upset-offering",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(doc_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unknown-stage",
   "metadata": {},
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "moral-minority",
   "metadata": {},
   "source": [
    "Now we have a matrix where each row corresponds to a one of the 1212 utterances and each column corresponds to one of the words"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "under-covering",
   "metadata": {},
   "source": [
    "One way to look at this is to make a big heatmap. We're just going to do it for the first 100 rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "following-reducing",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "fig=plt.figure(figsize=(10, 15), dpi= 80, facecolor='w', edgecolor='k')\n",
    "n = len(vocab)\n",
    "part_of_X = X[:100]\n",
    "x_tick_marks = np.arange(n)\n",
    "y_tick_marks = np.arange(part_of_X.shape[0])\n",
    "plt.xticks(x_tick_marks, vocab, fontsize=8, rotation=90)\n",
    "# plt.yticks(y_tick_marks, labels=None)\n",
    "plt.tick_params(\"x\", top=True, labeltop=True, bottom=False, labelbottom=False)\n",
    "plt.imshow(part_of_X, norm=matplotlib.colors.LogNorm(), interpolation='nearest', cmap='YlOrBr')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "later-leeds",
   "metadata": {},
   "source": [
    "## Get ready to have your mind blown\n",
    "\n",
    "This is the cool, tricky, surprising part. We can think of each column as telling us something about the meaning of the word. That means we have, for each word, a vector 1212 numbers long. And we can find the similarity in meaning of two words by finding the dot product of these vectors.\n",
    "\n",
    "The idea here is that \"we know a word by the company it keeps.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amazing-boston",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_column_vector(w, X):\n",
    "    cnumber = vocab.index(w)\n",
    "    return X[:, cnumber]\n",
    "\n",
    "def compare_words(w1, w2, X):\n",
    "    v1 = get_column_vector(w1, X)\n",
    "    v2 = get_column_vector(w2, X)\n",
    "    return np.dot(norm_vec(v1), norm_vec(v2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "demographic-investigator",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_column_vector(\"earth\", X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ancient-portland",
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_words(\"northern\", \"hemisphere\", X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "assigned-algorithm",
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_words(\"northern\", \"side\", X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sporting-karen",
   "metadata": {},
   "source": [
    "Using the power of math, it is easy to compare every word vector to every other word vector and to put it in a big array. This isn't particularly important. But it's nifty.\n",
    "\n",
    "First we need to normalize the columns. \n",
    "Then we transpose the array.\n",
    "Finally we multiply it times the original untransposed array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "social-healing",
   "metadata": {},
   "outputs": [],
   "source": [
    "Xnorm = normalize_columns(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "promising-spokesman",
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtnorm = np.transpose(Xnorm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "incomplete-stone",
   "metadata": {},
   "outputs": [],
   "source": [
    "mat = np.dot(Xtnorm, Xnorm)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceramic-islam",
   "metadata": {},
   "source": [
    "Now we can make a heatmap that displays this array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "continent-leave",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "fig=plt.figure(figsize=(10, 10), dpi= 80, facecolor='w', edgecolor='k')\n",
    "n = len(vocab)\n",
    "x_tick_marks = np.arange(n)\n",
    "y_tick_marks = np.arange(n)\n",
    "plt.xticks(x_tick_marks, vocab, fontsize=8, rotation=90)\n",
    "plt.yticks(y_tick_marks, vocab, fontsize=8)\n",
    "plt.tick_params(\"x\", top=True, labeltop=True, bottom=False, labelbottom=False)\n",
    "plt.imshow(mat, norm=matplotlib.colors.LogNorm(), interpolation='nearest', cmap='YlOrBr')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "passing-trail",
   "metadata": {},
   "source": [
    "### Wordvecs with a larger corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "moral-request",
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_tokenize(text):\n",
    "\n",
    "    # Separate most punctuation\n",
    "    text = re.sub(r\"(\\w)([\\.\\-\\/&\\\";:\\(\\)\\?\\!\\]\\[\\{\\}\\*])\", r'\\1 \\2 ', text)\n",
    "\n",
    "    # Separate commas if they're followed by space.\n",
    "    # (E.g., don't separate 2,500)\n",
    "    text = re.sub(r\"(,\\s)\", r' \\1', text)\n",
    "\n",
    "    # Separate leading and trailing single and double quotes .\n",
    "    text = re.sub(r\"('\\s)\", r' \\1', text)\n",
    "    text = re.sub(r\"(\\s\\')\", r'\\1 ', text)\n",
    "    text = re.sub(r\"(\\\"\\s)\", r' \\1', text)\n",
    "    text = re.sub(r\"(\\s\\\")\", r'\\1 ', text)\n",
    "\n",
    "    #Separate parentheses where appropriate\n",
    "    text = re.sub(r\"(\\)\\s)\", r' \\1', text)\n",
    "    text = re.sub(r\"(\\s\\()\", r'\\1 ', text)\n",
    "\n",
    "    # Separate periods that come before newline or end of string.\n",
    "    text = re.sub('\\. *(\\n|$)', ' . ', text)\n",
    "    return text.split()\n",
    "\n",
    "\n",
    "contraction_patterns = re.compile(r\"(?i)(.)('ll|'re|'ve|n't|'s|'m|'d)\\b\")\n",
    "def is_contraction(the_text):\n",
    "        return contraction_patterns.search(the_text)\n",
    "\n",
    "def alpha_only (ltext):\n",
    "    return [w.lower() for w in ltext if (len(w) > 0) and (w.isalpha() or w[0]=='<' or is_contraction(w))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "varying-lemon",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def load_training_corpus(fname):\n",
    "    f = open(fname)\n",
    "    raw = f.read().lower()\n",
    "    docs = re.findall(r\"<text>([^<]*)\", raw)\n",
    "    paras = []\n",
    "    for doc in docs:\n",
    "        paras += doc.split(\"\\n\\n\")\n",
    "    tdocs = [alpha_only(training_tokenize(para)) for para in paras]\n",
    "    return tdocs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "stupid-landing",
   "metadata": {},
   "outputs": [],
   "source": [
    "tdocs = load_training_corpus('corpora/seasons_training.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "quick-table",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tdocs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "altered-leader",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create the vocabulary\n",
    "import nltk\n",
    "all_words = []\n",
    "fdist = nltk.FreqDist()\n",
    "for tdoc in tdocs:\n",
    "    fdist.update(tdoc)\n",
    "f = open(\"lists/seasons_stop_list.txt\")\n",
    "stop_list = set(f.read().split(\"\\n\"))\n",
    "vocab = [w[0] for w in fdist.most_common(2000) if not (w[0] in stop_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "injured-sentence",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "doc_vectors = []\n",
    "for tdoc in tdocs:\n",
    "    dvec = [tdoc.count(w) for w in vocab]\n",
    "    doc_vectors.append(dvec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unexpected-exchange",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "rocky-trace",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_matrix = np.array(doc_vectors)\n",
    "dt_normed = normalize_columns(dt_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "known-triangle",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"winter\" in stop_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "minute-message",
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_words(\"northern\", \"hemisphere\", dt_normed)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}