W8-3 Reducing Dimensions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "excellent-beverage",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Preliminaries"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "preceding-disability",
   "metadata": {
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "All of the same preliminaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "false-lobby",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def norm_vec(v):\n",
    "    mag = np.linalg.norm(v)\n",
    "    if mag == 0:\n",
    "        return v\n",
    "    return v / np.linalg.norm(v)\n",
    "\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "def normalize_rows(x):\n",
    "    return normalize(x, axis=1)\n",
    "\n",
    "def normalize_columns(x):\n",
    "    return normalize(x, axis=0)\n",
    "\n",
    "def check_float(potential_float):\n",
    "    try:\n",
    "        float(potential_float)\n",
    "        return True\n",
    "    except ValueError:\n",
    "        return False\n",
    "\n",
    "def round_if_float(v, prec=3):\n",
    "    if check_float(v):\n",
    "        return round(float(v), prec)\n",
    "    return v\n",
    "\n",
    "from IPython.core.display import display, HTML\n",
    "def list_table(the_list, color_nums=False):\n",
    "    html = [\"<table style= 'border: 1px solid black; display:inline-block'>\"]\n",
    "    for row in the_list:\n",
    "        html.append(\"<tr>\")\n",
    "        for col in row:\n",
    "            if color_nums and check_float(col) and not float(col) == 0:\n",
    "                html.append(\"<td align='left' style='border: .5px solid gray; color: {1}; font-weight: bold'>{0}</td>\".format(round_if_float(col), color_nums))\n",
    "            else:\n",
    "                html.append(\"<td align='left' style='border: .5px solid gray;'>{0}</td>\".format(round_if_float(col)))\n",
    "        html.append(\"</tr>\")\n",
    "    html.append(\"</table>\")\n",
    "    return display(HTML(''.join(html)))\n",
    "\n",
    "def show_labeled_table(mat, col_names=None, row_names=None, nrows=10, ncols=10, color_nums=\"red\"):\n",
    "    sml = mat[:nrows, :ncols]\n",
    "    if col_names is not None:\n",
    "        sml = np.vstack([col_names[:ncols], sml])\n",
    "    if row_names is not None:\n",
    "        rnames = [[p] for p in row_names[:nrows]]\n",
    "        if col_names is not None:\n",
    "            new_col = np.array([[\"_\"]] + rnames)\n",
    "        else:\n",
    "            new_col = np.array(rnames)\n",
    "        sml = np.hstack((new_col, sml))\n",
    "    return list_table(sml, color_nums)\n",
    "\n",
    "def compute_doc_vector(tdoc, vocab):\n",
    "    return np.array([tdoc.count(w) for w in vocab])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "extraordinary-compatibility",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Word vectors for the seasons"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "small-folks",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Load the training corpus. \n",
    "\n",
    "This time we'll split it into sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lesbian-seating",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "fname = 'corpora/seasons_training.txt'\n",
    "f = open(fname)\n",
    "raw = f.read().lower()\n",
    "whole_training_docs = re.findall(r\"<text>([\\s\\S]*?)</text>\", raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lasting-checkout",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_docs = []\n",
    "para_names = []\n",
    "for i, d in enumerate(whole_training_docs):\n",
    "    new_docs = nltk.sent_tokenize(d)\n",
    "    training_docs += new_docs\n",
    "    new_names = [\"d{}p{}\".format(i, p) for p in range(len(new_docs))]\n",
    "    para_names += new_names"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "global-april",
   "metadata": {},
   "source": [
    "Now we have close to 14000 documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "reliable-provincial",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(training_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "deluxe-ridge",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Tokenize the training documents\n",
    "\n",
    "Now we tokenize the ~14000 training documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "universal-image",
   "metadata": {},
   "outputs": [],
   "source": [
    "from seasons_module import seasons_tokenize\n",
    "tokenized_training_docs = []\n",
    "for doc in training_docs:\n",
    "    tdoc = seasons_tokenize(doc)\n",
    "    tokenized_training_docs.append(tdoc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "awful-yahoo",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdist = nltk.FreqDist()\n",
    "for doc in tokenized_training_docs:\n",
    "    fdist.update(doc)\n",
    "\n",
    "f = open(\"lists/seasons_stop_list.txt\")\n",
    "stop_list = set(f.read().split(\"\\n\"))\n",
    "\n",
    "full_vocab = [w[0] for w in fdist.most_common() if w[0] not in stop_list]\n",
    "vocab = full_vocab[:500]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "physical-chamber",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "## Get the document vectors\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "scheduled-incentive",
   "metadata": {},
   "source": [
    "When I first tried to compute document vectors executing it took a long time to run - 5 or 10 minutes. I had to poke around the internet and fiddle a bit. I figured out that building the big matrix in chunks and then putting those chunks together made a big difference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "legitimate-thesis",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, clear_output\n",
    "\n",
    "def wfactor(tf):\n",
    "    if tf == 0:\n",
    "        result = 0\n",
    "    else:\n",
    "        result = (1 + np.log(tf))\n",
    "    return result\n",
    "\n",
    "def compute_doc_vector(token_list, vocab):\n",
    "    return np.array([wfactor(token_list.count(word)) for word in vocab])\n",
    "\n",
    "chunk_list = []\n",
    "empty_rows = 0\n",
    "chunk = np.array([], dtype=np.int64).reshape(0,len(vocab))\n",
    "for i, tdoc in enumerate(tokenized_training_docs):\n",
    "    new_row = compute_doc_vector(tdoc, vocab)\n",
    "    if i % 500 == 0:\n",
    "        clear_output(wait=True)\n",
    "        display(str(i))\n",
    "        chunk_list.append(chunk)\n",
    "        chunk = np.array([], dtype=np.int64).reshape(0,len(vocab))\n",
    "    if np.linalg.norm(new_row) == 0:\n",
    "        empty_rows += 1\n",
    "        continue\n",
    "    chunk = np.concatenate([chunk, np.array([new_row])])\n",
    "    \n",
    "chunk_list.append(chunk)\n",
    "clear_output(wait=True)\n",
    "display(str(i) + \" joining chunks\")\n",
    "\n",
    "training_dt_matrix2 = np.concatenate(chunk_list)\n",
    "print(\"empty rows {}\".format(empty_rows))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "finnish-webcam",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_dt_matrix2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "optimum-drinking",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "show_labeled_table(training_dt_matrix2, vocab, para_names, nrows=15, ncols=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acoustic-delta",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## Word vectors with reduced dimensions\n",
    "\n",
    "If, as before, we think of each of the columns in our table as a vector for the words, then each word is a vector with 13,264 dimensions.\n",
    "\n",
    "We can use something called the singular value decomposition to reduce the number of dimensions. You read about this in the paper by Landauer. This is the trick that is used in latent semantic analysis.\n",
    "\n",
    "First we pick a number of dimensions. I'm using a slider widget simply because it's fun."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "immune-pulse",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "import ipywidgets as widgets\n",
    "w = widgets.IntSlider(value=100, max=200, description=\"rdims\")\n",
    "display(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "coordinate-domestic",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "rdims = w.value\n",
    "# do the SVD and reduce dimensions\n",
    "T, S, Dt = np.linalg.svd(training_dt_matrix2.transpose(), full_matrices = False)\n",
    "T_reduced = T[:, 0:rdims]\n",
    "T_normed = normalize_rows(T_reduced)\n",
    "print(\"shape of t_normed is {}\".format(T_normed.shape))\n",
    "show_labeled_table(T_normed, None, vocab, nrows=10, ncols=100, color_nums=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "hybrid-interpretation",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "The SVD produces three matrices. We can think of the first one \"T\" as having rows corresponding to our vocabulary. And we can take as many columns as we want."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "equipped-designer",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "def get_word_vector(w, vocab, mat):\n",
    "    return norm_vec(mat[vocab.index(w)])\n",
    "\n",
    "def compare_word_vectors(w1, w2, vocab, mat):\n",
    "    return np.dot(get_word_vector(w1, vocab, mat), get_word_vector(w2, vocab, mat))\n",
    "\n",
    "def get_doc_vector(doc, vocab, td_mat):\n",
    "    s = np.zeros(td_mat.shape[1])\n",
    "    for w in doc:\n",
    "        if w in vocab:\n",
    "            s = s + get_word_vector(w, vocab, td_mat)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unsigned-springer",
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_word_vectors(\"close\", \"closer\", vocab, T_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "brave-timing",
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_similar(w, vocab, mat, n=10):\n",
    "    sims = []\n",
    "    for w2 in vocab:\n",
    "        if w2 == w:\n",
    "            continue\n",
    "        sims.append([w2, compare_word_vectors(w, w2, vocab, mat), fdist[w2]])\n",
    "    return sorted(sims, key=lambda item: item[1], reverse=True)[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "static-valuation",
   "metadata": {},
   "outputs": [],
   "source": [
    "most_similar(\"hemisphere\", vocab, T_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "center-somalia",
   "metadata": {},
   "outputs": [],
   "source": [
    "most_similar(\"tilt\", vocab, T_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "baking-monte",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## PCA for plotting\n",
    "\n",
    "I'll show you a way of squishing any higher dimensional matrix down to a smaller number of dimensions for the purpose of making plots."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "elect-youth",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "def squish_matrix(X, ncomponents=2):\n",
    "    pca = PCA(n_components=ncomponents)\n",
    "    return pca.fit_transform(X)\n",
    "\n",
    "def double_squish(X, ncomponents=2):\n",
    "    pca = PCA(n_components=50)\n",
    "    reduc = pca.fit_transform(X)\n",
    "    return TSNE(n_components=ncomponents, random_state=0, perplexity=15).fit_transform(reduc)\n",
    "\n",
    "def alt_squish(X, ncomponents=2):\n",
    "    return TSNE(n_components=ncomponents, random_state=0, perplexity=15).fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "greater-second",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "def plot_matrix(mat, labels, number_to_plot=10, figsize=(10, 10), c=\"red\"):\n",
    "    if mat.shape[1] > 3:\n",
    "        print(\"too many dimensions\")\n",
    "        return\n",
    "    if number_to_plot > mat.shape[0]:\n",
    "        number_to_plot = mat.shape[0]\n",
    "    fig=plt.figure(figsize=figsize, dpi= 80, facecolor='w', edgecolor='k')\n",
    "    if mat.shape[1] == 3:\n",
    "        ax = fig.add_subplot(111, projection='3d')\n",
    "        ax.scatter(mat[:, 0][:number_to_plot], mat[:, 1][:number_to_plot], mat[:, 2][:number_to_plot], c=c)\n",
    "        for i in range(number_to_plot):\n",
    "            ax.text(mat[i, 0], mat[i, 1], mat[i, 2], labels[i])\n",
    "\n",
    "    else:\n",
    "        plt.scatter(mat[:, 0][:number_to_plot], mat[:, 1][:number_to_plot], c=c)\n",
    "        for i in range(number_to_plot):\n",
    "            plt.annotate(labels[i], mat[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "reliable-engagement",
   "metadata": {},
   "outputs": [],
   "source": [
    "sm = squish_matrix(T_normed, 3)\n",
    "plot_matrix(sm, vocab, 10, figsize=(10, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "seeing-guyana",
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "subslide"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sm = squish_matrix(T_normed, 3)\n",
    "plot_matrix(sm, vocab, 10, figsize=(10, 10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "joined-tomato",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Let's instead plot a few selected words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "moderate-bench",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_words = [\"tilt\", \"hemisphere\", \"northern\", \"southern\", \"side\", \"day\", \"night\", \"moon\", \"spin\", \"closer\", \"close\", \"near\", \"far\", \"farther\"]\n",
    "folded_vocab_list = []\n",
    "found_vocab = []\n",
    "for w in selected_words:\n",
    "    if w not in vocab:\n",
    "        continue\n",
    "    found_vocab.append(w)\n",
    "    v = get_word_vector(w, vocab, T_normed)\n",
    "    # v = sm[vocab.index(w)]\n",
    "    folded_vocab_list.append(v)\n",
    "folded_vocab_matrix = np.array(folded_vocab_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "seasonal-letter",
   "metadata": {},
   "outputs": [],
   "source": [
    "show_labeled_table(folded_vocab_matrix, None, found_vocab, nrows=25, color_nums=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "particular-density",
   "metadata": {},
   "outputs": [],
   "source": [
    "sm = squish_matrix(folded_vocab_matrix, 2)\n",
    "plot_matrix(sm, found_vocab, 10, figsize=(5, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "stock-rings",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy, random\n",
    "def plot_similar(the_word, vocab, mat, dims=2, n=10):\n",
    "    similar_words = [w[0] for w in most_similar(the_word, vocab, mat)]\n",
    "    folded_vocab_list = [mat[vocab.index(the_word)]]\n",
    "    found_vocab = [the_word]\n",
    "    colors = [\"red\"]\n",
    "    for w in similar_words[:n]:\n",
    "        if w not in vocab:\n",
    "            continue\n",
    "        found_vocab.append(w)\n",
    "        v = mat[vocab.index(w)]\n",
    "        folded_vocab_list.append(v)\n",
    "        colors.append(\"blue\")\n",
    "    new_vocab = copy.deepcopy(vocab)\n",
    "    random.shuffle(new_vocab)\n",
    "    for w in new_vocab[:n]:\n",
    "        if w not in vocab:\n",
    "            continue\n",
    "        found_vocab.append(w)\n",
    "        v = mat[vocab.index(w)]\n",
    "        folded_vocab_list.append(v)\n",
    "        colors.append(\"green\")\n",
    "    folded_vocab_matrix = np.array(folded_vocab_list)\n",
    "    squished_matrix = alt_squish(folded_vocab_matrix, dims)\n",
    "    plot_matrix(squished_matrix, found_vocab, 50, c=colors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "happy-leave",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "plot_similar(\"hemisphere\", vocab, T_normed, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "monetary-examination",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "316.59375px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}