W7-4 Seasons with LSA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from numpy import linalg\n",
    "from numpy import dot\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def norm_vec(v):\n",
    "    return v / np.linalg.norm(v)\n",
    "np.set_printoptions(precision=3)\n",
    "\n",
    "def normalize_rows(x):\n",
    "    return x/np.linalg.norm(x, ord=2, axis=1, keepdims=True)\n",
    "\n",
    "def normalize_columns(x):\n",
    "    return x/np.linalg.norm(x, ord=2, axis=0, keepdims=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Seasons with LSA</h1>\n",
    "\n",
    "This is all about squishing down the number of dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utilities import *\n",
    "from seasons_module import load_seasons_corpus_as_utterances\n",
    "from seasons_module import load_seasons_comparison_files\n",
    "\n",
    "import numpy as np\n",
    "def norm_vec(vec):\n",
    "    mag = np.dot(vec, vec)\n",
    "    if mag == 0:\n",
    "        return vec\n",
    "    else:\n",
    "        return(vec / np.sqrt(mag))\n",
    "    \n",
    "def pure_tf(tf, df, cf, N):\n",
    "    return tf\n",
    "\n",
    "def tf(tf):\n",
    "    if tf == 0:\n",
    "        result = 0\n",
    "    else:\n",
    "        result = (1 + np.log(tf))\n",
    "    return result\n",
    "\n",
    "def weighted_word(the_text, word):\n",
    "    return tf(the_text.count(word))\n",
    "\n",
    "def compute_normed_doc_vector(word_list, vocab):\n",
    "    return norm_vec(np.array([weighted_word(word_list, word) for word in vocab]))\n",
    "\n",
    "def compute_doc_vector(word_list, vocab):\n",
    "    return np.array([weighted_word(word_list, word) for word in vocab])\n",
    "\n",
    "def prepare_seasons_corpus_by_utterance(vocab_size=50, norm_vecs=False):\n",
    "    seasons_corpus = load_seasons_corpus_as_utterances()\n",
    "    f = open(\"lists/seasons_stop_list.txt\")\n",
    "    stop_list = set(f.read().split(\"\\n\"))\n",
    "    word_fdist = nltk.FreqDist()\n",
    "    for fname in seasons_corpus.keys():\n",
    "        for utterance in seasons_corpus[fname][0]:\n",
    "            pruned_transcript_words = [w for w in utterance if w not in stop_list]\n",
    "            word_fdist.update(pruned_transcript_words)\n",
    "    new_vocab = [w[0] for w in word_fdist.most_common(vocab_size) if w not in stop_list]\n",
    "    \n",
    "    # compute the document vector for each document\n",
    "    doc_vectors = []\n",
    "    for fname in seasons_corpus.keys():\n",
    "        for utterance in seasons_corpus[fname][0]:\n",
    "            if norm_vecs:\n",
    "                doc_vectors.append(compute_normed_doc_vector(utterance, new_vocab))\n",
    "            else:\n",
    "                doc_vectors.append(compute_doc_vector(utterance, new_vocab))\n",
    "    return doc_vectors, new_vocab\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_vectors, vocab = prepare_seasons_corpus_by_utterance()\n",
    "X = np.array(doc_vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Seasons with LSI</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The SVD gives us a way of taking the word vectors and squishing them down to fewer dimensions.\n",
    "\n",
    "We are going to squish them all the way down to two dimensions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dims = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "T, S, Dt = np.linalg.svd(X.transpose(), full_matrices = False)\n",
    "T_reduced = T[:, 0:dims]\n",
    "T_normed = normalize_rows(T_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now T_reduced is an array where each row is a vector corresponding to each of the words in our vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "T_reduced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_row_vector(w, T):\n",
    "    rnumber = vocab.index(w)\n",
    "    return T[rnumber]\n",
    "\n",
    "def compare_words(w1, w2, T):\n",
    "    v1 = get_row_vector(w1, T)\n",
    "    v2 = get_row_vector(w2, T)\n",
    "    return np.dot(norm_vec(v1), norm_vec(v2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_row_vector(\"tilt\", T_reduced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_words(\"farther\", \"closer\", T_reduced)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since there are only two dimensions we can show the word vectors on a plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig=plt.figure(figsize=(20, 10), dpi= 80, facecolor='w', edgecolor='k')\n",
    "plt.scatter(T_reduced.transpose()[0], T_reduced.transpose()[1])\n",
    "for i in range(len(vocab)):\n",
    "    plt.annotate(vocab[i], T_reduced[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig=plt.figure(figsize=(20, 10), dpi=80, facecolor='w', edgecolor='k')\n",
    "plt.scatter(T_normed.transpose()[0], T_normed.transpose()[1])\n",
    "for i in range(len(vocab)):\n",
    "    plt.annotate(vocab[i], T_normed[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}