W2-3 Bigrams and Wikidata.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "intense-albania",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import pandas as pd\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cross-application",
   "metadata": {},
   "source": [
    "# N-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "urban-practitioner",
   "metadata": {},
   "outputs": [],
   "source": [
    "some_text = \"This is a first sentence. Now comes the next sentence.\".lower()\n",
    "words = nltk.word_tokenize(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "harmful-istanbul",
   "metadata": {},
   "outputs": [],
   "source": [
    "bgs = list(nltk.bigrams(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "related-testimony",
   "metadata": {},
   "outputs": [],
   "source": [
    "tgs = list(nltk.ngrams(words, n=3))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "marked-diving",
   "metadata": {},
   "source": [
    "We might not want bigrams that have the last word of one sentence and the first word of the next."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "recent-eligibility",
   "metadata": {},
   "outputs": [],
   "source": [
    "sents = nltk.sent_tokenize(some_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eleven-character",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_bgs = []\n",
    "for sent in sents:\n",
    "    new_bgs += nltk.bigrams(nltk.word_tokenize(sent))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "committed-progress",
   "metadata": {},
   "source": [
    "# Getting some data from wikipedia"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "beneficial-clark",
   "metadata": {},
   "source": [
    "There's a nice [library](https://pypi.org/project/Wikipedia-API/) that makes grabbing pages from wikipedia pretty easy.\n",
    "\n",
    "You always start by creating this `wiki_wiki` object."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "legendary-findings",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipediaapi\n",
    "wiki_wiki = wikipediaapi.Wikipedia(\n",
    "        language='en',\n",
    "        extract_format=wikipediaapi.ExtractFormat.WIKI\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "smoking-success",
   "metadata": {},
   "source": [
    "Let's grab some learning sciences related pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cosmetic-addiction",
   "metadata": {},
   "outputs": [],
   "source": [
    "pages = [\n",
    "    \"Learning Sciences\",\n",
    "    \"Educational psychology\",\n",
    "    \"Learning\",\n",
    "    \"Informal learning\",\n",
    "    \"Design-based research\",\n",
    "    \"The Journal of the Learning Sciences\",\n",
    "    \"Janet L. Kolodner\",\n",
    "    \"Computer-supported collaborative learning\",\n",
    "    \"Educational technolog\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "constant-reproduction",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def underscorize(pagename):\n",
    "    return re.sub(\" \", \"_\", pagename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "greatest-spectacular",
   "metadata": {},
   "outputs": [],
   "source": [
    "page_dict = {}\n",
    "for page in pages:\n",
    "    pagename = underscorize(page)\n",
    "    print(pagename)\n",
    "    p_wiki = wiki_wiki.page(pagename)\n",
    "    page_dict[pagename] = p_wiki.text.lower()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "seeing-screen",
   "metadata": {},
   "source": [
    "## Collocations in this LS Corpus\n",
    "We are interested in finding **collocations**. Collocations are pairs of words, or phrases, that have limited compositionality. This means that the meaning of the phrase cannot be determined simply from the meaning of the parts. The whole thing is, from a meaning point of view, a unit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "labeled-wholesale",
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_fdist = nltk.FreqDist()\n",
    "for page_text in page_dict.values():\n",
    "    for sent in nltk.sent_tokenize(page_text):\n",
    "        sent_bgs = nltk.bigrams(nltk.word_tokenize(sent))\n",
    "        bigram_fdist.update(sent_bgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "driving-album",
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_fdist.most_common(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "occupied-germany",
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open(\"lists/stop-words_english_5_en.txt\")\n",
    "stop_list = f.read().split(\"\\n\")\n",
    "stop_list += list('!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~’')\n",
    "stop_list += list(\"abcdefghijklmnopqrstuvwxyz0123456789\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "decreased-yellow",
   "metadata": {},
   "outputs": [],
   "source": [
    "most_common_bigrams = bigram_fdist.most_common(2000)\n",
    "most_common_pruned = []\n",
    "for bg_entry in most_common_bigrams:\n",
    "    if bg_entry[0][0] in stop_list or bg_entry[0][1] in stop_list:\n",
    "        continue\n",
    "    most_common_pruned.append(bg_entry)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "north-navigation",
   "metadata": {},
   "outputs": [],
   "source": [
    "most_common_pruned[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "polar-satellite",
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf = pd.DataFrame(most_common_pruned, columns=[\"words\", \"count\"])\n",
    "tdf.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "earned-calculation",
   "metadata": {},
   "source": [
    "# Are some bigrams more common than to be expected? Which are most unexpected?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "solar-flood",
   "metadata": {},
   "outputs": [],
   "source": [
    "word_fdist = nltk.FreqDist()\n",
    "for page_text in page_dict.values():\n",
    "    for word in nltk.word_tokenize(page_text):\n",
    "        if word not in stop_list:\n",
    "            word_fdist[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "smooth-visibility",
   "metadata": {},
   "outputs": [],
   "source": [
    "fraction_expected = word_fdist[\"learning\"] / word_fdist.N()\n",
    "print(\"fraction_expected=\" + str(p_learning))\n",
    "\n",
    "fraction_observed = bigram_fdist[(\"informal\", \"learning\")] / word_fdist[\"informal\"]\n",
    "print(\"fraction_observed=\" + str(fraction_observed))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "brown-hotel",
   "metadata": {},
   "source": [
    "So we would expect to see \"learning\" after \"informal\" 3% of the times. But we see it over 70% of the time."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "manual-satisfaction",
   "metadata": {},
   "source": [
    "## t_test as measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "spanish-average",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_1samp\n",
    "def student_t(w1, w2, word_fidst, bigram_fdist):\n",
    "    mu = word_fdist[w1] * word_fdist[w2] / (word_fdist.N() * word_fdist.N())\n",
    "    blist = bigram_fdist[(w1, w2)] * [1.0] + (bigram_fdist.N() - bigram_fdist[(w1, w2)]) * [0]\n",
    "    result = ttest_1samp(blist, mu)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beautiful-communications",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "student_t(\"informal\", \"learning\", word_fdist, bigram_fdist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "documentary-superior",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = []\n",
    "for mc in most_common_pruned[:50]:\n",
    "    w1 = mc[0][0]\n",
    "    w2 = mc[0][1]\n",
    "    t.append([w1, w2, mc[1], round(student_t(w1, w2, word_fdist, bigram_fdist).statistic, 3)])\n",
    "df = pd.DataFrame(t, columns=[\"w1\", \"w2\", \"count\", \"t\"])\n",
    "df.sort_values(by=\"t\", ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "agreed-andrews",
   "metadata": {},
   "source": [
    "# Some random exploration of word co-occurrences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "short-brisbane",
   "metadata": {},
   "outputs": [],
   "source": [
    "pruned_ls_words = []\n",
    "pruned_ls_sents = []\n",
    "for page in page_dict.values():\n",
    "    for sent in nltk.sent_tokenize(page):\n",
    "        psent = [w for w in nltk.word_tokenize(sent) if w not in stop_list]\n",
    "        pruned_ls_words += psent\n",
    "        pruned_ls_sents.append(psent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "favorite-association",
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "pword_fdist = nltk.FreqDist(pruned_ls_words)\n",
    "most_common = [w[0] for w in pword_fdist.most_common(50)]\n",
    "mrow = [0 for r in most_common]\n",
    "mat = [copy.copy(mrow) for r in most_common]\n",
    "for sent in pruned_ls_sents:\n",
    "    for n1, w1 in enumerate(most_common):\n",
    "        if w1 in sent:\n",
    "            for n2, w2 in enumerate(most_common):\n",
    "                if w2 in sent:\n",
    "                    mat[n1][n2] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ambient-waste",
   "metadata": {},
   "source": [
    "## Matplotlib digression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quantitative-practice",
   "metadata": {},
   "source": [
    "https://matplotlib.org/api/pyplot_summary.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "broadband-cycle",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot([1, 3, 4, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eligible-penetration",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot([1, 3, 4, 2], \"bo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "super-pocket",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar([1, 2, 3, 4], [1, 3, 4, 2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "known-psychology",
   "metadata": {},
   "source": [
    "## Back to co-occurrence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fitted-bridge",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "fig=plt.figure(figsize=(10, 10), dpi= 80, facecolor='w', edgecolor='k')\n",
    "n = len(most_common)\n",
    "plt.xticks(x_tick_marks, most_common, fontsize=8, rotation=90)\n",
    "plt.yticks(y_tick_marks, most_common, fontsize=8)\n",
    "plt.tick_params(\"x\", top=True, labeltop=True, bottom=False, labelbottom=False)\n",
    "plt.imshow(mat, norm=matplotlib.colors.LogNorm(), interpolation='nearest', cmap='YlOrBr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "realistic-tablet",
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "possible_pairs = list( itertools.combinations(most_common[:25], 2)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "developing-primary",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "fig=plt.figure(figsize=(10, 10), dpi= 80, facecolor='w', edgecolor='k')\n",
    "G2 = nx.Graph()\n",
    "for pair in possible_pairs:\n",
    "    wgt = mat[most_common.index(pair[0])][most_common.index(pair[1])]\n",
    "    G2.add_edge(pair[0], pair[1], weight=wgt)\n",
    "widths = list(nx.get_edge_attributes(G2, \"weight\").values())\n",
    "widths = [w / 25 for w in widths]\n",
    "pos = nx.spring_layout(G2, iterations=40, k=.1, weight=\"weight\") \n",
    "nx.draw(G2, pos, with_labels=True, width=widths, node_color=\"gold\", edge_color=\"orange\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "copyrighted-browser",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}