2-1-Deriving-N-Grams-from-Text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deriving N-Grams from Text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on [N-Gram-Based Text Categorization: Categorizing Text With Python by Alejandro Nolla](http://blog.alejandronolla.com/2013/05/20/n-gram-based-text-categorization-categorizing-text-with-python/)\n",
    "\n",
    "What are n-grams? See [here](http://cloudmark.github.io/Language-Detection/)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = \"Le temps est un grand maître, dit-on, le malheur est qu'il tue ses élèves.\"\n",
    "s = s.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['le',\n",
       " 'temps',\n",
       " 'est',\n",
       " 'un',\n",
       " 'grand',\n",
       " 'maître',\n",
       " 'dit',\n",
       " 'on',\n",
       " 'le',\n",
       " 'malheur',\n",
       " 'est',\n",
       " \"qu'il\",\n",
       " 'tue',\n",
       " 'ses',\n",
       " 'élèves']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.tokenize import RegexpTokenizer\n",
    "tokenizer = RegexpTokenizer(\"[a-zA-Z'`éèî]+\")\n",
    "s_tokenized = tokenizer.tokenize(s)\n",
    "s_tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[('_', '_', '_', 'l'),\n",
       "  ('_', '_', 'l', 'e'),\n",
       "  ('_', 'l', 'e', '_'),\n",
       "  ('l', 'e', '_', '_'),\n",
       "  ('e', '_', '_', '_')],\n",
       " [('_', '_', '_', 't'),\n",
       "  ('_', '_', 't', 'e'),\n",
       "  ('_', 't', 'e', 'm'),\n",
       "  ('t', 'e', 'm', 'p'),\n",
       "  ('e', 'm', 'p', 's'),\n",
       "  ('m', 'p', 's', '_'),\n",
       "  ('p', 's', '_', '_'),\n",
       "  ('s', '_', '_', '_')],\n",
       " [('_', '_', '_', 'e'),\n",
       "  ('_', '_', 'e', 's'),\n",
       "  ('_', 'e', 's', 't'),\n",
       "  ('e', 's', 't', '_'),\n",
       "  ('s', 't', '_', '_'),\n",
       "  ('t', '_', '_', '_')],\n",
       " [('_', '_', '_', 'u'),\n",
       "  ('_', '_', 'u', 'n'),\n",
       "  ('_', 'u', 'n', '_'),\n",
       "  ('u', 'n', '_', '_'),\n",
       "  ('n', '_', '_', '_')],\n",
       " [('_', '_', '_', 'g'),\n",
       "  ('_', '_', 'g', 'r'),\n",
       "  ('_', 'g', 'r', 'a'),\n",
       "  ('g', 'r', 'a', 'n'),\n",
       "  ('r', 'a', 'n', 'd'),\n",
       "  ('a', 'n', 'd', '_'),\n",
       "  ('n', 'd', '_', '_'),\n",
       "  ('d', '_', '_', '_')],\n",
       " [('_', '_', '_', 'm'),\n",
       "  ('_', '_', 'm', 'a'),\n",
       "  ('_', 'm', 'a', 'î'),\n",
       "  ('m', 'a', 'î', 't'),\n",
       "  ('a', 'î', 't', 'r'),\n",
       "  ('î', 't', 'r', 'e'),\n",
       "  ('t', 'r', 'e', '_'),\n",
       "  ('r', 'e', '_', '_'),\n",
       "  ('e', '_', '_', '_')],\n",
       " [('_', '_', '_', 'd'),\n",
       "  ('_', '_', 'd', 'i'),\n",
       "  ('_', 'd', 'i', 't'),\n",
       "  ('d', 'i', 't', '_'),\n",
       "  ('i', 't', '_', '_'),\n",
       "  ('t', '_', '_', '_')],\n",
       " [('_', '_', '_', 'o'),\n",
       "  ('_', '_', 'o', 'n'),\n",
       "  ('_', 'o', 'n', '_'),\n",
       "  ('o', 'n', '_', '_'),\n",
       "  ('n', '_', '_', '_')],\n",
       " [('_', '_', '_', 'l'),\n",
       "  ('_', '_', 'l', 'e'),\n",
       "  ('_', 'l', 'e', '_'),\n",
       "  ('l', 'e', '_', '_'),\n",
       "  ('e', '_', '_', '_')],\n",
       " [('_', '_', '_', 'm'),\n",
       "  ('_', '_', 'm', 'a'),\n",
       "  ('_', 'm', 'a', 'l'),\n",
       "  ('m', 'a', 'l', 'h'),\n",
       "  ('a', 'l', 'h', 'e'),\n",
       "  ('l', 'h', 'e', 'u'),\n",
       "  ('h', 'e', 'u', 'r'),\n",
       "  ('e', 'u', 'r', '_'),\n",
       "  ('u', 'r', '_', '_'),\n",
       "  ('r', '_', '_', '_')],\n",
       " [('_', '_', '_', 'e'),\n",
       "  ('_', '_', 'e', 's'),\n",
       "  ('_', 'e', 's', 't'),\n",
       "  ('e', 's', 't', '_'),\n",
       "  ('s', 't', '_', '_'),\n",
       "  ('t', '_', '_', '_')],\n",
       " [('_', '_', '_', 'q'),\n",
       "  ('_', '_', 'q', 'u'),\n",
       "  ('_', 'q', 'u', \"'\"),\n",
       "  ('q', 'u', \"'\", 'i'),\n",
       "  ('u', \"'\", 'i', 'l'),\n",
       "  (\"'\", 'i', 'l', '_'),\n",
       "  ('i', 'l', '_', '_'),\n",
       "  ('l', '_', '_', '_')],\n",
       " [('_', '_', '_', 't'),\n",
       "  ('_', '_', 't', 'u'),\n",
       "  ('_', 't', 'u', 'e'),\n",
       "  ('t', 'u', 'e', '_'),\n",
       "  ('u', 'e', '_', '_'),\n",
       "  ('e', '_', '_', '_')],\n",
       " [('_', '_', '_', 's'),\n",
       "  ('_', '_', 's', 'e'),\n",
       "  ('_', 's', 'e', 's'),\n",
       "  ('s', 'e', 's', '_'),\n",
       "  ('e', 's', '_', '_'),\n",
       "  ('s', '_', '_', '_')],\n",
       " [('_', '_', '_', 'é'),\n",
       "  ('_', '_', 'é', 'l'),\n",
       "  ('_', 'é', 'l', 'è'),\n",
       "  ('é', 'l', 'è', 'v'),\n",
       "  ('l', 'è', 'v', 'e'),\n",
       "  ('è', 'v', 'e', 's'),\n",
       "  ('v', 'e', 's', '_'),\n",
       "  ('e', 's', '_', '_'),\n",
       "  ('s', '_', '_', '_')]]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.util import ngrams\n",
    "generated_4grams = []\n",
    "\n",
    "for word in s_tokenized:\n",
    "    generated_4grams.append(list(ngrams(word, 4, pad_left=True, pad_right=True, left_pad_symbol='_', right_pad_symbol='_'))) # n is 4.\n",
    "generated_4grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems that `generated_4grams` needs flattening since it's supposed to be a list of 4-grams:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('_', '_', '_', 'l'),\n",
       " ('_', '_', 'l', 'e'),\n",
       " ('_', 'l', 'e', '_'),\n",
       " ('l', 'e', '_', '_'),\n",
       " ('e', '_', '_', '_'),\n",
       " ('_', '_', '_', 't'),\n",
       " ('_', '_', 't', 'e'),\n",
       " ('_', 't', 'e', 'm'),\n",
       " ('t', 'e', 'm', 'p'),\n",
       " ('e', 'm', 'p', 's')]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_4grams = [word for sublist in generated_4grams for word in sublist]\n",
    "generated_4grams[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Obtaining n-grams (n = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['___l',\n",
       " '__le',\n",
       " '_le_',\n",
       " 'le__',\n",
       " 'e___',\n",
       " '___t',\n",
       " '__te',\n",
       " '_tem',\n",
       " 'temp',\n",
       " 'emps',\n",
       " 'mps_',\n",
       " 'ps__',\n",
       " 's___',\n",
       " '___e',\n",
       " '__es',\n",
       " '_est',\n",
       " 'est_',\n",
       " 'st__',\n",
       " 't___',\n",
       " '___u',\n",
       " '__un',\n",
       " '_un_',\n",
       " 'un__',\n",
       " 'n___',\n",
       " '___g',\n",
       " '__gr',\n",
       " '_gra',\n",
       " 'gran',\n",
       " 'rand',\n",
       " 'and_',\n",
       " 'nd__',\n",
       " 'd___',\n",
       " '___m',\n",
       " '__ma',\n",
       " '_maî',\n",
       " 'maît',\n",
       " 'aîtr',\n",
       " 'ître',\n",
       " 'tre_',\n",
       " 're__',\n",
       " 'e___',\n",
       " '___d',\n",
       " '__di',\n",
       " '_dit',\n",
       " 'dit_',\n",
       " 'it__',\n",
       " 't___',\n",
       " '___o',\n",
       " '__on',\n",
       " '_on_',\n",
       " 'on__',\n",
       " 'n___',\n",
       " '___l',\n",
       " '__le',\n",
       " '_le_',\n",
       " 'le__',\n",
       " 'e___',\n",
       " '___m',\n",
       " '__ma',\n",
       " '_mal',\n",
       " 'malh',\n",
       " 'alhe',\n",
       " 'lheu',\n",
       " 'heur',\n",
       " 'eur_',\n",
       " 'ur__',\n",
       " 'r___',\n",
       " '___e',\n",
       " '__es',\n",
       " '_est',\n",
       " 'est_',\n",
       " 'st__',\n",
       " 't___',\n",
       " '___q',\n",
       " '__qu',\n",
       " \"_qu'\",\n",
       " \"qu'i\",\n",
       " \"u'il\",\n",
       " \"'il_\",\n",
       " 'il__',\n",
       " 'l___',\n",
       " '___t',\n",
       " '__tu',\n",
       " '_tue',\n",
       " 'tue_',\n",
       " 'ue__',\n",
       " 'e___',\n",
       " '___s',\n",
       " '__se',\n",
       " '_ses',\n",
       " 'ses_',\n",
       " 'es__',\n",
       " 's___',\n",
       " '___é',\n",
       " '__él',\n",
       " '_élè',\n",
       " 'élèv',\n",
       " 'lève',\n",
       " 'èves',\n",
       " 'ves_',\n",
       " 'es__',\n",
       " 's___']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ng_list_4grams = generated_4grams\n",
    "for idx, val in enumerate(generated_4grams):\n",
    "    ng_list_4grams[idx] = ''.join(val)\n",
    "ng_list_4grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Sorting n-grams by frequency (n = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('e___', 4),\n",
       " ('s___', 3),\n",
       " ('t___', 3),\n",
       " ('___l', 2),\n",
       " ('__le', 2),\n",
       " ('_le_', 2),\n",
       " ('le__', 2),\n",
       " ('___t', 2),\n",
       " ('___e', 2),\n",
       " ('__es', 2),\n",
       " ('_est', 2),\n",
       " ('est_', 2),\n",
       " ('st__', 2),\n",
       " ('n___', 2),\n",
       " ('___m', 2),\n",
       " ('__ma', 2),\n",
       " ('es__', 2),\n",
       " ('__te', 1),\n",
       " ('_tem', 1),\n",
       " ('temp', 1),\n",
       " ('emps', 1),\n",
       " ('mps_', 1),\n",
       " ('ps__', 1),\n",
       " ('___u', 1),\n",
       " ('__un', 1),\n",
       " ('_un_', 1),\n",
       " ('un__', 1),\n",
       " ('___g', 1),\n",
       " ('__gr', 1),\n",
       " ('_gra', 1),\n",
       " ('gran', 1),\n",
       " ('rand', 1),\n",
       " ('and_', 1),\n",
       " ('nd__', 1),\n",
       " ('d___', 1),\n",
       " ('_maî', 1),\n",
       " ('maît', 1),\n",
       " ('aîtr', 1),\n",
       " ('ître', 1),\n",
       " ('tre_', 1),\n",
       " ('re__', 1),\n",
       " ('___d', 1),\n",
       " ('__di', 1),\n",
       " ('_dit', 1),\n",
       " ('dit_', 1),\n",
       " ('it__', 1),\n",
       " ('___o', 1),\n",
       " ('__on', 1),\n",
       " ('_on_', 1),\n",
       " ('on__', 1),\n",
       " ('_mal', 1),\n",
       " ('malh', 1),\n",
       " ('alhe', 1),\n",
       " ('lheu', 1),\n",
       " ('heur', 1),\n",
       " ('eur_', 1),\n",
       " ('ur__', 1),\n",
       " ('r___', 1),\n",
       " ('___q', 1),\n",
       " ('__qu', 1),\n",
       " (\"_qu'\", 1),\n",
       " (\"qu'i\", 1),\n",
       " (\"u'il\", 1),\n",
       " (\"'il_\", 1),\n",
       " ('il__', 1),\n",
       " ('l___', 1),\n",
       " ('__tu', 1),\n",
       " ('_tue', 1),\n",
       " ('tue_', 1),\n",
       " ('ue__', 1),\n",
       " ('___s', 1),\n",
       " ('__se', 1),\n",
       " ('_ses', 1),\n",
       " ('ses_', 1),\n",
       " ('___é', 1),\n",
       " ('__él', 1),\n",
       " ('_élè', 1),\n",
       " ('élèv', 1),\n",
       " ('lève', 1),\n",
       " ('èves', 1),\n",
       " ('ves_', 1)]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freq_4grams = {}\n",
    "\n",
    "for ngram in ng_list_4grams:\n",
    "    if ngram not in freq_4grams:\n",
    "        freq_4grams.update({ngram: 1})\n",
    "    else:\n",
    "        ngram_occurrences = freq_4grams[ngram]\n",
    "        freq_4grams.update({ngram: ngram_occurrences + 1})\n",
    "        \n",
    "from operator import itemgetter # The operator module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, operator.add(x, y) is equivalent to the expression x + y.\n",
    "\n",
    "freq_4grams_sorted = sorted(freq_4grams.items(), key=itemgetter(1), reverse=True)[0:300] # We only keep the 300 most popular n-grams. This was suggested in the original paper written about n-grams.\n",
    "freq_4grams_sorted"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Obtaining n-grams for multiple values of n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get n-grams for n = 1, 2, 3 and 4 we can use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"le temps est un grand maître dit on le malheur est qu'il tue ses élèves\""
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk import everygrams\n",
    "\n",
    "s_clean = ' '.join(s_tokenized) # For the code below we need the raw sentence as opposed to the tokens.\n",
    "s_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['l',\n",
       " 'e',\n",
       " 't',\n",
       " 'e',\n",
       " 'm',\n",
       " 'p',\n",
       " 's',\n",
       " 'e',\n",
       " 's',\n",
       " 't',\n",
       " 'u',\n",
       " 'n',\n",
       " 'g',\n",
       " 'r',\n",
       " 'a',\n",
       " 'n',\n",
       " 'd',\n",
       " 'm',\n",
       " 'a',\n",
       " 'î',\n",
       " 't',\n",
       " 'r',\n",
       " 'e',\n",
       " 'd',\n",
       " 'i',\n",
       " 't',\n",
       " 'o',\n",
       " 'n',\n",
       " 'l',\n",
       " 'e',\n",
       " 'm',\n",
       " 'a',\n",
       " 'l',\n",
       " 'h',\n",
       " 'e',\n",
       " 'u',\n",
       " 'r',\n",
       " 'e',\n",
       " 's',\n",
       " 't',\n",
       " 'q',\n",
       " 'u',\n",
       " \"'\",\n",
       " 'i',\n",
       " 'l',\n",
       " 't',\n",
       " 'u',\n",
       " 'e',\n",
       " 's',\n",
       " 'e',\n",
       " 's',\n",
       " 'é',\n",
       " 'l',\n",
       " 'è',\n",
       " 'v',\n",
       " 'e',\n",
       " 's',\n",
       " 'le',\n",
       " 'e_',\n",
       " '_t',\n",
       " 'te',\n",
       " 'em',\n",
       " 'mp',\n",
       " 'ps',\n",
       " 's_',\n",
       " '_e',\n",
       " 'es',\n",
       " 'st',\n",
       " 't_',\n",
       " '_u',\n",
       " 'un',\n",
       " 'n_',\n",
       " '_g',\n",
       " 'gr',\n",
       " 'ra',\n",
       " 'an',\n",
       " 'nd',\n",
       " 'd_',\n",
       " '_m',\n",
       " 'ma',\n",
       " 'aî',\n",
       " 'ît',\n",
       " 'tr',\n",
       " 're',\n",
       " 'e_',\n",
       " '_d',\n",
       " 'di',\n",
       " 'it',\n",
       " 't_',\n",
       " '_o',\n",
       " 'on',\n",
       " 'n_',\n",
       " '_l',\n",
       " 'le',\n",
       " 'e_',\n",
       " '_m',\n",
       " 'ma',\n",
       " 'al',\n",
       " 'lh',\n",
       " 'he',\n",
       " 'eu',\n",
       " 'ur',\n",
       " 'r_',\n",
       " '_e',\n",
       " 'es',\n",
       " 'st',\n",
       " 't_',\n",
       " '_q',\n",
       " 'qu',\n",
       " \"u'\",\n",
       " \"'i\",\n",
       " 'il',\n",
       " 'l_',\n",
       " '_t',\n",
       " 'tu',\n",
       " 'ue',\n",
       " 'e_',\n",
       " '_s',\n",
       " 'se',\n",
       " 'es',\n",
       " 's_',\n",
       " '_é',\n",
       " 'él',\n",
       " 'lè',\n",
       " 'èv',\n",
       " 've',\n",
       " 'es',\n",
       " 'le_',\n",
       " '_te',\n",
       " 'tem',\n",
       " 'emp',\n",
       " 'mps',\n",
       " 'ps_',\n",
       " '_es',\n",
       " 'est',\n",
       " 'st_',\n",
       " '_un',\n",
       " 'un_',\n",
       " '_gr',\n",
       " 'gra',\n",
       " 'ran',\n",
       " 'and',\n",
       " 'nd_',\n",
       " '_ma',\n",
       " 'maî',\n",
       " 'aît',\n",
       " 'îtr',\n",
       " 'tre',\n",
       " 're_',\n",
       " '_di',\n",
       " 'dit',\n",
       " 'it_',\n",
       " '_on',\n",
       " 'on_',\n",
       " '_le',\n",
       " 'le_',\n",
       " '_ma',\n",
       " 'mal',\n",
       " 'alh',\n",
       " 'lhe',\n",
       " 'heu',\n",
       " 'eur',\n",
       " 'ur_',\n",
       " '_es',\n",
       " 'est',\n",
       " 'st_',\n",
       " '_qu',\n",
       " \"qu'\",\n",
       " \"u'i\",\n",
       " \"'il\",\n",
       " 'il_',\n",
       " '_tu',\n",
       " 'tue',\n",
       " 'ue_',\n",
       " '_se',\n",
       " 'ses',\n",
       " 'es_',\n",
       " '_él',\n",
       " 'élè',\n",
       " 'lèv',\n",
       " 'ève',\n",
       " 'ves',\n",
       " '_tem',\n",
       " 'temp',\n",
       " 'emps',\n",
       " 'mps_',\n",
       " '_est',\n",
       " 'est_',\n",
       " '_un_',\n",
       " '_gra',\n",
       " 'gran',\n",
       " 'rand',\n",
       " 'and_',\n",
       " '_maî',\n",
       " 'maît',\n",
       " 'aîtr',\n",
       " 'ître',\n",
       " 'tre_',\n",
       " '_dit',\n",
       " 'dit_',\n",
       " '_on_',\n",
       " '_le_',\n",
       " '_mal',\n",
       " 'malh',\n",
       " 'alhe',\n",
       " 'lheu',\n",
       " 'heur',\n",
       " 'eur_',\n",
       " '_est',\n",
       " 'est_',\n",
       " \"_qu'\",\n",
       " \"qu'i\",\n",
       " \"u'il\",\n",
       " \"'il_\",\n",
       " '_tue',\n",
       " 'tue_',\n",
       " '_ses',\n",
       " 'ses_',\n",
       " '_élè',\n",
       " 'élèv',\n",
       " 'lève',\n",
       " 'èves']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def ngram_extractor(sent):\n",
    "    return [''.join(ng) for ng in everygrams(sent.replace(' ', '_ _'), 1, 4) \n",
    "            if ' ' not in ng and '\\n' not in ng and ng != ('_',)]\n",
    "\n",
    "ngram_extractor(s_clean)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}