Text_Mining.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /srv/conda/envs/notebook/lib/python3.7/site-packages (3.4.3)\n",
      "Requirement already satisfied: six in /srv/conda/envs/notebook/lib/python3.7/site-packages (from nltk) (1.12.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DM_09_04\n",
    "### Import packages\n",
    "We'll use \"codecs\" for reading the text files, \"re\" (for \"regular expressions\") and \"collections\" for working with tokens, and \"nltk\" (\"natural language toolkit\") in several operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "import codecs\n",
    "import re\n",
    "import copy\n",
    "import collections\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import WordPunctTokenizer\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need some specialized functions from NLTK that are not included by default. It is possible to download just the \"stopwords\" portion but it may be easier to simply download everything in NLTK. Note that this is very time consuming; it took over 30 minutes on my machine."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading collection 'all'\n",
      "[nltk_data]    | \n",
      "[nltk_data]    | Downloading package abc to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/abc.zip.\n",
      "[nltk_data]    | Downloading package alpino to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/alpino.zip.\n",
      "[nltk_data]    | Downloading package biocreative_ppi to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/biocreative_ppi.zip.\n",
      "[nltk_data]    | Downloading package brown to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/brown.zip.\n",
      "[nltk_data]    | Downloading package brown_tei to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/brown_tei.zip.\n",
      "[nltk_data]    | Downloading package cess_cat to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/cess_cat.zip.\n",
      "[nltk_data]    | Downloading package cess_esp to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/cess_esp.zip.\n",
      "[nltk_data]    | Downloading package chat80 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/chat80.zip.\n",
      "[nltk_data]    | Downloading package city_database to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/city_database.zip.\n",
      "[nltk_data]    | Downloading package cmudict to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/cmudict.zip.\n",
      "[nltk_data]    | Downloading package comparative_sentences to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/comparative_sentences.zip.\n",
      "[nltk_data]    | Downloading package comtrans to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package conll2000 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/conll2000.zip.\n",
      "[nltk_data]    | Downloading package conll2002 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/conll2002.zip.\n",
      "[nltk_data]    | Downloading package conll2007 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package crubadan to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/crubadan.zip.\n",
      "[nltk_data]    | Downloading package dependency_treebank to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/dependency_treebank.zip.\n",
      "[nltk_data]    | Downloading package dolch to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/dolch.zip.\n",
      "[nltk_data]    | Downloading package europarl_raw to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/europarl_raw.zip.\n",
      "[nltk_data]    | Downloading package floresta to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/floresta.zip.\n",
      "[nltk_data]    | Downloading package framenet_v15 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/framenet_v15.zip.\n",
      "[nltk_data]    | Downloading package framenet_v17 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/framenet_v17.zip.\n",
      "[nltk_data]    | Downloading package gazetteers to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/gazetteers.zip.\n",
      "[nltk_data]    | Downloading package genesis to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/genesis.zip.\n",
      "[nltk_data]    | Downloading package gutenberg to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/gutenberg.zip.\n",
      "[nltk_data]    | Downloading package ieer to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/ieer.zip.\n",
      "[nltk_data]    | Downloading package inaugural to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/inaugural.zip.\n",
      "[nltk_data]    | Downloading package indian to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/indian.zip.\n",
      "[nltk_data]    | Downloading package jeita to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package kimmo to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/kimmo.zip.\n",
      "[nltk_data]    | Downloading package knbc to /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package lin_thesaurus to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/lin_thesaurus.zip.\n",
      "[nltk_data]    | Downloading package mac_morpho to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/mac_morpho.zip.\n",
      "[nltk_data]    | Downloading package machado to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package masc_tagged to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package moses_sample to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping models/moses_sample.zip.\n",
      "[nltk_data]    | Downloading package movie_reviews to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/movie_reviews.zip.\n",
      "[nltk_data]    | Downloading package names to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/names.zip.\n",
      "[nltk_data]    | Downloading package nombank.1.0 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package nps_chat to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/nps_chat.zip.\n",
      "[nltk_data]    | Downloading package omw to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/omw.zip.\n",
      "[nltk_data]    | Downloading package opinion_lexicon to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/opinion_lexicon.zip.\n",
      "[nltk_data]    | Downloading package paradigms to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/paradigms.zip.\n",
      "[nltk_data]    | Downloading package pil to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/pil.zip.\n",
      "[nltk_data]    | Downloading package pl196x to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/pl196x.zip.\n",
      "[nltk_data]    | Downloading package ppattach to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/ppattach.zip.\n",
      "[nltk_data]    | Downloading package problem_reports to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/problem_reports.zip.\n",
      "[nltk_data]    | Downloading package propbank to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package ptb to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/ptb.zip.\n",
      "[nltk_data]    | Downloading package product_reviews_1 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/product_reviews_1.zip.\n",
      "[nltk_data]    | Downloading package product_reviews_2 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/product_reviews_2.zip.\n",
      "[nltk_data]    | Downloading package pros_cons to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/pros_cons.zip.\n",
      "[nltk_data]    | Downloading package qc to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/qc.zip.\n",
      "[nltk_data]    | Downloading package reuters to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package rte to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/rte.zip.\n",
      "[nltk_data]    | Downloading package semcor to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package senseval to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/senseval.zip.\n",
      "[nltk_data]    | Downloading package sentiwordnet to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/sentiwordnet.zip.\n",
      "[nltk_data]    | Downloading package sentence_polarity to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/sentence_polarity.zip.\n",
      "[nltk_data]    | Downloading package shakespeare to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data]    |   Unzipping corpora/shakespeare.zip.\n",
      "[nltk_data]    | Downloading package sinica_treebank to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/sinica_treebank.zip.\n",
      "[nltk_data]    | Downloading package smultron to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/smultron.zip.\n",
      "[nltk_data]    | Downloading package state_union to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/state_union.zip.\n",
      "[nltk_data]    | Downloading package stopwords to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/stopwords.zip.\n",
      "[nltk_data]    | Downloading package subjectivity to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/subjectivity.zip.\n",
      "[nltk_data]    | Downloading package swadesh to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/swadesh.zip.\n",
      "[nltk_data]    | Downloading package switchboard to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/switchboard.zip.\n",
      "[nltk_data]    | Downloading package timit to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/timit.zip.\n",
      "[nltk_data]    | Downloading package toolbox to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/toolbox.zip.\n",
      "[nltk_data]    | Downloading package treebank to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/treebank.zip.\n",
      "[nltk_data]    | Downloading package twitter_samples to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/twitter_samples.zip.\n",
      "[nltk_data]    | Downloading package udhr to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/udhr.zip.\n",
      "[nltk_data]    | Downloading package udhr2 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/udhr2.zip.\n",
      "[nltk_data]    | Downloading package unicode_samples to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/unicode_samples.zip.\n",
      "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package verbnet to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/verbnet.zip.\n",
      "[nltk_data]    | Downloading package verbnet3 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/verbnet3.zip.\n",
      "[nltk_data]    | Downloading package webtext to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/webtext.zip.\n",
      "[nltk_data]    | Downloading package wordnet to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/wordnet.zip.\n",
      "[nltk_data]    | Downloading package wordnet_ic to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/wordnet_ic.zip.\n",
      "[nltk_data]    | Downloading package words to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/words.zip.\n",
      "[nltk_data]    | Downloading package ycoe to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/ycoe.zip.\n",
      "[nltk_data]    | Downloading package rslp to /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping stemmers/rslp.zip.\n",
      "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n",
      "[nltk_data]    | Downloading package universal_tagset to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping taggers/universal_tagset.zip.\n",
      "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping chunkers/maxent_ne_chunker.zip.\n",
      "[nltk_data]    | Downloading package punkt to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data]    | Downloading package book_grammars to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping grammars/book_grammars.zip.\n",
      "[nltk_data]    | Downloading package sample_grammars to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping grammars/sample_grammars.zip.\n",
      "[nltk_data]    | Downloading package spanish_grammars to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping grammars/spanish_grammars.zip.\n",
      "[nltk_data]    | Downloading package basque_grammars to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping grammars/basque_grammars.zip.\n",
      "[nltk_data]    | Downloading package large_grammars to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping grammars/large_grammars.zip.\n",
      "[nltk_data]    | Downloading package tagsets to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping help/tagsets.zip.\n",
      "[nltk_data]    | Downloading package snowball_data to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping models/bllip_wsj_no_aux.zip.\n",
      "[nltk_data]    | Downloading package word2vec_sample to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping models/word2vec_sample.zip.\n",
      "[nltk_data]    | Downloading package panlex_swadesh to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package mte_teip5 to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/mte_teip5.zip.\n",
      "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
      "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping\n",
      "[nltk_data]    |       taggers/averaged_perceptron_tagger_ru.zip.\n",
      "[nltk_data]    | Downloading package perluniprops to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping misc/perluniprops.zip.\n",
      "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping corpora/nonbreaking_prefixes.zip.\n",
      "[nltk_data]    | Downloading package vader_lexicon to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    | Downloading package porter_test to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping stemmers/porter_test.zip.\n",
      "[nltk_data]    | Downloading package wmt15_eval to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping models/wmt15_eval.zip.\n",
      "[nltk_data]    | Downloading package mwa_ppdb to\n",
      "[nltk_data]    |     /home/jovyan/nltk_data...\n",
      "[nltk_data]    |   Unzipping misc/mwa_ppdb.zip.\n",
      "[nltk_data]    | \n",
      "[nltk_data]  Done downloading collection all\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('all')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the \"stopwords\" package from NLTK."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open(\"JaneEyre.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    jane_eyre = f.read()\n",
    "with codecs.open(\"WutheringHeights.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    wuthering_heights = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process data\n",
    "Check for English stopwords."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "esw = stopwords.words('english')\n",
    "esw.append(\"would\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter tokens (using regular expressions)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_pattern = re.compile(\"^\\w+$\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Create a token counter function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_text_counter(text):\n",
    "    tokens = WordPunctTokenizer().tokenize(PorterStemmer().stem(text))\n",
    "    tokens = list(map(lambda x: x.lower(), tokens))\n",
    "    tokens = [token for token in tokens if re.match(word_pattern, token) and token not in esw]\n",
    "    return collections.Counter(tokens), len(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a function to calculate the absolute frequency and relative frequency of the most common words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_df(counter, size):\n",
    "    abs_freq = np.array([el[1] for el in counter])\n",
    "    rel_freq = abs_freq / size\n",
    "    index = [el[0] for el in counter]\n",
    "    df = pd.DataFrame(data=np.array([abs_freq, rel_freq]).T, index=index, columns=[\"Absolute frequency\", \"Relative frequency\"])\n",
    "    df.index.name = \"Most common words\"\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze individual texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the most common words of _Jane Eyre_. This takes a while. Then display the 10 most common."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Absolute frequency</th>\n",
       "      <th>Relative frequency</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Most common words</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>593.0</td>\n",
       "      <td>0.006789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>said</th>\n",
       "      <td>584.0</td>\n",
       "      <td>0.006686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mr</th>\n",
       "      <td>543.0</td>\n",
       "      <td>0.006217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>could</th>\n",
       "      <td>504.0</td>\n",
       "      <td>0.005770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>like</th>\n",
       "      <td>397.0</td>\n",
       "      <td>0.004545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rochester</th>\n",
       "      <td>366.0</td>\n",
       "      <td>0.004190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>well</th>\n",
       "      <td>348.0</td>\n",
       "      <td>0.003984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jane</th>\n",
       "      <td>342.0</td>\n",
       "      <td>0.003916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>little</th>\n",
       "      <td>341.0</td>\n",
       "      <td>0.003904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sir</th>\n",
       "      <td>315.0</td>\n",
       "      <td>0.003607</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Absolute frequency  Relative frequency\n",
       "Most common words                                        \n",
       "one                             593.0            0.006789\n",
       "said                            584.0            0.006686\n",
       "mr                              543.0            0.006217\n",
       "could                           504.0            0.005770\n",
       "like                            397.0            0.004545\n",
       "rochester                       366.0            0.004190\n",
       "well                            348.0            0.003984\n",
       "jane                            342.0            0.003916\n",
       "little                          341.0            0.003904\n",
       "sir                             315.0            0.003607"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "je_counter, je_size = get_text_counter(jane_eyre)\n",
    "make_df(je_counter.most_common(10), je_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the 1000 most common words of _Jane Eyre_ to CSV."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "je_df = make_df(je_counter.most_common(1000), je_size)\n",
    "je_df.to_csv(\"JE_1000.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the most common words of _Wuthering Heights_. This also takes a while. Display the 10 most common."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Absolute frequency</th>\n",
       "      <th>Relative frequency</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Most common words</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>heathcliff</th>\n",
       "      <td>475.0</td>\n",
       "      <td>0.008735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>linton</th>\n",
       "      <td>404.0</td>\n",
       "      <td>0.007429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>catherine</th>\n",
       "      <td>379.0</td>\n",
       "      <td>0.006970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>said</th>\n",
       "      <td>375.0</td>\n",
       "      <td>0.006896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mr</th>\n",
       "      <td>312.0</td>\n",
       "      <td>0.005738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>290.0</td>\n",
       "      <td>0.005333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>could</th>\n",
       "      <td>279.0</td>\n",
       "      <td>0.005131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>master</th>\n",
       "      <td>205.0</td>\n",
       "      <td>0.003770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shall</th>\n",
       "      <td>191.0</td>\n",
       "      <td>0.003512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>come</th>\n",
       "      <td>190.0</td>\n",
       "      <td>0.003494</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Absolute frequency  Relative frequency\n",
       "Most common words                                        \n",
       "heathcliff                      475.0            0.008735\n",
       "linton                          404.0            0.007429\n",
       "catherine                       379.0            0.006970\n",
       "said                            375.0            0.006896\n",
       "mr                              312.0            0.005738\n",
       "one                             290.0            0.005333\n",
       "could                           279.0            0.005131\n",
       "master                          205.0            0.003770\n",
       "shall                           191.0            0.003512\n",
       "come                            190.0            0.003494"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wh_counter, wh_size = get_text_counter(wuthering_heights)\n",
    "make_df(wh_counter.most_common(10), wh_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the 1000 most common words of _Wuthering Heights_ to CSV."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "wh_df = make_df(wh_counter.most_common(1000), wh_size)\n",
    "wh_df.to_csv(\"WH_1000.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compare texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the most common words across the two documents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_counter = wh_counter + je_counter\n",
    "all_df = make_df(wh_counter.most_common(1000), 1)\n",
    "most_common_words = all_df.index.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a data frame with the word frequency differences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = []\n",
    "for word in most_common_words:\n",
    "    je_c = je_counter.get(word, 0) / je_size\n",
    "    wh_c = wh_counter.get(word, 0) / wh_size\n",
    "    d = abs(je_c - wh_c)\n",
    "    df_data.append([je_c, wh_c, d])\n",
    "dist_df = pd.DataFrame(data=df_data, index=most_common_words,\n",
    "                       columns=[\"Jane Eyre relative frequency\", \"Wuthering Heights relative frequency\",\n",
    "                                \"Relative frequency difference\"])\n",
    "dist_df.index.name = \"Most common words\"\n",
    "dist_df.sort_values(\"Relative frequency difference\", ascending=False, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Display the most distinctive words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Jane Eyre relative frequency</th>\n",
       "      <th>Wuthering Heights relative frequency</th>\n",
       "      <th>Relative frequency difference</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Most common words</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>heathcliff</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008735</td>\n",
       "      <td>0.008735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>linton</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007429</td>\n",
       "      <td>0.007429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>catherine</th>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.006970</td>\n",
       "      <td>0.006958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hareton</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003292</td>\n",
       "      <td>0.003292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sir</th>\n",
       "      <td>0.003607</td>\n",
       "      <td>0.000791</td>\n",
       "      <td>0.002816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>master</th>\n",
       "      <td>0.001133</td>\n",
       "      <td>0.003770</td>\n",
       "      <td>0.002636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>joseph</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002575</td>\n",
       "      <td>0.002575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>earnshaw</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002372</td>\n",
       "      <td>0.002372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cathy</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002280</td>\n",
       "      <td>0.002280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>edgar</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002133</td>\n",
       "      <td>0.002133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   Jane Eyre relative frequency  \\\n",
       "Most common words                                 \n",
       "heathcliff                             0.000000   \n",
       "linton                                 0.000000   \n",
       "catherine                              0.000011   \n",
       "hareton                                0.000000   \n",
       "sir                                    0.003607   \n",
       "master                                 0.001133   \n",
       "joseph                                 0.000000   \n",
       "earnshaw                               0.000000   \n",
       "cathy                                  0.000000   \n",
       "edgar                                  0.000000   \n",
       "\n",
       "                   Wuthering Heights relative frequency  \\\n",
       "Most common words                                         \n",
       "heathcliff                                     0.008735   \n",
       "linton                                         0.007429   \n",
       "catherine                                      0.006970   \n",
       "hareton                                        0.003292   \n",
       "sir                                            0.000791   \n",
       "master                                         0.003770   \n",
       "joseph                                         0.002575   \n",
       "earnshaw                                       0.002372   \n",
       "cathy                                          0.002280   \n",
       "edgar                                          0.002133   \n",
       "\n",
       "                   Relative frequency difference  \n",
       "Most common words                                 \n",
       "heathcliff                              0.008735  \n",
       "linton                                  0.007429  \n",
       "catherine                               0.006958  \n",
       "hareton                                 0.003292  \n",
       "sir                                     0.002816  \n",
       "master                                  0.002636  \n",
       "joseph                                  0.002575  \n",
       "earnshaw                                0.002372  \n",
       "cathy                                   0.002280  \n",
       "edgar                                   0.002133  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist_df.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the full list of distinctive words to a CSV entitled \"bronte.csv.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_df.to_csv(\"bronte.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}