ted_clean_explore.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP on Ted Talk transcropts using nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-02T16:10:40.169482Z",
     "start_time": "2017-11-02T16:10:40.166522Z"
    }
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-15T19:38:48.243042Z",
     "start_time": "2017-11-15T19:38:48.173542Z"
    }
   },
   "outputs": [],
   "source": [
    "import nltk, re, pickle, os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "from textblob import TextBlob\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize, wordpunct_tokenize, MWETokenizer\n",
    "from nltk.stem import porter, WordNetLemmatizer\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.util import ngrams\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation,  TruncatedSVD \n",
    "\n",
    "path = '/Volumes/ext200/Dropbox/metis/p4_fletcher/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/ada/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package brown to /Users/ada/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/brown.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt')\n",
    "nltk.download('brown')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /Users/ada/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import our 2 databases to pandas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:02:51.838596Z",
     "start_time": "2017-11-13T20:02:50.971925Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>transcript</th>\n",
       "      <th>url</th>\n",
       "      <th>comments</th>\n",
       "      <th>description</th>\n",
       "      <th>duration</th>\n",
       "      <th>event</th>\n",
       "      <th>film_date</th>\n",
       "      <th>languages</th>\n",
       "      <th>main_speaker</th>\n",
       "      <th>name</th>\n",
       "      <th>num_speaker</th>\n",
       "      <th>published_date</th>\n",
       "      <th>ratings</th>\n",
       "      <th>related_talks</th>\n",
       "      <th>speaker_occupation</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "      <th>views</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Good morning. How are you?(Laughter)It's been ...</td>\n",
       "      <td>https://www.ted.com/talks/ken_robinson_says_sc...</td>\n",
       "      <td>4553</td>\n",
       "      <td>Sir Ken Robinson makes an entertaining and pro...</td>\n",
       "      <td>1164</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140825600</td>\n",
       "      <td>60</td>\n",
       "      <td>Ken Robinson</td>\n",
       "      <td>Ken Robinson: Do schools kill creativity?</td>\n",
       "      <td>1</td>\n",
       "      <td>1151367060</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 19645}, {...</td>\n",
       "      <td>[{'id': 865, 'hero': 'https://pe.tedcdn.com/im...</td>\n",
       "      <td>Author/educator</td>\n",
       "      <td>['children', 'creativity', 'culture', 'dance',...</td>\n",
       "      <td>Do schools kill creativity?</td>\n",
       "      <td>47227110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Thank you so much, Chris. And it's truly a gre...</td>\n",
       "      <td>https://www.ted.com/talks/al_gore_on_averting_...</td>\n",
       "      <td>265</td>\n",
       "      <td>With the same humor and humanity he exuded in ...</td>\n",
       "      <td>977</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140825600</td>\n",
       "      <td>43</td>\n",
       "      <td>Al Gore</td>\n",
       "      <td>Al Gore: Averting the climate crisis</td>\n",
       "      <td>1</td>\n",
       "      <td>1151367060</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 544}, {'i...</td>\n",
       "      <td>[{'id': 243, 'hero': 'https://pe.tedcdn.com/im...</td>\n",
       "      <td>Climate advocate</td>\n",
       "      <td>['alternative energy', 'cars', 'climate change...</td>\n",
       "      <td>Averting the climate crisis</td>\n",
       "      <td>3200520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(Music: \"The Sound of Silence,\" Simon &amp; Garfun...</td>\n",
       "      <td>https://www.ted.com/talks/david_pogue_says_sim...</td>\n",
       "      <td>124</td>\n",
       "      <td>New York Times columnist David Pogue takes aim...</td>\n",
       "      <td>1286</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140739200</td>\n",
       "      <td>26</td>\n",
       "      <td>David Pogue</td>\n",
       "      <td>David Pogue: Simplicity sells</td>\n",
       "      <td>1</td>\n",
       "      <td>1151367060</td>\n",
       "      <td>[{'id': 7, 'name': 'Funny', 'count': 964}, {'i...</td>\n",
       "      <td>[{'id': 1725, 'hero': 'https://pe.tedcdn.com/i...</td>\n",
       "      <td>Technology columnist</td>\n",
       "      <td>['computers', 'entertainment', 'interface desi...</td>\n",
       "      <td>Simplicity sells</td>\n",
       "      <td>1636292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>If you're here today — and I'm very happy that...</td>\n",
       "      <td>https://www.ted.com/talks/majora_carter_s_tale...</td>\n",
       "      <td>200</td>\n",
       "      <td>In an emotionally charged talk, MacArthur-winn...</td>\n",
       "      <td>1116</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140912000</td>\n",
       "      <td>35</td>\n",
       "      <td>Majora Carter</td>\n",
       "      <td>Majora Carter: Greening the ghetto</td>\n",
       "      <td>1</td>\n",
       "      <td>1151367060</td>\n",
       "      <td>[{'id': 3, 'name': 'Courageous', 'count': 760}...</td>\n",
       "      <td>[{'id': 1041, 'hero': 'https://pe.tedcdn.com/i...</td>\n",
       "      <td>Activist for environmental justice</td>\n",
       "      <td>['MacArthur grant', 'activism', 'business', 'c...</td>\n",
       "      <td>Greening the ghetto</td>\n",
       "      <td>1697550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>About 10 years ago, I took on the task to teac...</td>\n",
       "      <td>https://www.ted.com/talks/hans_rosling_shows_t...</td>\n",
       "      <td>593</td>\n",
       "      <td>You've never seen data presented like this. Wi...</td>\n",
       "      <td>1190</td>\n",
       "      <td>TED2006</td>\n",
       "      <td>1140566400</td>\n",
       "      <td>48</td>\n",
       "      <td>Hans Rosling</td>\n",
       "      <td>Hans Rosling: The best stats you've ever seen</td>\n",
       "      <td>1</td>\n",
       "      <td>1151440680</td>\n",
       "      <td>[{'id': 9, 'name': 'Ingenious', 'count': 3202}...</td>\n",
       "      <td>[{'id': 2056, 'hero': 'https://pe.tedcdn.com/i...</td>\n",
       "      <td>Global health expert; data visionary</td>\n",
       "      <td>['Africa', 'Asia', 'Google', 'demo', 'economic...</td>\n",
       "      <td>The best stats you've ever seen</td>\n",
       "      <td>12005869</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          transcript  \\\n",
       "0  Good morning. How are you?(Laughter)It's been ...   \n",
       "1  Thank you so much, Chris. And it's truly a gre...   \n",
       "2  (Music: \"The Sound of Silence,\" Simon & Garfun...   \n",
       "3  If you're here today — and I'm very happy that...   \n",
       "4  About 10 years ago, I took on the task to teac...   \n",
       "\n",
       "                                                 url  comments  \\\n",
       "0  https://www.ted.com/talks/ken_robinson_says_sc...      4553   \n",
       "1  https://www.ted.com/talks/al_gore_on_averting_...       265   \n",
       "2  https://www.ted.com/talks/david_pogue_says_sim...       124   \n",
       "3  https://www.ted.com/talks/majora_carter_s_tale...       200   \n",
       "4  https://www.ted.com/talks/hans_rosling_shows_t...       593   \n",
       "\n",
       "                                         description  duration    event  \\\n",
       "0  Sir Ken Robinson makes an entertaining and pro...      1164  TED2006   \n",
       "1  With the same humor and humanity he exuded in ...       977  TED2006   \n",
       "2  New York Times columnist David Pogue takes aim...      1286  TED2006   \n",
       "3  In an emotionally charged talk, MacArthur-winn...      1116  TED2006   \n",
       "4  You've never seen data presented like this. Wi...      1190  TED2006   \n",
       "\n",
       "    film_date  languages   main_speaker  \\\n",
       "0  1140825600         60   Ken Robinson   \n",
       "1  1140825600         43        Al Gore   \n",
       "2  1140739200         26    David Pogue   \n",
       "3  1140912000         35  Majora Carter   \n",
       "4  1140566400         48   Hans Rosling   \n",
       "\n",
       "                                            name  num_speaker  published_date  \\\n",
       "0      Ken Robinson: Do schools kill creativity?            1      1151367060   \n",
       "1           Al Gore: Averting the climate crisis            1      1151367060   \n",
       "2                  David Pogue: Simplicity sells            1      1151367060   \n",
       "3             Majora Carter: Greening the ghetto            1      1151367060   \n",
       "4  Hans Rosling: The best stats you've ever seen            1      1151440680   \n",
       "\n",
       "                                             ratings  \\\n",
       "0  [{'id': 7, 'name': 'Funny', 'count': 19645}, {...   \n",
       "1  [{'id': 7, 'name': 'Funny', 'count': 544}, {'i...   \n",
       "2  [{'id': 7, 'name': 'Funny', 'count': 964}, {'i...   \n",
       "3  [{'id': 3, 'name': 'Courageous', 'count': 760}...   \n",
       "4  [{'id': 9, 'name': 'Ingenious', 'count': 3202}...   \n",
       "\n",
       "                                       related_talks  \\\n",
       "0  [{'id': 865, 'hero': 'https://pe.tedcdn.com/im...   \n",
       "1  [{'id': 243, 'hero': 'https://pe.tedcdn.com/im...   \n",
       "2  [{'id': 1725, 'hero': 'https://pe.tedcdn.com/i...   \n",
       "3  [{'id': 1041, 'hero': 'https://pe.tedcdn.com/i...   \n",
       "4  [{'id': 2056, 'hero': 'https://pe.tedcdn.com/i...   \n",
       "\n",
       "                     speaker_occupation  \\\n",
       "0                       Author/educator   \n",
       "1                      Climate advocate   \n",
       "2                  Technology columnist   \n",
       "3    Activist for environmental justice   \n",
       "4  Global health expert; data visionary   \n",
       "\n",
       "                                                tags  \\\n",
       "0  ['children', 'creativity', 'culture', 'dance',...   \n",
       "1  ['alternative energy', 'cars', 'climate change...   \n",
       "2  ['computers', 'entertainment', 'interface desi...   \n",
       "3  ['MacArthur grant', 'activism', 'business', 'c...   \n",
       "4  ['Africa', 'Asia', 'Google', 'demo', 'economic...   \n",
       "\n",
       "                             title     views  \n",
       "0      Do schools kill creativity?  47227110  \n",
       "1      Averting the climate crisis   3200520  \n",
       "2                 Simplicity sells   1636292  \n",
       "3              Greening the ghetto   1697550  \n",
       "4  The best stats you've ever seen  12005869  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ted_main = pd.read_csv(path + 'ted-talks/ted_main.csv')\n",
    "ted_trans = pd.read_csv(path +'ted-talks/transcripts.csv')    \n",
    "\n",
    "ted_all = pd.merge(ted_trans,right=ted_main,on='url')\n",
    "ted_all.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:02:57.311843Z",
     "start_time": "2017-11-13T20:02:56.956018Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(path + 'pick/ted_all.pkl', 'wb') as picklefile:\n",
    "    pickle.dump(ted_all, picklefile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# add an id col (of the index) for later and for iterating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:02:59.876941Z",
     "start_time": "2017-11-13T20:02:59.871517Z"
    }
   },
   "outputs": [],
   "source": [
    "ted_all['id'] = ted_all.index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# keep only the transcripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:03:01.447949Z",
     "start_time": "2017-11-13T20:03:01.444381Z"
    }
   },
   "outputs": [],
   "source": [
    "talks = ted_all['transcript']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T23:11:40.289422Z",
     "start_time": "2017-11-13T23:11:40.281867Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Good morning. How are you?(Laughter)It's been great, hasn't it? I've been blown away by the whole thing. In fact, I'm leaving.(Laughter)There have been three themes running through the conference which are relevant to what I want to talk about. One is the extraordinary evidence of human creativity in all of the presentations that we've had and in all of the people here. Just the variety of it and the range of it. The second is that it's put us in a place where we have no idea what's going to hap\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "talks[0][0:500]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Non-speech sounds, events\n",
    "While looking at initial ngrams, I notice that we get a lot of \"thank you  applause\". So, I started looking at all the non-word behavior that is transcribed (see below). Luckily, they put all of the speaker's parenthetical comments, and some titles of songs(?) played into brackets [ ] and all of the audience sounds, viedos, music, etc in parenthases. \n",
    "\n",
    "So, it is safe to first go and take out everything that is in parentheses before we even tokenize so that we can just look at speech.  \n",
    "\n",
    "It would be interesting to collect these and keep a count in the main matrix, especially for things like 'laughter' or applause or multimedia (present/not present) in making recommendations or calculating the popularity of a talk. \n",
    "\n",
    "### Examples of parentheticals (non-speech sounds)\n",
    "\n",
    "(Applause)(Applause ends)(Pre-recorded applause)(Pre-recorded applause and cheering)(Audience cheers)(Laughter)(Shouting)(Mock sob)(Breathes in)(Baby cooing)(Video)(Singing)(Heroic music)(Loud music)(Music)(Music ends)(Plays notes)(Sighs)(Clears throat)(Whispering)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:04:46.891465Z",
     "start_time": "2017-11-13T20:04:46.883084Z"
    }
   },
   "outputs": [],
   "source": [
    "# number of transcripts you want to analyze\n",
    "fileids = range(0,51)\n",
    "\n",
    "# remove parethetical non-speech sounds from text\n",
    "clean_parens_docs= [re.sub(r'\\([^)]*\\)', ' ', talks[fileid]) \\\n",
    "                    for fileid in fileids]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize (split) into sentences\n",
    "\n",
    "Below are multiple methods for sentence tokenization\n",
    "\n",
    "1. the built in text blob.sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:04:57.062118Z",
     "start_time": "2017-11-13T20:04:56.432363Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_sents = [TextBlob(clean_parens_docs[fileid])\n",
    "             .sentences for fileid in fileids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:04:58.687362Z",
     "start_time": "2017-11-13T20:04:58.681156Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Sentence(\"Good morning.\"), Sentence(\"How are you?\"), Sentence(\"It's been great, hasn't it?\"), Sentence(\"I've been blown away by the whole thing.\"), Sentence(\"In fact, I'm leaving.\")]\n"
     ]
    }
   ],
   "source": [
    "print(doc_sents[0][0:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### another methods for sentence tokenization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:10.495545Z",
     "start_time": "2017-11-13T20:05:10.096737Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good morning.\n",
      "-----\n",
      "How are you?\n",
      "-----\n",
      "It's been great, hasn't\n"
     ]
    }
   ],
   "source": [
    "doc_sents1 = [sent_tokenize(clean_parens_docs[fileid]) for fileid in fileids]\n",
    "print('\\n-----\\n'.join(sent_tokenize(clean_parens_docs[0][0:50])))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word tokenization \n",
    "\n",
    "1. text blob.words "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:19.438003Z",
     "start_time": "2017-11-13T20:05:16.867702Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_words = [TextBlob(str(doc_sents1[fileid]))\n",
    "              .words for fileid in fileids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:21.580540Z",
     "start_time": "2017-11-13T20:05:21.483916Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'Good\n",
      "-----\n",
      "morning\n",
      "-----\n",
      "'How\n",
      "-----\n",
      "are\n",
      "-----\n",
      "you\n"
     ]
    }
   ],
   "source": [
    "print('\\n-----\\n'.join(TextBlob(str(doc_sents1[0][0:2])).words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## this is another tokenizer. the one below is leaving in punctuation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:27.220000Z",
     "start_time": "2017-11-13T20:05:25.779032Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_words1 = [word_tokenize(clean_parens_docs[fileid]) \\\n",
    "             for fileid in fileids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:28.334009Z",
     "start_time": "2017-11-13T20:05:28.286184Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good\n",
      "-----\n",
      "morning\n",
      "-----\n",
      ".\n",
      "-----\n",
      "How\n",
      "-----\n",
      "ar\n"
     ]
    }
   ],
   "source": [
    "print('\\n-----\\n'.join(word_tokenize(clean_parens_docs[0][0:20])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### the wordpunct version takes care of all of the punctuaiton nicely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:30.057043Z",
     "start_time": "2017-11-13T20:05:29.977823Z"
    }
   },
   "outputs": [],
   "source": [
    "doc_words2 = [wordpunct_tokenize(clean_parens_docs[fileid]) \\\n",
    "             for fileid in fileids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:31.728004Z",
     "start_time": "2017-11-13T20:05:31.716481Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good\n",
      "-----\n",
      "morning\n",
      "-----\n",
      ".\n",
      "-----\n",
      "How\n",
      "-----\n",
      "ar\n"
     ]
    }
   ],
   "source": [
    "print('\\n-----\\n'.join(wordpunct_tokenize(clean_parens_docs[0][0:20])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# text blob allows us to pull out interesting things"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:34.044765Z",
     "start_time": "2017-11-13T20:05:34.017700Z"
    }
   },
   "outputs": [],
   "source": [
    "talks_blob = [TextBlob(clean_parens_docs[fileid]) for fileid in fileids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:39.199827Z",
     "start_time": "2017-11-13T20:05:35.041497Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "good morning\n",
      "-----\n",
      "whole thing\n",
      "-----\n",
      "extraordinary evidence\n",
      "-----\n",
      "human creativity\n"
     ]
    }
   ],
   "source": [
    "# pulls all the nouns and all the things that are associated with it \n",
    "print('\\n-----\\n'.join(talks_blob[0][0:500].noun_phrases))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:05:42.779247Z",
     "start_time": "2017-11-13T20:05:42.706012Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Truthfully, what happens is, as children grow up, we start to educate them progressively from the waist up.\n",
      "\n",
      "Sentiment(polarity=0.5, subjectivity=0.5)\n"
     ]
    }
   ],
   "source": [
    "print(talks_blob[0].sentences[129] + '\\n')\n",
    "print(talks_blob[0].sentences[129].sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T22:53:26.170376Z",
     "start_time": "2017-11-13T22:53:26.157842Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# def get_count(item):\n",
    "#     return item[1]\n",
    "\n",
    "# for word, count in sorted(talks_blob[1]\n",
    "#                           .word_counts\n",
    "#                           .items(), key=get_count, reverse=True):\n",
    "#     print(\"%15s %i\" % (word, count))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lemmatizer\n",
    "\n",
    "an alternative method for getting the word roots\n",
    "This one appears to be more conservative and also more 'correct' in that it will replace the ending with the correct letters instead of chopping it off.  i.e. children -> child,   capacities -> capacity, but also, unpredictability -> unpredictability . \n",
    "\n",
    "Thus, we run this one first, and then do the stemming on that result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:07:23.996217Z",
     "start_time": "2017-11-13T20:06:44.952044Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good Good\n",
      "morning morning\n",
      "How How\n",
      "are are\n",
      "you you\n",
      "It It\n",
      "'s 's\n",
      "been been\n",
      "great great\n",
      "ha has\n"
     ]
    }
   ],
   "source": [
    "lemmizer = WordNetLemmatizer()\n",
    "\n",
    "for fileid in fileids[0:1]: \n",
    "    doc = TextBlob(clean_parens_docs[fileid]).words\n",
    "    for w in doc[0:10]:\n",
    "        print(lemmizer.lemmatize(w), w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# clean up text: stemming\n",
    "\n",
    "print out the original word next to the stemmed word to check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:08:03.412947Z",
     "start_time": "2017-11-13T20:07:23.998780Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "good Good\n",
      "morn morning\n",
      "how How\n",
      "are are\n",
      "you you\n",
      "it It\n",
      "'s 's\n",
      "been been\n",
      "great great\n",
      "ha has\n"
     ]
    }
   ],
   "source": [
    "stemmer = nltk.stem.porter.PorterStemmer()\n",
    "\n",
    "for fileid in fileids[0:1]: \n",
    "    doc = TextBlob(clean_parens_docs[fileid]).words\n",
    "    for w in doc[0:10]:\n",
    "        print(stemmer.stem(w.lower()),w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## now stem using the tokenized version that separated punctuation better"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "good Good\n",
      "morn morning\n",
      ". .\n",
      "how How\n",
      "are are\n",
      "you you\n",
      "? ?\n",
      "it It\n",
      "' '\n",
      "s s\n"
     ]
    }
   ],
   "source": [
    "stemmer = nltk.stem.porter.PorterStemmer()\n",
    "\n",
    "for fileid in fileids[0:1]: \n",
    "    for w in doc_words2[fileid][0:10]:\n",
    "        print(stemmer.stem(w.lower()),w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploration OVER: Now, we will clean it up in a nice function based on the best methods from above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a function to clean one document only\n",
    "\n",
    "def clean_text_onedoc(text):\n",
    "\n",
    "    lemmizer = WordNetLemmatizer()\n",
    "    stop = stopwords.words('english')\n",
    "    stop += ['.', ',',':','...','!\"','?\"', \"'\", '\"',' - ',' — ',',\"','.\"','!', ';',\\\n",
    "             '.\\'\"','[',']','—',\".\\'\", 'ok','okay','yeah','ya','stuff', ' 000 ',' em ','get','got',\\\n",
    "             ' oh ','la','was','wa','?','like','go',' le ',' ca ',' I ',\" ? \",\"s\", \" t \",\"ve\",\"re\"]\n",
    "    \n",
    "    for word in wordpunct_tokenize(text): \n",
    "        cleaned = []\n",
    "        if word.lower() not in stop:\n",
    "            keepw = lemmizer.lemmatize(word)\n",
    "            if keepw.lower not in stop:\n",
    "                cleaned.append(keepw.lower())\n",
    "                \n",
    "    return cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T23:03:29.257095Z",
     "start_time": "2017-11-13T23:03:29.145774Z"
    }
   },
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    \n",
    "    \"\"\" \n",
    "    Takes in a corpus of documents and cleans. ONly works with multiple docs for now\n",
    "    \n",
    "    1. remove parentheticals\n",
    "    2. tokenize into words using wordpunct\n",
    "    3. lowercase and remove stop words\n",
    "    4. lemmatize \n",
    "    5. lowercase and remove stop words\n",
    "    \n",
    "    \n",
    "    OUT: cleaned text = a list (documents) of lists (cleaned word in each doc)\n",
    "    \"\"\"\n",
    "\n",
    "    lemmizer = WordNetLemmatizer()\n",
    "    #stemmer = porter.PorterStemmer()\n",
    "\n",
    "    stop = stopwords.words('english')\n",
    "    stop += ['.', ',',':','...','!\"','?\"', \"'\", '\"',' - ',' — ',',\"','.\"','!', ';','♫♫','♫',\\\n",
    "             '.\\'\"','[',']','—',\".\\'\", 'ok','okay','yeah','ya','stuff', ' 000 ',' em ',\\\n",
    "             ' oh ','thank','thanks','la','was','wa','?','like','go',' le ',' ca ',' I ',\" ? \",\"s\", \" t \",\"ve\",\"re\"]\n",
    "    #stop = set(stop)\n",
    "\n",
    "    cleaned_text = []\n",
    "    \n",
    "    for post in text:\n",
    "        cleaned_words = []\n",
    "        \n",
    "        # remove parentheticals\n",
    "        clean_parens = re.sub(r'\\([^)]*\\)', ' ', post)\n",
    "        \n",
    "        # tokenize into words\n",
    "        for word  in wordpunct_tokenize(clean_parens):  \n",
    "            \n",
    "            # lowercase and throw out any words in stop words\n",
    "            if word.lower() not in stop:\n",
    "            \n",
    "                # lemmatize  to roots\n",
    "                low_word = lemmizer.lemmatize(word)  \n",
    "\n",
    "                # stem and lowercase ( an alternative to lemmatize)\n",
    "                #low_word = stemmer.stem(root.lower())  \n",
    "            \n",
    "                # keep if not in stopwords (yes, again)\n",
    "                if low_word.lower() not in stop: \n",
    "                    \n",
    "                    # put into a list of words for each document\n",
    "                    cleaned_words.append(low_word.lower())\n",
    "        \n",
    "        # keep corpus of cleaned words for each document    \n",
    "        cleaned_text.append(' '.join(cleaned_words))\n",
    "    \n",
    "    return cleaned_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T23:10:23.734602Z",
     "start_time": "2017-11-13T23:09:47.312411Z"
    }
   },
   "outputs": [],
   "source": [
    "cleaned_talks = clean_text(talks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T20:44:35.528419Z",
     "start_time": "2017-11-13T20:44:35.373564Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(path + 'cleaned_talks.pkl', 'wb') as picklefile:\n",
    "    pickle.dump(cleaned_talks, picklefile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T23:10:41.909418Z",
     "start_time": "2017-11-13T23:10:41.903706Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'good morning great blown away whole thing fact leaving three theme running conference relevant want talk one extraordinary evidence human creativity presentation people variety range second put u place idea going happen term future idea may play interest education actually find everybody interest ed'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_talks[0][0:300]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Top n-grams, your favorite breakfast cereal\n",
    "\n",
    "Note that these tri-grams are not very informative aside from new york city and X year ago, which will still get picked up in the bi-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T23:20:59.751454Z",
     "start_time": "2017-11-13T23:15:43.692648Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       new york city 236\n",
      "        000 year ago 135\n",
      "      new york times 123\n",
      "         10 year ago 118\n",
      "    every single day 109\n",
      "    million year ago 109\n",
      " people around world 101\n",
      "        two year ago 100\n",
      "        world war ii 99\n",
      "       one two three 97\n",
      "     couple year ago 96\n",
      "         20 year ago 83\n",
      "       five year old 78\n",
      "     talk little bit 71\n",
      "      spend lot time 71\n",
      "    every single one 69\n",
      "      three year ago 69\n",
      "        six year old 69\n",
      "  sub saharan africa 68\n",
      "        last 20 year 67\n",
      "     tell little bit 66\n",
      "         12 year old 65\n",
      "       four year old 64\n",
      "         10 000 year 64\n",
      "        last 10 year 64\n",
      "      world around u 63\n",
      "       five year ago 61\n",
      "            da da da 61\n",
      "       let take look 60\n",
      "       four year ago 59\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "from operator import itemgetter\n",
    "\n",
    "counter = Counter()\n",
    "\n",
    "n = 3\n",
    "for doc in cleaned_talks:\n",
    "    words = TextBlob(doc).words\n",
    "    bigrams = ngrams(words, n)\n",
    "    counter += Counter(bigrams)\n",
    "\n",
    "for phrase, count in counter.most_common(30):\n",
    "    print('%20s %i' % (\" \".join(phrase), count))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bi-grams\n",
    "these are much better. Still some useless items like 'one thing' and 'can not'. interesting that we get some little musical notes in here ? i guess the way it was transcribed was using some other encoding for that?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-13T23:56:11.167663Z",
     "start_time": "2017-11-13T23:53:52.643238Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            year ago 2074\n",
      "          little bit 1607\n",
      "            year old 1365\n",
      "       united states 1103\n",
      "           one thing 1041\n",
      "        around world 938\n",
      "            new york 894\n",
      "             can not 877\n",
      "          first time 751\n",
      "           every day 692\n",
      "         many people 656\n",
      "           last year 604\n",
      "        every single 573\n",
      "             one day 559\n",
      "             10 year 541\n",
      "              tell u 521\n",
      "         even though 519\n",
      "      million people 499\n",
      "           come back 492\n",
      "          lot people 485\n",
      "            two year 474\n",
      "           long time 471\n",
      "             20 year 464\n",
      "           would say 464\n",
      "           five year 449\n",
      "      climate change 437\n",
      "          every time 406\n",
      "          year later 405\n",
      "         high school 388\n",
      "           going get 381\n"
     ]
    }
   ],
   "source": [
    "counter = Counter()\n",
    "\n",
    "n = 2\n",
    "for doc in cleaned_talks:\n",
    "    words = TextBlob(doc).words\n",
    "    bigrams = ngrams(words, n)\n",
    "    counter += Counter(bigrams)\n",
    "\n",
    "for phrase, count in counter.most_common(30):\n",
    "    print('%20s %i' % (\" \".join(phrase), count))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorize the data (only)  also look at binomial vecorizer in additino to count vectorizer for other topic modeling methods\n",
    "Using Sklearn algorithms with text data\n",
    "CountVectorizer: Convert a collection of text documents to a matrix of token counts This implementation produces a sparse representation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-10T02:12:53.508282Z",
     "start_time": "2017-11-10T02:11:40.354134Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# CountVectorizer is a class; so `vectorizer` below represents an instance of that object.\n",
    "c_vectorizer = CountVectorizer(ngram_range=(1,3), \n",
    "                             stop_words='english', \n",
    "                             max_df = 0.6, \n",
    "                             max_features=10000)\n",
    "\n",
    "t_vectorizer = TfidfVectorizer(ngram_range=(1, 3),  \n",
    "                                   stop_words='english', \n",
    "                                   token_pattern=\"\\\\b[a-z][a-z]+\\\\b\",\n",
    "                                   lowercase=True,\n",
    "                                   max_df = 0.6)\n",
    "\n",
    "\n",
    "# call `fit` to build the vocabulary\n",
    "c_vectorizer.fit(cleaned_talks)\n",
    "# finally, call `transform` to convert text to a bag of words\n",
    "c_x = c_vectorizer.transform(cleaned_talks)\n",
    "\n",
    "\n",
    "# call `fit` to build the vocabulary\n",
    "t_vectorizer.fit(cleaned_talks)\n",
    "# finally, call `transform` to convert text to a bag of words\n",
    "t_x = t_vectorizer.transform(cleaned_talks)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}