text_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Processing\n",
    "\n",
    "## Capturing Text Data\n",
    "\n",
    "### Plain Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Read in a plain text file\n",
    "with open(os.path.join(\"data\", \"hieroglyph.txt\"), \"r\") as f:\n",
    "    text = f.read()\n",
    "    print(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tabular Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Extract text column from a dataframe\n",
    "df = pd.read_csv(os.path.join(\"data\", \"news.csv\"))\n",
    "df.head()[['publisher', 'title']]\n",
    "\n",
    "# Convert text column to lowercase\n",
    "df['title'] = df['title'].str.lower()\n",
    "df.head()[['publisher', 'title']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Online Resource"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "\n",
    "# Fetch data from a REST API\n",
    "r = requests.get(\n",
    "    \"https://quotes.rest/qod.json\")\n",
    "res = r.json()\n",
    "print(json.dumps(res, indent=4))\n",
    "\n",
    "# Extract relevant object and field\n",
    "q = res[\"contents\"][\"quotes\"][0]\n",
    "print(q[\"quote\"], \"\\n--\", q[\"author\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "# Fetch a web page\n",
    "r = requests.get(\"https://news.ycombinator.com\")\n",
    "print(r.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# Remove HTML tags using RegEx\n",
    "pattern = re.compile(r'<.*?>')  # tags look like <...>\n",
    "print(pattern.sub('', r.text))  # replace them with blank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Remove HTML tags using Beautiful Soup library\n",
    "soup = BeautifulSoup(r.text, \"html5lib\")\n",
    "print(soup.get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Find all articles\n",
    "summaries = soup.find_all(\"tr\", class_=\"athing\")\n",
    "summaries[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract title\n",
    "summaries[0].find(\"a\", class_=\"storylink\").get_text().strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find all articles, extract titles\n",
    "articles = []\n",
    "summaries = soup.find_all(\"tr\", class_=\"athing\")\n",
    "for summary in summaries:\n",
    "    title = summary.find(\"a\", class_=\"storylink\").get_text().strip()\n",
    "    articles.append((title))\n",
    "\n",
    "print(len(articles), \"Article summaries found. Sample:\")\n",
    "print(articles[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Case Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Sample text\n",
    "text = \"The first time you see The Second Renaissance it may look boring. Look at it at least twice and definitely watch part 2. It will change your view of the matrix. Are the human people the ones who started the war ? Is AI a bad thing ?\"\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert to lowercase\n",
    "text = text.lower() \n",
    "print(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Punctuation Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# Remove punctuation characters\n",
    "text = re.sub(r\"[^a-zA-Z0-9]\", \" \", text) \n",
    "print(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split text into tokens (words)\n",
    "words = text.split()\n",
    "print(words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLTK: Natural Language ToolKit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import nltk\n",
    "nltk.data.path.append(os.path.join(os.getcwd(), \"nltk_data\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Another sample text\n",
    "text = \"Dr. Smith graduated from the University of Washington. He later started an analytics firm called Lux, which catered to enterprise customers.\"\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "# Split text into words using NLTK\n",
    "words = word_tokenize(text)\n",
    "print(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "# Split text into sentences\n",
    "sentences = sent_tokenize(text)\n",
    "print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List stop words\n",
    "from nltk.corpus import stopwords\n",
    "print(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset text\n",
    "text = \"The first time you see The Second Renaissance it may look boring. Look at it at least twice and definitely watch part 2. It will change your view of the matrix. Are the human people the ones who started the war ? Is AI a bad thing ?\"\n",
    "\n",
    "# Normalize it\n",
    "text = re.sub(r\"[^a-zA-Z0-9]\", \" \", text.lower())\n",
    "\n",
    "# Tokenize it\n",
    "words = text.split()\n",
    "print(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove stop words\n",
    "words = [w for w in words if w not in stopwords.words(\"english\")]\n",
    "print(words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sentence Parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "\n",
    "# Define a custom grammar\n",
    "my_grammar = nltk.CFG.fromstring(\"\"\"\n",
    "S -> NP VP\n",
    "PP -> P NP\n",
    "NP -> Det N | Det N PP | 'I'\n",
    "VP -> V NP | VP PP\n",
    "Det -> 'an' | 'my'\n",
    "N -> 'elephant' | 'pajamas'\n",
    "V -> 'shot'\n",
    "P -> 'in'\n",
    "\"\"\")\n",
    "parser = nltk.ChartParser(my_grammar)\n",
    "\n",
    "# Parse a sentence\n",
    "sentence = word_tokenize(\"I shot an elephant in my pajamas\")\n",
    "for tree in parser.parse(sentence):\n",
    "    print(tree)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stemming & Lemmatization\n",
    "\n",
    "### Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "# Reduce words to their stems\n",
    "stemmed = [PorterStemmer().stem(w) for w in words]\n",
    "print(stemmed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "# Reduce words to their root form\n",
    "lemmed = [WordNetLemmatizer().lemmatize(w) for w in words]\n",
    "print(lemmed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lemmatize verbs by specifying pos\n",
    "lemmed = [WordNetLemmatizer().lemmatize(w, pos='v') for w in lemmed]\n",
    "print(lemmed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:nlpnd]",
   "language": "python",
   "name": "conda-env-nlpnd-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}