1_text_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center><h1>Text Preprocessing</h1></center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text preprocessing is one of the important step while working on text. \n",
    "## We have to spend more time in cleaning the data. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import nltk\n",
    "import string\n",
    "import itertools\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = PorterStemmer()\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Remove URL's from the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_url_from_text(text):\n",
    "    text = re.sub(r\"http\\S+\", \"\", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Remove Email id from the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_email_from_text(text):\n",
    "    text = re.sub(r'[\\w\\.-]+@[\\w\\.-]+', ' ', text, flags=re.MULTILINE)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Remove extra characters from the text.\n",
    "`Example: sooooo -> so`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_extra_char_from_words(text):\n",
    "    text = ''.join(''.join(s)[:2] for _, s in itertools.groupby(text))\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Remove symbols  from the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_symbols(text):\n",
    "    text = re.sub(r'[^A-Za-z\\s]',r'',text)\n",
    "    text = re.sub(r'\\n',r'',text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Tokenize the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_text(text):\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Convert text to lower case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_to_lower_case(tokens):\n",
    "    tokens = [word.lower() for word in tokens]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Remove punctuation from the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_punctuation(tokens):\n",
    "    table = str.maketrans('', '', string.punctuation)\n",
    "    stripped = [w.translate(table) for w in tokens]\n",
    "    tokens = [word for word in stripped if word.isalpha()]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Remove stop words from the text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(tokens):\n",
    "    stop_words = stopwords.words('english')\n",
    "    token_list = []\n",
    "    for word in tokens:\n",
    "        if not word in stop_words:\n",
    "            token_list.append(word)\n",
    "    return token_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Stem the words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem_words(tokens):\n",
    "    tokens = [stemmer.stem(token) for token in tokens]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. Lemmatize words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_words(tokens):\n",
    "    tokens = [lemmatizer.lemmatize(token, pos='v') for token in tokens]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use the above functions to preprocess text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    text = remove_url_from_text(text)\n",
    "    text = remove_email_from_text(text)\n",
    "    text = remove_extra_char_from_words(text)\n",
    "    text = remove_symbols(text)\n",
    "    tokens = tokenize_text(text)\n",
    "    tokens = text_to_lower_case(tokens)\n",
    "    tokens = remove_punctuation(tokens)\n",
    "    tokens = remove_stopwords(tokens)\n",
    "    tokens = stem_words(tokens)\n",
    "    tokens = lemmatize_words(tokens)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['gmail', 'id', 'url', 'websit']\n"
     ]
    }
   ],
   "source": [
    "input_text = 'My gmail id is google@gmail.com. The url of my website is https://google.com.'\n",
    "tokens = preprocess_text(input_text)\n",
    "print(tokens)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}