NLP Task-5 Jan 10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task -1. Lexical Source "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1 Stop words in English +1 Language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['au',\n",
       " 'aux',\n",
       " 'avec',\n",
       " 'ce',\n",
       " 'ces',\n",
       " 'dans',\n",
       " 'de',\n",
       " 'des',\n",
       " 'du',\n",
       " 'elle',\n",
       " 'en',\n",
       " 'et',\n",
       " 'eux',\n",
       " 'il',\n",
       " 'ils',\n",
       " 'je',\n",
       " 'la',\n",
       " 'le',\n",
       " 'les',\n",
       " 'leur',\n",
       " 'lui',\n",
       " 'ma',\n",
       " 'mais',\n",
       " 'me',\n",
       " 'même',\n",
       " 'mes',\n",
       " 'moi',\n",
       " 'mon',\n",
       " 'ne',\n",
       " 'nos',\n",
       " 'notre',\n",
       " 'nous',\n",
       " 'on',\n",
       " 'ou',\n",
       " 'par',\n",
       " 'pas',\n",
       " 'pour',\n",
       " 'qu',\n",
       " 'que',\n",
       " 'qui',\n",
       " 'sa',\n",
       " 'se',\n",
       " 'ses',\n",
       " 'son',\n",
       " 'sur',\n",
       " 'ta',\n",
       " 'te',\n",
       " 'tes',\n",
       " 'toi',\n",
       " 'ton',\n",
       " 'tu',\n",
       " 'un',\n",
       " 'une',\n",
       " 'vos',\n",
       " 'votre',\n",
       " 'vous',\n",
       " 'c',\n",
       " 'd',\n",
       " 'j',\n",
       " 'l',\n",
       " 'à',\n",
       " 'm',\n",
       " 'n',\n",
       " 's',\n",
       " 't',\n",
       " 'y',\n",
       " 'été',\n",
       " 'étée',\n",
       " 'étées',\n",
       " 'étés',\n",
       " 'étant',\n",
       " 'étante',\n",
       " 'étants',\n",
       " 'étantes',\n",
       " 'suis',\n",
       " 'es',\n",
       " 'est',\n",
       " 'sommes',\n",
       " 'êtes',\n",
       " 'sont',\n",
       " 'serai',\n",
       " 'seras',\n",
       " 'sera',\n",
       " 'serons',\n",
       " 'serez',\n",
       " 'seront',\n",
       " 'serais',\n",
       " 'serait',\n",
       " 'serions',\n",
       " 'seriez',\n",
       " 'seraient',\n",
       " 'étais',\n",
       " 'était',\n",
       " 'étions',\n",
       " 'étiez',\n",
       " 'étaient',\n",
       " 'fus',\n",
       " 'fut',\n",
       " 'fûmes',\n",
       " 'fûtes',\n",
       " 'furent',\n",
       " 'sois',\n",
       " 'soit',\n",
       " 'soyons',\n",
       " 'soyez',\n",
       " 'soient',\n",
       " 'fusse',\n",
       " 'fusses',\n",
       " 'fût',\n",
       " 'fussions',\n",
       " 'fussiez',\n",
       " 'fussent',\n",
       " 'ayant',\n",
       " 'ayante',\n",
       " 'ayantes',\n",
       " 'ayants',\n",
       " 'eu',\n",
       " 'eue',\n",
       " 'eues',\n",
       " 'eus',\n",
       " 'ai',\n",
       " 'as',\n",
       " 'avons',\n",
       " 'avez',\n",
       " 'ont',\n",
       " 'aurai',\n",
       " 'auras',\n",
       " 'aura',\n",
       " 'aurons',\n",
       " 'aurez',\n",
       " 'auront',\n",
       " 'aurais',\n",
       " 'aurait',\n",
       " 'aurions',\n",
       " 'auriez',\n",
       " 'auraient',\n",
       " 'avais',\n",
       " 'avait',\n",
       " 'avions',\n",
       " 'aviez',\n",
       " 'avaient',\n",
       " 'eut',\n",
       " 'eûmes',\n",
       " 'eûtes',\n",
       " 'eurent',\n",
       " 'aie',\n",
       " 'aies',\n",
       " 'ait',\n",
       " 'ayons',\n",
       " 'ayez',\n",
       " 'aient',\n",
       " 'eusse',\n",
       " 'eusses',\n",
       " 'eût',\n",
       " 'eussions',\n",
       " 'eussiez',\n",
       " 'eussent']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords.words('French')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['aber',\n",
       " 'alle',\n",
       " 'allem',\n",
       " 'allen',\n",
       " 'aller',\n",
       " 'alles',\n",
       " 'als',\n",
       " 'also',\n",
       " 'am',\n",
       " 'an',\n",
       " 'ander',\n",
       " 'andere',\n",
       " 'anderem',\n",
       " 'anderen',\n",
       " 'anderer',\n",
       " 'anderes',\n",
       " 'anderm',\n",
       " 'andern',\n",
       " 'anderr',\n",
       " 'anders',\n",
       " 'auch',\n",
       " 'auf',\n",
       " 'aus',\n",
       " 'bei',\n",
       " 'bin',\n",
       " 'bis',\n",
       " 'bist',\n",
       " 'da',\n",
       " 'damit',\n",
       " 'dann',\n",
       " 'der',\n",
       " 'den',\n",
       " 'des',\n",
       " 'dem',\n",
       " 'die',\n",
       " 'das',\n",
       " 'dass',\n",
       " 'daß',\n",
       " 'derselbe',\n",
       " 'derselben',\n",
       " 'denselben',\n",
       " 'desselben',\n",
       " 'demselben',\n",
       " 'dieselbe',\n",
       " 'dieselben',\n",
       " 'dasselbe',\n",
       " 'dazu',\n",
       " 'dein',\n",
       " 'deine',\n",
       " 'deinem',\n",
       " 'deinen',\n",
       " 'deiner',\n",
       " 'deines',\n",
       " 'denn',\n",
       " 'derer',\n",
       " 'dessen',\n",
       " 'dich',\n",
       " 'dir',\n",
       " 'du',\n",
       " 'dies',\n",
       " 'diese',\n",
       " 'diesem',\n",
       " 'diesen',\n",
       " 'dieser',\n",
       " 'dieses',\n",
       " 'doch',\n",
       " 'dort',\n",
       " 'durch',\n",
       " 'ein',\n",
       " 'eine',\n",
       " 'einem',\n",
       " 'einen',\n",
       " 'einer',\n",
       " 'eines',\n",
       " 'einig',\n",
       " 'einige',\n",
       " 'einigem',\n",
       " 'einigen',\n",
       " 'einiger',\n",
       " 'einiges',\n",
       " 'einmal',\n",
       " 'er',\n",
       " 'ihn',\n",
       " 'ihm',\n",
       " 'es',\n",
       " 'etwas',\n",
       " 'euer',\n",
       " 'eure',\n",
       " 'eurem',\n",
       " 'euren',\n",
       " 'eurer',\n",
       " 'eures',\n",
       " 'für',\n",
       " 'gegen',\n",
       " 'gewesen',\n",
       " 'hab',\n",
       " 'habe',\n",
       " 'haben',\n",
       " 'hat',\n",
       " 'hatte',\n",
       " 'hatten',\n",
       " 'hier',\n",
       " 'hin',\n",
       " 'hinter',\n",
       " 'ich',\n",
       " 'mich',\n",
       " 'mir',\n",
       " 'ihr',\n",
       " 'ihre',\n",
       " 'ihrem',\n",
       " 'ihren',\n",
       " 'ihrer',\n",
       " 'ihres',\n",
       " 'euch',\n",
       " 'im',\n",
       " 'in',\n",
       " 'indem',\n",
       " 'ins',\n",
       " 'ist',\n",
       " 'jede',\n",
       " 'jedem',\n",
       " 'jeden',\n",
       " 'jeder',\n",
       " 'jedes',\n",
       " 'jene',\n",
       " 'jenem',\n",
       " 'jenen',\n",
       " 'jener',\n",
       " 'jenes',\n",
       " 'jetzt',\n",
       " 'kann',\n",
       " 'kein',\n",
       " 'keine',\n",
       " 'keinem',\n",
       " 'keinen',\n",
       " 'keiner',\n",
       " 'keines',\n",
       " 'können',\n",
       " 'könnte',\n",
       " 'machen',\n",
       " 'man',\n",
       " 'manche',\n",
       " 'manchem',\n",
       " 'manchen',\n",
       " 'mancher',\n",
       " 'manches',\n",
       " 'mein',\n",
       " 'meine',\n",
       " 'meinem',\n",
       " 'meinen',\n",
       " 'meiner',\n",
       " 'meines',\n",
       " 'mit',\n",
       " 'muss',\n",
       " 'musste',\n",
       " 'nach',\n",
       " 'nicht',\n",
       " 'nichts',\n",
       " 'noch',\n",
       " 'nun',\n",
       " 'nur',\n",
       " 'ob',\n",
       " 'oder',\n",
       " 'ohne',\n",
       " 'sehr',\n",
       " 'sein',\n",
       " 'seine',\n",
       " 'seinem',\n",
       " 'seinen',\n",
       " 'seiner',\n",
       " 'seines',\n",
       " 'selbst',\n",
       " 'sich',\n",
       " 'sie',\n",
       " 'ihnen',\n",
       " 'sind',\n",
       " 'so',\n",
       " 'solche',\n",
       " 'solchem',\n",
       " 'solchen',\n",
       " 'solcher',\n",
       " 'solches',\n",
       " 'soll',\n",
       " 'sollte',\n",
       " 'sondern',\n",
       " 'sonst',\n",
       " 'über',\n",
       " 'um',\n",
       " 'und',\n",
       " 'uns',\n",
       " 'unsere',\n",
       " 'unserem',\n",
       " 'unseren',\n",
       " 'unser',\n",
       " 'unseres',\n",
       " 'unter',\n",
       " 'viel',\n",
       " 'vom',\n",
       " 'von',\n",
       " 'vor',\n",
       " 'während',\n",
       " 'war',\n",
       " 'waren',\n",
       " 'warst',\n",
       " 'was',\n",
       " 'weg',\n",
       " 'weil',\n",
       " 'weiter',\n",
       " 'welche',\n",
       " 'welchem',\n",
       " 'welchen',\n",
       " 'welcher',\n",
       " 'welches',\n",
       " 'wenn',\n",
       " 'werde',\n",
       " 'werden',\n",
       " 'wie',\n",
       " 'wieder',\n",
       " 'will',\n",
       " 'wir',\n",
       " 'wird',\n",
       " 'wirst',\n",
       " 'wo',\n",
       " 'wollen',\n",
       " 'wollte',\n",
       " 'würde',\n",
       " 'würden',\n",
       " 'zu',\n",
       " 'zum',\n",
       " 'zur',\n",
       " 'zwar',\n",
       " 'zwischen']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords.words('german')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2 CMU wordlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "133737"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk \n",
    "entries = nltk.corpus.cmudict.entries()\n",
    "len(entries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('belford', ['B', 'EH1', 'L', 'F', 'ER0', 'D'])\n",
      "('belfry', ['B', 'EH1', 'L', 'F', 'R', 'IY0'])\n",
      "('belgacom', ['B', 'EH1', 'L', 'G', 'AH0', 'K', 'AA0', 'M'])\n",
      "('belgacom', ['B', 'EH1', 'L', 'JH', 'AH0', 'K', 'AA0', 'M'])\n",
      "('belgard', ['B', 'EH0', 'L', 'G', 'AA1', 'R', 'D'])\n",
      "('belgarde', ['B', 'EH0', 'L', 'G', 'AA1', 'R', 'D', 'IY0'])\n",
      "('belge', ['B', 'EH1', 'L', 'JH', 'IY0'])\n",
      "('belger', ['B', 'EH1', 'L', 'G', 'ER0'])\n",
      "('belgian', ['B', 'EH1', 'L', 'JH', 'AH0', 'N'])\n",
      "('belgians', ['B', 'EH1', 'L', 'JH', 'AH0', 'N', 'Z'])\n",
      "('belgique', ['B', 'EH0', 'L', 'ZH', 'IY1', 'K'])\n",
      "(\"belgique's\", ['B', 'EH0', 'L', 'JH', 'IY1', 'K', 'S'])\n",
      "('belgium', ['B', 'EH1', 'L', 'JH', 'AH0', 'M'])\n",
      "(\"belgium's\", ['B', 'EH1', 'L', 'JH', 'AH0', 'M', 'Z'])\n",
      "('belgo', ['B', 'EH1', 'L', 'G', 'OW2'])\n",
      "('belgrade', ['B', 'EH1', 'L', 'G', 'R', 'EY0', 'D'])\n",
      "('belgrade', ['B', 'EH1', 'L', 'G', 'R', 'AA2', 'D'])\n",
      "(\"belgrade's\", ['B', 'EH1', 'L', 'G', 'R', 'EY0', 'D', 'Z'])\n",
      "(\"belgrade's\", ['B', 'EH1', 'L', 'G', 'R', 'AA2', 'D', 'Z'])\n",
      "('belgrave', ['B', 'EH1', 'L', 'G', 'R', 'EY2', 'V'])\n",
      "('beli', ['B', 'EH1', 'L', 'IY0'])\n",
      "('belich', ['B', 'EH1', 'L', 'IH0', 'K'])\n",
      "('belie', ['B', 'IH0', 'L', 'AY1'])\n",
      "('belied', ['B', 'IH0', 'L', 'AY1', 'D'])\n",
      "('belief', ['B', 'IH0', 'L', 'IY1', 'F'])\n"
     ]
    }
   ],
   "source": [
    "for entry in entries [10000:10025]:\n",
    "    print(entry)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.3 Wordnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Synset('house.n.01'),\n",
       " Synset('firm.n.01'),\n",
       " Synset('house.n.03'),\n",
       " Synset('house.n.04'),\n",
       " Synset('house.n.05'),\n",
       " Synset('house.n.06'),\n",
       " Synset('house.n.07'),\n",
       " Synset('sign_of_the_zodiac.n.01'),\n",
       " Synset('house.n.09'),\n",
       " Synset('family.n.01'),\n",
       " Synset('theater.n.01'),\n",
       " Synset('house.n.12'),\n",
       " Synset('house.v.01'),\n",
       " Synset('house.v.02')]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import wordnet as wn\n",
    "wn.synsets(\"phone\") # you get an id for subsets.\n",
    "wn.synsets(\"house\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['call', 'telephone', 'call_up', 'phone', 'ring']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn.synset('call.v.03').lemma_names() #head words/lemmas in the subset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['telephone', 'phone', 'telephone_set']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn.synset('telephone.n.01').lemma_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['family', 'household', 'house', 'home', 'menage']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn.synset('family.n.01').lemma_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task-2  NLTK Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Christopher', 'NNP'), ('Hemsworth', 'NNP'), ('born', 'VBD'), ('11', 'CD'), ('August', 'NNP'), ('1983', 'CD'), ('is', 'VBZ'), ('an', 'DT'), ('Australian', 'JJ'), ('actor', 'NN'), ('.', '.')]\n",
      "[('He', 'PRP'), ('rose', 'VBD'), ('to', 'TO'), ('prominence', 'VB'), ('playing', 'VBG'), ('Kim', 'NNP'), ('Hyde', 'NNP'), ('in', 'IN'), ('the', 'DT'), ('Australian', 'JJ'), ('TV', 'NN'), ('series', 'NN'), ('Home', 'NNP'), ('and', 'CC'), ('Away', 'NNP'), ('(', '('), ('2004–07', 'CD'), (')', ')'), ('before', 'IN'), ('beginning', 'VBG'), ('a', 'DT'), ('film', 'NN'), ('career', 'NN'), ('in', 'IN'), ('Hollywood', 'NNP'), ('by', 'IN'), ('taking', 'VBG'), ('on', 'IN'), ('parts', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('science', 'NN'), ('fiction', 'NN'), ('film', 'NN'), ('Star', 'NNP'), ('Trek', 'NNP'), ('(', '('), ('2009', 'CD'), (')', ')'), ('and', 'CC'), ('the', 'DT'), ('thriller', 'NN'), ('A', 'NNP'), ('Perfect', 'NNP'), ('Getaway', 'NNP'), ('(', '('), ('2009', 'CD'), (')', ')'), ('.', '.')]\n",
      "[('Hemsworth', 'NNP'), ('went', 'VBD'), ('on', 'IN'), ('to', 'TO'), ('star', 'VB'), ('in', 'IN'), ('the', 'DT'), ('fantasy', 'NN'), ('film', 'NN'), ('Snow', 'NNP'), ('White', 'NNP'), ('and', 'CC'), ('the', 'DT'), ('Huntsman', 'NNP'), ('(', '('), ('2012', 'CD'), (')', ')'), (',', ','), ('the', 'DT'), ('war', 'NN'), ('film', 'NN'), ('Red', 'NNP'), ('Dawn', 'NNP'), ('(', '('), ('2012', 'CD'), (')', ')'), (',', ','), ('the', 'DT'), ('action', 'NN'), ('thriller', 'NN'), ('Blackhat', 'NNP'), ('(', '('), ('2015', 'CD'), (')', ')'), (',', ','), ('the', 'DT'), ('biographical', 'JJ'), ('thriller', 'NN'), ('In', 'IN'), ('the', 'DT'), ('Heart', 'NNP'), ('of', 'IN'), ('the', 'DT'), ('Sea', 'NNP'), ('(', '('), ('2015', 'CD'), (')', ')'), (',', ','), ('the', 'DT'), ('comedy', 'NN'), ('Ghostbusters', 'NNP'), ('(', '('), ('2016', 'CD'), (')', ')'), (',', ','), ('and', 'CC'), ('the', 'DT'), ('Men', 'NNP'), ('in', 'IN'), ('Black', 'NNP'), ('film', 'NN'), ('series', 'NN'), ('spin-off', 'NN'), ('Men', 'NNP'), ('in', 'IN'), ('Black', 'NNP'), (':', ':'), ('International', 'NNP'), ('(', '('), ('2019', 'CD'), (')', ')'), ('.', '.')]\n",
      "[('His', 'PRP$'), ('most', 'RBS'), ('critically', 'RB'), ('acclaimed', 'JJ'), ('roles', 'NNS'), ('include', 'VBP'), ('the', 'DT'), ('comedy', 'NN'), ('horror', 'VBD'), ('The', 'DT'), ('Cabin', 'NNP'), ('in', 'IN'), ('the', 'DT'), ('Woods', 'NNP'), ('(', '('), ('2012', 'CD'), (')', ')'), ('and', 'CC'), ('the', 'DT'), ('biographical', 'JJ'), ('sports', 'NNS'), ('film', 'NN'), ('Rush', 'NNP'), ('(', '('), ('2013', 'CD'), (')', ')'), (',', ','), ('in', 'IN'), ('which', 'WDT'), ('he', 'PRP'), ('portrayed', 'VBD'), ('James', 'NNP'), ('Hunt', 'NNP'), ('.', '.')]\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "texts = [\"\"\" Christopher Hemsworth born 11 August 1983 is an Australian actor. He rose to prominence playing Kim Hyde in the Australian TV series Home and Away (2004–07) \n",
    "before beginning a film career in Hollywood by taking on parts in the science fiction film Star Trek (2009) and the thriller A Perfect Getaway (2009).\n",
    "Hemsworth went on to star in the fantasy film Snow White and the Huntsman (2012), the war film Red Dawn (2012), the action thriller Blackhat (2015), \n",
    "the biographical thriller In the Heart of the Sea (2015),\n",
    "the comedy Ghostbusters (2016), and the Men in Black film series spin-off Men in Black: International (2019). \n",
    "His most critically acclaimed roles include the comedy horror The Cabin in the Woods (2012) and the biographical sports film Rush (2013), in which he portrayed James Hunt.\n",
    "\"\"\"]\n",
    "\n",
    "#the text from wikipedia about chris hemsworth.\n",
    "for text in texts:\n",
    "    sentences =nltk.sent_tokenize(text)\n",
    "    for sentence in sentences:\n",
    "        words =nltk.word_tokenize(sentence)\n",
    "        tagged_words =nltk.pos_tag(words)\n",
    "        print(tagged_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task-3 Implementing tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['After',\n",
       " 'years',\n",
       " 'of',\n",
       " 'rebuilding',\n",
       " 'OTHER',\n",
       " 'nations',\n",
       " ',',\n",
       " 'we',\n",
       " 'are',\n",
       " 'finally',\n",
       " 'rebuilding',\n",
       " 'OUR',\n",
       " 'nation',\n",
       " '.',\n",
       " 'We',\n",
       " 'are',\n",
       " 'finally',\n",
       " 'putting',\n",
       " 'AMERICA',\n",
       " 'FIRST',\n",
       " '!',\n",
       " ':)']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Twitter aware Tokenizer\n",
    "import nltk\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "text =' After years of rebuilding OTHER nations, we are finally rebuilding OUR nation. We are finally putting AMERICA FIRST! :) ' #Donald J trump Tweet\n",
    "twtkn =TweetTokenizer()\n",
    "twtkn.tokenize(text)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Task-4 Frequency Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "can: 94\n",
      " could: 87\n",
      " may: 93\n",
      " might: 38\n",
      " must: 53\n",
      " will: 389\n",
      " what: 95\n",
      " "
     ]
    }
   ],
   "source": [
    "# Brown Corpus:\n",
    "from nltk.corpus import brown\n",
    "news_text =brown.words(categories ='news')\n",
    "fdist =nltk.FreqDist(w.lower() for w in news_text)\n",
    "modals =['can','could','may','might','must','will','what']\n",
    "for m in modals:\n",
    "    print(m+ ':',fdist[m],end='\\n ')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}