baggingWithLogisticRegression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to C:\\Users\\wel\n",
      "[nltk_data]     come\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to C:\\Users\\wel\n",
      "[nltk_data]     come\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#@unpublished{SARC,\n",
    "#  authors={Mikhail Khodak and Nikunj Saunshi and Kiran Vodrahalli},\n",
    "#  title={A Large Self-Annotated Corpus for Sarcasm},\n",
    "#  url={https://arxiv.org/abs/1704.05579},\n",
    "#  year=2017\n",
    "#}\n",
    "#https://www.kaggle.com/danofer/sarcasm#train-balanced-sarcasm.csv\n",
    "import os\n",
    "import string\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type:\n",
      "Labels array  <class 'pandas.core.series.Series'>\n",
      "Comments array  <class 'pandas.core.series.Series'>\n",
      "one comment line  <class 'str'>\n",
      "Shape:\n",
      "Labels array  (1010773,)\n",
      "Comments array  (1010773,)\n",
      "Two first entries:\n",
      "0 NC and NH.\n",
      "0 You do know west teams play against west teams more than east teams right?\n"
     ]
    }
   ],
   "source": [
    "# df - data frame\n",
    "df = pd.read_csv('C:/Akshata/Courses/Machine_Learning/Final_Project/sarcasm/train-balanced-sarcasm.csv')\n",
    "# dropping empty comment entries\n",
    "df.dropna(subset=['comment'], inplace=True)\n",
    "\n",
    "print('Type:')\n",
    "print('Labels array ',type(df.label))\n",
    "print('Comments array ',type(df.comment))\n",
    "print('one comment line ', type(df.comment[0]))\n",
    "print('Shape:')\n",
    "print('Labels array ',df.label.shape)\n",
    "print('Comments array ',df.comment.shape)\n",
    "print('Two first entries:')\n",
    "print (df.label[0], df.comment[0])\n",
    "print (df.label[1], df.comment[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "167435\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# create the transform\n",
    "tf_idf_vectorizer = TfidfVectorizer()\n",
    "# tokenize and build vocab\n",
    "T = tf_idf_vectorizer.fit(df.comment)\n",
    "print(len(T.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocess with nltk\n",
    "def my_tokenizer(corpus):\n",
    "    corpus_tokenized = []\n",
    "    tokenizer = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "    sbs = nltk.stem.SnowballStemmer('english', ignore_stopwords=False)\n",
    "    for comment in corpus:\n",
    "        words = tokenizer.tokenize(comment)\n",
    "        # Remove stop words\n",
    "        stop_words = set(stopwords.words('english'))\n",
    "        words = [word for word in words if not word in stop_words]\n",
    "        \n",
    "        cmnt_t = []\n",
    "        for token in words:\n",
    "            cmnt_t.append(sbs.stem(token))\n",
    "            # make a string to be compatible with TfidfVectorizer\n",
    "            c = ' '.join(cmnt_t)\n",
    "        # Lemmitize\n",
    "        #words = [lemmatizer.lemmatize(word) for word in words]\n",
    "        \n",
    "        corpus_tokenized.append(c)\n",
    "    return corpus_tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = my_tokenizer(df.comment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NC and NH.\n",
      "nc nh \n",
      "\n",
      "You do know west teams play against west teams more than east teams right?\n",
      "you know west team play west team east team right \n",
      "\n",
      "They were underdogs earlier today, but since Gronk's announcement this afternoon, the Vegas line has moved to patriots -1\n",
      "they underdog earlier today sinc gronk announc afternoon vega line move patriot 1 \n",
      "\n",
      "This meme isn't funny none of the \"new york nigga\" ones are.\n",
      "this meme funni none new york nigga one \n",
      "\n",
      "I could use one of those tools.\n",
      "i could use one tool \n",
      "\n",
      "I don't pay attention to her, but as long as she's legal I wouldn't kick her out of bed (before she took a load)\n",
      "i pay attent long legal i kick bed took load \n",
      "\n",
      "Trick or treating in general is just weird...\n",
      "trick treat general weird \n",
      "\n",
      "Blade Mastery+Masamune or GTFO!\n",
      "blade masteri masamun gtfo \n",
      "\n",
      "You don't have to, you have a good build, buy games or save it\n",
      "you good build buy game save \n",
      "\n",
      "I would love to see him at lolla.\n",
      "i would love see lolla \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(df.comment[i])\n",
    "    print(df2[i], '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['nc nh', 'you know west team play west team east team right', 'they underdog earlier today sinc gronk announc afternoon vega line move patriot 1', 'this meme funni none new york nigga one', 'i could use one tool'] \n",
      "\n",
      "<class 'pandas.core.series.Series'> \n",
      "\n",
      "<class 'str'>\n",
      "0                                                nc nh\n",
      "1    you know west team play west team east team right\n",
      "2    they underdog earlier today sinc gronk announc...\n",
      "3              this meme funni none new york nigga one\n",
      "4                                 i could use one tool\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "print(df2[0:5], '\\n')\n",
    "# convert to pandas Series type type to be compatible with TfidfVectorizer\n",
    "df3 = pd.Series((v for v in df2))\n",
    "print(type(df3), '\\n')\n",
    "print(type(df3[0]))\n",
    "print(df3[0:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131021\n"
     ]
    }
   ],
   "source": [
    "tf_idf_vectorizer2 = TfidfVectorizer()\n",
    "T2 = tf_idf_vectorizer2.fit(df3)\n",
    "print(len(T2.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\wel come\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "# divide into train and test sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(df.comment, df.label, train_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(333555, 131021)\n",
      "  (0, 125869)\t0.23934118306789068\n",
      "  (0, 124870)\t0.3509017771796655\n",
      "  (0, 124689)\t0.22686181040398212\n",
      "  (0, 114778)\t0.21200387462694437\n",
      "  (0, 102686)\t0.41752672154338055\n",
      "  (0, 83738)\t0.3149355493431261\n",
      "  (0, 77608)\t0.34135165191028866\n",
      "  (0, 71837)\t0.32970632803230326\n",
      "  (0, 60616)\t0.20241073535983167\n",
      "  (0, 58715)\t0.2953394110764406\n",
      "  (0, 35320)\t0.31021954646680633\n",
      "  (1, 119764)\t0.8811426115264012\n",
      "  (1, 114778)\t0.47285060870473\n",
      "  (2, 127805)\t0.4495168780435586\n",
      "  (2, 101099)\t0.3687708996621765\n",
      "  (2, 52507)\t0.6267718697281396\n",
      "  (2, 34179)\t0.5187481308244044\n"
     ]
    }
   ],
   "source": [
    "# Vetorize the training data set\n",
    "X_train = tf_idf_vectorizer2.transform(X_train)\n",
    "print(X_train.shape)\n",
    "print(X_train[0:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(677218, 167435)\n",
      "  (0, 148192)\t0.27672006433000734\n",
      "  (0, 86933)\t0.46059859311383466\n",
      "  (0, 57268)\t0.45024645274653885\n",
      "  (0, 52924)\t0.713129072333972\n",
      "  (1, 161232)\t0.20897403315091123\n",
      "  (1, 149296)\t0.3507752301743271\n",
      "  (1, 148055)\t0.14161807247053007\n",
      "  (1, 147871)\t0.4348831561857896\n",
      "  (1, 147395)\t0.11348779883018499\n",
      "  (1, 137869)\t0.2853638097017254\n",
      "  (1, 110381)\t0.2511925776572341\n",
      "  (1, 105958)\t0.1194865780167631\n",
      "  (1, 104293)\t0.14937557296331172\n",
      "  (1, 98257)\t0.300265336949117\n",
      "  (1, 88017)\t0.2949222218914805\n",
      "  (1, 33758)\t0.2695358128386277\n",
      "  (1, 19173)\t0.1580710675352837\n",
      "  (1, 18945)\t0.14453443552468329\n",
      "  (1, 12361)\t0.36666489391528584\n",
      "  (2, 163619)\t0.25201247174047986\n",
      "  (2, 137869)\t0.25742003015709025\n",
      "  (2, 133628)\t0.26304380217798806\n",
      "  (2, 101542)\t0.39686492712337973\n",
      "  (2, 77895)\t0.10162300008640673\n",
      "  (2, 77613)\t0.1050313581895657\n",
      "  (2, 76787)\t0.23916197112984935\n",
      "  (2, 69661)\t0.19251529072686677\n",
      "  (2, 65073)\t0.19771088224173394\n",
      "  (2, 64068)\t0.22938997376602555\n",
      "  (2, 50753)\t0.40769691901239064\n",
      "  (2, 45349)\t0.1508059864739178\n",
      "  (2, 33758)\t0.2431419637334476\n",
      "  (2, 26313)\t0.12680799919777186\n",
      "  (2, 18945)\t0.1303811397471577\n",
      "  (2, 15657)\t0.38007724985700725\n",
      "  (2, 11906)\t0.10289123854402644\n"
     ]
    }
   ],
   "source": [
    "# Vetorize the test data set\n",
    "X_test = tf_idf_vectorizer.transform(X_test)\n",
    "print(X_test.shape)\n",
    "print(X_test[0:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oob_score is  0.6580384044610335\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "# bagging with Logistic Regressoin\n",
    "# oob Out-Of-Bag\n",
    "bag_log_reg = BaggingClassifier(\n",
    "      LogisticRegression(), n_estimators=50,\n",
    "    max_samples=50000, bootstrap=True, n_jobs=-1, oob_score=True)\n",
    "\n",
    "bag_log_reg.fit(X_train,y_train)\n",
    "print('oob_score is ', bag_log_reg.oob_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "oob_score is  0.6580384044610335\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rnd_frst = RandomForestClassifier(\n",
    "      n_estimators=100, max_leaf_nodes=1000, n_jobs=-1, oob_score=True)\n",
    "rnd_frst.fit(X_train,y_train)\n",
    "print('oob_score is ', bag_log_reg.oob_score_)\n",
    "\n",
    "# 5-folds cross validation with grid search\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# I tried with n_jobs=-1 but kernel died after a few hours\n",
    "params = [{'n_estimators': [100, 200], 'min_samples_leaf': [1, 10, 50, 100],\n",
    "                     'random_state': [47], 'n_jobs': [2]}]\n",
    "rnd_frst = GridSearchCV(RandomForestClassifier(), params, cv=5, scoring='accuracy')\n",
    "\n",
    "#clf = RandomForestClassifier(n_estimators=200, min_samples_leaf=50, random_state=31, n_jobs = -1)\n",
    "rnd_frst.fit(X_train, y_train)\n",
    "print('oob_score is ', rnd_frst.oob_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}