MOUD_Text_Translation_Production.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text analysis with translation on the MOUD dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains a model to evaluate text data from the MOUD dataset using the bag of words approach."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The path of the train and test transcriptions\n",
    "# The data is seperated in an 80-20 ratio and the test directory is untouched. \n",
    "train_path = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\VideoReviews\\transcriptions\\train\\*.csv\"\n",
    "test_path = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\VideoReviews\\transcriptions\\test\\*.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#endtime</th>\n",
       "      <th>#starttime</th>\n",
       "      <th>Speech</th>\n",
       "      <th>sentimentAnnotation</th>\n",
       "      <th>sentimentAnnotations</th>\n",
       "      <th>sentimentannotations</th>\n",
       "      <th>speech</th>\n",
       "      <th>transcription</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.642</td>\n",
       "      <td>0.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yo habia visto resenas que decian que picaba c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.552</td>\n",
       "      <td>3.642</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>y la verdad es que si la use una vez y t- y te...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.197</td>\n",
       "      <td>9.552</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>y dije no: puede ser posible tanto la deseaba ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20.545</td>\n",
       "      <td>14.197</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>esta tambien tira un poquito de pelo pero haga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23.275</td>\n",
       "      <td>20.545</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pero igual con las lavadas se ha dejado de tir...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #endtime  #starttime Speech  sentimentAnnotation  sentimentAnnotations  \\\n",
       "0     3.642       0.000    NaN                  NaN                  -1.0   \n",
       "1     9.552       3.642    NaN                  NaN                  -1.0   \n",
       "2    14.197       9.552    NaN                  NaN                  -1.0   \n",
       "3    20.545      14.197    NaN                  NaN                  -1.0   \n",
       "4    23.275      20.545    NaN                  NaN                   1.0   \n",
       "\n",
       "   sentimentannotations speech  \\\n",
       "0                   NaN    NaN   \n",
       "1                   NaN    NaN   \n",
       "2                   NaN    NaN   \n",
       "3                   NaN    NaN   \n",
       "4                   NaN    NaN   \n",
       "\n",
       "                                       transcription  \n",
       "0  yo habia visto resenas que decian que picaba c...  \n",
       "1  y la verdad es que si la use una vez y t- y te...  \n",
       "2  y dije no: puede ser posible tanto la deseaba ...  \n",
       "3  esta tambien tira un poquito de pelo pero haga...  \n",
       "4  pero igual con las lavadas se ha dejado de tir...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "for f in glob.glob(train_path):\n",
    "    df = df.append(pd.read_csv(f,sep=';'),ignore_index=True)\n",
    "# df = pd.concat((pd.read_csv(f,sep=';'),ignore_index=True for f in glob.glob(path)))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As shown, there are multiple speech and annotation columns. This requires data consolidation and managing missing values.    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# funcion to append all utterances to dataframe\n",
    "def create_data_df(df_name,data_path):\n",
    "    \n",
    "    # Creating dataframe of entire transcriptions\n",
    "    for f in glob.glob(data_path):\n",
    "        df_name = df_name.append(pd.read_csv(f,sep=';'),ignore_index=True)\n",
    "    \n",
    "    # combine multiple speech, annotation columns to one and drop rest of columns\n",
    "    if 'Speech' not in df_name.columns:\n",
    "        df_name['Speech'] = ''    \n",
    "    if 'speech' in df_name.columns:\n",
    "        df_name['Speech'] = df_name[['Speech','speech']].fillna('').sum(axis=1)   \n",
    "    if 'transcription' in df_name.columns:\n",
    "        df_name['Speech'] = df_name[['Speech','transcription']].fillna('').sum(axis=1)\n",
    "    \n",
    "    if 'sentimentAnnotation' not in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = 0    \n",
    "    if 'sentimentAnnotations' in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = df_name[['sentimentAnnotation','sentimentAnnotations']].fillna(0).sum(axis=1)\n",
    "    if 'sentimentannotations' in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = df_name[['sentimentAnnotation','sentimentannotations']].fillna(0).sum(axis=1)\n",
    "    \n",
    "    # Remove neutral annotations\n",
    "    df_name = df_name.query('sentimentAnnotation != 0')\n",
    "    \n",
    "    df_name = df_name[['Speech','sentimentAnnotation']].reset_index(drop=True)  \n",
    "    return df_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cleaned transcriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speech</th>\n",
       "      <th>sentimentAnnotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pero de verdad lo recomiendo</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>porque es que: eh: tiene de todo o sea no es el:</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>no he leido nunca ningun libro de esos zombies</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>pero de peliculas y tal a mi que tampoco me su...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>que la verdad no me gusta, pero no estamos hab...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Speech  sentimentAnnotation\n",
       "0                       pero de verdad lo recomiendo                    1\n",
       "1   porque es que: eh: tiene de todo o sea no es el:                    1\n",
       "2     no he leido nunca ningun libro de esos zombies                   -1\n",
       "3  pero de peliculas y tal a mi que tampoco me su...                    1\n",
       "4  que la verdad no me gusta, pero no estamos hab...                   -1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df_t = pd.DataFrame()\n",
    "df = create_data_df(df,train_path)\n",
    "df_t = create_data_df(df_t,test_path)\n",
    "df_t.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ALTERNATE Converting video to test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This alternate section is for feeding a video to a trained model and perform sentiment analysis on it. A video is fed to the path in the next code section and IBM Bluemix Speech to Text is used to obtain the transcriptions.\n",
    "\n",
    "Else skip to Data processing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert video to audio, insert path\n",
    "# vpath = r\"C:\\Datasets\\MOUD\\VideoReviews\\178_makeup.mp4\"\n",
    "\n",
    "# Using a wav from video-audio-converter as video file did not operate as intended \n",
    "apath = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\Python\\MOUD\\Text_Video\\178_makeup.wav\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Using IBM Bluemix to convert Speech to Text \n",
    "import json\n",
    "from os.path import join, dirname\n",
    "from watson_developer_cloud import SpeechToTextV1\n",
    "\n",
    "speech_to_text = SpeechToTextV1(\n",
    "    username='',\n",
    "    password='',\n",
    "    x_watson_learning_opt_out=False\n",
    ")\n",
    "\n",
    "with open(apath,\n",
    "          'rb') as audio_file:\n",
    "    trn = speech_to_text.recognize(\n",
    "        audio_file, content_type='audio/wav', timestamps=False, model='es-ES_BroadbandModel',\n",
    "        word_confidence=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_t = pd.DataFrame(columns = ['Speech','sentimentAnnotation'])\n",
    "for i in range(len(trn['results'])):\n",
    "    df_t.loc[len(df_t)]=[trn['results'][i]['alternatives'][0]['transcript'],'Unknown']\n",
    "print(\"Converted Speech to Text utterances in the Speech column\")\n",
    "df_t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data cleaning, translation and text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from https://www.kaggle.com/c/word2vec-nlp-tutorial/\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "\n",
    "# execute the following commented step to install the data packages if you don't already have it  \n",
    "# nltk.download()\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "#using text translation API\n",
    "from watson_developer_cloud import LanguageTranslatorV2\n",
    "language_translator = LanguageTranslatorV2(\n",
    "    username=\"\", #insert API username here \n",
    "    password=\"\") # insert API password here\n",
    "\n",
    "# resuable function to convert raw speech to preprocessed\n",
    "def utterance_to_words(raw_utterance):\n",
    "    # 1. Removing HTML elements from text\n",
    "    utterance_text = BeautifulSoup(raw_utterance, \"lxml\").get_text()\n",
    "    # TRANSLATION\n",
    "    translated_utterance = language_translator.translate(utterance_text, source='es',target='en')\n",
    "    # 2. Keeping only letters\n",
    "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", utterance_text) \n",
    "    # 3. Converting to lower case and splitting into individual words\n",
    "    lowercase_words = letters_only.lower().split()\n",
    "    # 4. converting the stop words to a set to help faster execution\n",
    "    spanish_stops = set(stopwords.words(\"english\"))\n",
    "    # 5. Removing stop words from the text\n",
    "    meaningful_words = [w for w in lowercase_words if not w in spanish_stops]\n",
    "    # 6. Join the words back into one string separated by space, and return the result.\n",
    "    return( \" \".join( meaningful_words ))\n",
    "    \n",
    "# applying the function to the speech columns\n",
    "df['Speech'] = df['Speech'].apply(lambda x: utterance_to_words(x))\n",
    "df_t['Speech'] = df_t['Speech'].apply(lambda x: utterance_to_words(x))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Machine learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_trn, y_trn = df[['Speech']],df[['sentimentAnnotation']]\n",
    "\n",
    "# countVectorizer initialization\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(analyzer = \"word\",   \\\n",
    "                             tokenizer = None,    \\\n",
    "                             preprocessor = None, \\\n",
    "                             stop_words = None,   \\\n",
    "                             lowercase = True,    \\\n",
    "                             max_features = 5000) \n",
    "\n",
    "# create bag of words vector for the training set using countVectorizer\n",
    "train_data_features = vectorizer.fit_transform(X_trn['Speech'].values)\n",
    "\n",
    "# tf-idf transformer initialization\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "# create tfidf transformed vector  for the training set using tf-idf transformer\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(train_data_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_tst, y_tst = df_t[['Speech']],df_t[['sentimentAnnotation']]\n",
    "\n",
    "# transformation of test data\n",
    "test_data_features = vectorizer.transform(X_tst['Speech'].values)\n",
    "X_test_tfidf = tfidf_transformer.transform(test_data_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# SVM model creation and fitting train vector to annotations\n",
    "\n",
    "from sklearn import svm\n",
    "model_tf = svm.SVC(kernel='linear', C=1, gamma=1).fit(X_train_tfidf,y_trn['sentimentAnnotation'].values)\n",
    "\n",
    "# generate predictions\n",
    "predicted_tf = model_tf.predict(X_test_tfidf)\n",
    "\n",
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf))\n",
    "\n",
    "#create df to show results\n",
    "disp = X_tst.join(y_tst).reset_index(drop=True).join(pd.DataFrame(predicted_tf,columns=['Prediction']))\n",
    "disp = disp.join(pd.DataFrame(disp['sentimentAnnotation']==disp['Prediction'],columns=['Right/Wrong']))\n",
    "\n",
    "scores = model_tf.score(X_test_tfidf,y_tst['sentimentAnnotation'].values)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\\n\" % (scores.mean(), scores.std() * 2))\n",
    "print(\"Mean sentiment: {!r}.\".format('Positive' if disp['sentimentAnnotation'].mean()>=0 else 'Negative'))\n",
    "                            \n",
    "print(\"Predicted mean sentiment: {!r}.\".format('Positive' if disp['Prediction'].mean()>=0 else 'Negative'))\n",
    "disp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cross validation of training set using SVM\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf_cv = svm.SVC(kernel='linear', C=1, gamma=1)\n",
    "scores = cross_val_score(clf_cv, X_train_tfidf, y_trn['sentimentAnnotation'].values, cv=10)\n",
    "scores\n",
    "print(\"Cross validated accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}