MOUD_Text_Translated_Video_Production.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Video analysis on the MOUD dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains a model to evaluate facial behaviors from videos from the MOUD dataset obtaining and processing data obtained from OpenFace toolkit. LINK: https://github.com/TadasBaltrusaitis/OpenFace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The path of the train and test transcriptions\n",
    "# The data is seperated in an 80-20 ratio and the test directory is untouched. \n",
    "train_path = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\VideoReviews\\transcriptions\\train\\*.csv\"\n",
    "test_path = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\VideoReviews\\transcriptions\\test\\*.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# funcion to combine multiple speech, annotation columns to one and drop rest of columns\n",
    "def clean_moud(df_name):\n",
    "    if 'Speech' not in df_name.columns:\n",
    "        df_name['Speech'] = ''    \n",
    "    if 'speech' in df_name.columns:\n",
    "        df_name['Speech'] = df_name[['Speech','speech']].fillna('').sum(axis=1)   \n",
    "    if 'transcription' in df_name.columns:\n",
    "        df_name['Speech'] = df_name[['Speech','transcription']].fillna('').sum(axis=1)\n",
    "\n",
    "    if 'sentimentAnnotation' not in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = 0    \n",
    "    if 'sentimentAnnotations' in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = df_name[['sentimentAnnotation','sentimentAnnotations']].fillna(0).sum(axis=1)\n",
    "    if 'sentimentannotations' in df_name.columns:\n",
    "        df_name['sentimentAnnotation'] = df_name[['sentimentAnnotation','sentimentannotations']].fillna(0).sum(axis=1)\n",
    "    \n",
    "    return df_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# funcion to append all utterances to dataframe\n",
    "def create_data_df(df_name,data_path):\n",
    "    '''\n",
    "    Returns a text dataframe with two columns 'Speech' and 'sentimentAnnotation'\n",
    "    Returns a sparse matrix of video features to be combined with the text tfidf later'\n",
    "    '''\n",
    "    # Creating video df\n",
    "    v_cols = []\n",
    "    skeleton_path = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Python\\MOUD\\Text_Video\\video_skeleton.csv\"\n",
    "    df_v = pd.DataFrame(pd.read_csv(skeleton_path, sep = ','))\n",
    "    df_v = df_v.drop([df_v.columns.values[0]],axis=1)\n",
    "\n",
    "    for f in glob.glob(data_path):\n",
    "        \n",
    "        # TEXT \n",
    "        # append speech utterances to text dataframe \n",
    "        df_name = df_name.append(pd.read_csv(f,sep=';'),ignore_index=True)\n",
    "        \n",
    "        # VIDEO\n",
    "        # Create sparse video matrix for each file consecutively while creating text dataframe\n",
    "        # It is done at this particular point to extract time related groups before the starttime and endtimes are lost\n",
    "        \n",
    "        # Creating a temporary text df to get times and clean\n",
    "        df_name_temp = pd.read_csv(f,sep=';')\n",
    "\n",
    "        df_name_temp = clean_moud(df_name_temp)\n",
    "        \n",
    "        # Remove neutral annotations\n",
    "        df_name_temp = df_name_temp.query('sentimentAnnotation != 0')\n",
    "         \n",
    "        # Creating a df of the corredponding OpenFace features file \n",
    "        v_name = r\"C:\\Users\\Roshan Sridhar\\Google Drive\\Documents\\NYU\\GILAB\\MMML\\Datasets\\MOUD\\OpenFaceFeatures\" + \"\\\\\" + f.rsplit(\"\\\\\",1)[1].split(\".\")[0] + \".mp4.csv\"\n",
    "        df_v_name = pd.read_csv(v_name, sep = \", \", engine = \"python\")\n",
    "    \n",
    "        # Splitting the video data by utterances\n",
    "        for starttime,endtime in zip(df_name_temp['#starttime'],df_name_temp['#endtime']):    \n",
    "            # Generate mean and standard deviation upto endtime of utterance, new df because columns need to be dropped\n",
    "            df_v_name_temp = df_v_name.query('timestamp >='+str(starttime)+'& timestamp <='+str(endtime)).agg(['mean','std'])\n",
    "            # Drop unwanted labels after querying because timestamp is required to filter in prev line\n",
    "            df_v_name_temp.drop(['frame','timestamp','confidence','success'], axis = 1)\n",
    "            # append single row of means and stds to the main dataframe\n",
    "            \n",
    "            df_v.loc[len(df_v)] = np.array(df_v_name_temp).ravel()\n",
    "\n",
    "    # TEXT \n",
    "    # combine multiple speech, annotation columns to one and drop rest of columns\n",
    "    df_name = clean_moud(df_name)\n",
    "    \n",
    "    # Remove neutral annotations\n",
    "    df_name = df_name.query('sentimentAnnotation != 0')\n",
    "    \n",
    "    df_name = df_name[['Speech','sentimentAnnotation']].reset_index(drop=True)  \n",
    "    \n",
    "    return df_name, df_v.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Speech</th>\n",
       "      <th>sentimentAnnotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yo habia visto resenas que decian que picaba c...</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>y la verdad es que si la use una vez y t- y te...</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>y dije no: puede ser posible tanto la deseaba ...</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>esta tambien tira un poquito de pelo pero haga...</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pero igual con las lavadas se ha dejado de tir...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Speech  sentimentAnnotation\n",
       "0  yo habia visto resenas que decian que picaba c...                 -1.0\n",
       "1  y la verdad es que si la use una vez y t- y te...                 -1.0\n",
       "2  y dije no: puede ser posible tanto la deseaba ...                 -1.0\n",
       "3  esta tambien tira un poquito de pelo pero haga...                 -1.0\n",
       "4  pero igual con las lavadas se ha dejado de tir...                  1.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame()\n",
    "df_t = pd.DataFrame()\n",
    "\n",
    "# Clean dataframe and create sparse video matrix\n",
    "df, v_train = create_data_df(df,train_path)\n",
    "df_t, v_test = create_data_df(df_t,test_path)\n",
    "\n",
    "#converted to sparse matrix for faster computation\n",
    "v_train_sparse = scipy.sparse.csr_matrix(v_train)\n",
    "v_test_sparse = scipy.sparse.csr_matrix(v_test)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data cleaning and text preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section 'Data cleaning and text preprocessing' is to preprocess the text for text+video analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from https://www.kaggle.com/c/word2vec-nlp-tutorial/\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import nltk\n",
    "\n",
    "# execute the following commented step to install the data packages if you don't already have it  \n",
    "# nltk.download()\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "#using text translation API\n",
    "from watson_developer_cloud import LanguageTranslatorV2\n",
    "language_translator = LanguageTranslatorV2(\n",
    "    username=\"\",\n",
    "    password=\"\")\n",
    "\n",
    "# resuable function to convert raw speech to preprocessed\n",
    "def utterance_to_words(raw_utterance):\n",
    "    # 1. Removing any HTML elements\n",
    "    utterance_text = BeautifulSoup(raw_utterance, \"lxml\").get_text()\n",
    "    # TRANSLATION\n",
    "    translated_utterance = language_translator.translate(utterance_text, source='es',target='en')\n",
    "    # 2. Remove non-letters\n",
    "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", utterance_text) \n",
    "    # 3. Convert to lower case, split into individual words\n",
    "    words = letters_only.lower().split()\n",
    "    # 4. convert the stop words to a set\n",
    "    stops = set(stopwords.words(\"english\"))\n",
    "    # 5. Remove stop words\n",
    "    meaningful_words = [w for w in words if not w in stops]\n",
    "    # 6. Join the words back into one string separated by space, and return the result.\n",
    "    return( \" \".join( meaningful_words ))\n",
    "\n",
    "# applying the function to the speech column\n",
    "df['Speech'] = df['Speech'].apply(lambda x: utterance_to_words(x))\n",
    "df_t['Speech'] = df_t['Speech'].apply(lambda x: utterance_to_words(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import train_test_split\n",
    "\n",
    "# # splitting dataset into train and test in stratified fashion and a ratio of 80% - 20%\n",
    "# X, y = df[['Speech']],df[['sentimentAnnotation']]\n",
    "# X_trn, X_tst, y_trn, y_tst = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)\n",
    "X_trn, y_trn = df[['Speech']],df[['sentimentAnnotation']]\n",
    "X_tst, y_tst = df_t[['Speech']],df_t[['sentimentAnnotation']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Utterance level video-ONLY analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following 'video-only analysis' code is present here due to dependency of timestamps from text dataset.\n",
    "This section performs analysis on only the video features extracted. \n",
    "\n",
    "The next section 'Machine Learning' contains both the video and text stacked using the 'early fusion' method. (See section 6.1 https://arxiv.org/pdf/1705.09406.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "# SVM model creation and fitting train vector to annotations\n",
    "model_tf_v = svm.SVC(kernel='linear', C=1, gamma=1).fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "\n",
    "# generate predictions\n",
    "predicted_tf_v = model_tf_v.predict(v_test_sparse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf_v))\n",
    "\n",
    "#create df to show results\n",
    "disp = y_tst.reset_index(drop=True).join(pd.DataFrame(predicted_tf_v,columns=['Prediction']))\n",
    "disp = disp.join(pd.DataFrame(disp['sentimentAnnotation']==disp['Prediction'],columns=['Right/Wrong']))\n",
    "scores = model_tf_v.score(v_test_sparse,y_tst['sentimentAnnotation'].values)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "print(\"Mean sentiment: {!r}. Predicted mean sentiment: {!r}.\".format('Positive' if disp['sentimentAnnotation'].mean()>=0 else 'Negative','Positive' if disp['Prediction'].mean()>=0 else 'Negative'))\n",
    "disp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cross validation of training set\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf_cv = svm.SVC(kernel='linear', C=1, gamma=1)\n",
    "scores = cross_val_score(clf_cv, v_train_sparse, y_trn['sentimentAnnotation'].values, cv=10)\n",
    "scores\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "model_lr = LogisticRegression().fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_lr = model_lr.predict(v_test_sparse)\n",
    "# Classification report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_lr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "model_dt = DecisionTreeClassifier().fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_dt = model_dt.predict(v_test_sparse)\n",
    "# Classification report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_dt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "model_rf = RandomForestClassifier().fit(v_train_sparse,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_rf = model_rf.predict(v_test_sparse)\n",
    "# Classification report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_rf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf_v))\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_lr))\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_dt))\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_rf))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Machine learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# countVectorizer initialization\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(analyzer = \"word\",   \\\n",
    "                             tokenizer = None,    \\\n",
    "                             preprocessor = None, \\\n",
    "                             stop_words = None,   \\\n",
    "                             lowercase = True,    \\\n",
    "                             max_features = 5000) \n",
    "\n",
    "# create bag of words vector for the training set using countVectorizer\n",
    "train_data_features = vectorizer.fit_transform(X_trn['Speech'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# transformation of test data\n",
    "test_data_features = vectorizer.transform(X_tst['Speech'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf-idf transformer initialization\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "# create tfidf transformed vector for the training set using tf-idf transformer\n",
    "X_train_tfidf = tfidf_transformer.fit_transform(train_data_features)\n",
    "X_test_tfidf = tfidf_transformer.transform(test_data_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Stacking Video to Text\n",
    "train_data_features_v = scipy.sparse.hstack([X_train_tfidf, v_train_sparse])\n",
    "test_data_features_v = scipy.sparse.hstack([X_test_tfidf, v_test_sparse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# SVM model creation and fitting train vector to annotations\n",
    "from sklearn import svm\n",
    "model_tf = svm.SVC(kernel='linear', C=1, gamma=1).fit(train_data_features_v,y_trn['sentimentAnnotation'].values)\n",
    "\n",
    "# generate predictions\n",
    "predicted_tf = model_tf.predict(test_data_features_v)\n",
    "\n",
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf))\n",
    "\n",
    "#create df to show results\n",
    "disp = X_tst.join(y_tst).reset_index(drop=True).join(pd.DataFrame(predicted_tf,columns=['Prediction']))\n",
    "disp = disp.join(pd.DataFrame(disp['sentimentAnnotation']==disp['Prediction'],columns=['Right/Wrong']))\n",
    "scores = model_tf.score(test_data_features_v,y_tst['sentimentAnnotation'].values)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "print(\"Mean sentiment: {!r}. Predicted mean sentiment: {!r}.\".format('Positive' if disp['sentimentAnnotation'].mean()>=0 else 'Negative','Positive' if disp['Prediction'].mean()>=0 else 'Negative'))\n",
    "disp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# cross validation of training set\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf_cv = svm.SVC(kernel='linear', C=1, gamma=1)\n",
    "scores = cross_val_score(clf_cv, train_data_features_v, y_trn['sentimentAnnotation'].values, cv=10)\n",
    "scores\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# LR model creation and fitting train vector to annotations\n",
    "model_tf = LogisticRegression().fit(train_data_features_v,y_trn['sentimentAnnotation'].values)\n",
    "# generate predictions\n",
    "predicted_tf = model_tf.predict(test_data_features_v)\n",
    "\n",
    "# Classification report\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_tst['sentimentAnnotation'].values, predicted_tf))\n",
    "\n",
    "#create df to show results\n",
    "disp = X_tst.join(y_tst).reset_index(drop=True).join(pd.DataFrame(predicted_tf,columns=['Prediction']))\n",
    "disp = disp.join(pd.DataFrame(disp['sentimentAnnotation']==disp['Prediction'],columns=['Right/Wrong']))\n",
    "scores = model_tf.score(test_data_features_v,y_tst['sentimentAnnotation'].values)\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
    "print(\"Mean sentiment: {!r}. Predicted mean sentiment: {!r}.\".format('Positive' if disp['sentimentAnnotation'].mean()>=0 else 'Negative','Positive' if disp['Prediction'].mean()>=0 else 'Negative'))\n",
    "disp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cross validation of training set\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf_cv = svm.SVC(kernel='linear', C=1, gamma=1)\n",
    "scores = cross_val_score(clf_cv, X_train_tfidf, y_trn['sentimentAnnotation'].values, cv=10)\n",
    "scores\n",
    "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}