artemis/notebooks/deep_nets/emotions/utterance_to_emotion_with_transformer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a text classifier (BERT-based) that predicts the underlying explained emotion.\n",
    "- This notebook requires that you have install the __simpletransformers__\n",
    "    - e.g., _pip install simpletransformers_\n",
    "- in 4 GPUs, this can take several hours (e.g., 18 hours)\n",
    "- Parameters are the same as in the paper (of course)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "import logging\n",
    "import pandas as pd\n",
    "import os.path as osp\n",
    "from simpletransformers.classification import ClassificationModel\n",
    "\n",
    "from artemis.emotions import ARTEMIS_EMOTIONS, IDX_TO_EMOTION, positive_negative_else\n",
    "from artemis.in_out.basics import create_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_labels = len(ARTEMIS_EMOTIONS)\n",
    "model_name =  'bert-base-uncased'\n",
    "load_best_model = False # assuming you have already done the training and you want to train it more, or evaluate!\n",
    "do_trainining = True\n",
    "\n",
    "max_train_epochs = 50\n",
    "subsample_data = False  # set to True if you want to test speed etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/optas/DATA/OUT/artemis/neural_nets/txt_to_emotion/bert_based/outputs/best_model'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# where the output model, logger etc. are or will be.\n",
    "my_out_dir = '/home/optas/DATA/OUT/artemis/neural_nets/txt_to_emotion/bert_based'\n",
    "create_dir(my_out_dir)\n",
    "best_model_dir = osp.join(my_out_dir, 'outputs', 'best_model')\n",
    "create_dir(best_model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "if load_best_model: # load already trained\n",
    "    model_name =  best_model_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### In cell below you need to use YOUR PATH.\n",
    "- I will use the pre-processed ArtEmis dataset; as prepared by the script __preprocess_artemis_data.py --preprocess-for-deep-nets True__ (see STEP.1 at top-README) \n",
    "- Specifically this way, I can utilize the same train/test/val splits accross all my neural-based experiments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessed_artemis = '/home/optas/DATA/OUT/artemis/preprocessed_data/for_neural_nets/artemis_preprocessed.csv'\n",
    "df = pd.read_csv(preprocessed_artemis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "# convert the data in the expected format for simpletransformers: 1)input-text, 2)label \n",
    "#\n",
    "data_splits = dict()\n",
    "for split in ['train', 'test', 'val']:\n",
    "    mask = (df['split'] == split)\n",
    "    sub_df = pd.concat([df.utterance_spelled[mask], df.emotion_label[mask]], axis=1)\n",
    "    # note that here I am not using my artemis.utils.vocabulary. Instead I rely on the default tokenization etc. of simpletransformers\n",
    "    sub_df.reset_index(drop=True, inplace=True)\n",
    "    sub_df.columns = [\"text\", \"labels\"]\n",
    "    if subsample_data:\n",
    "        sub_df = sub_df.sample(1000)\n",
    "    sub_df.reset_index(drop=True, inplace=True)\n",
    "    data_splits[split] = sub_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(level=logging.INFO)\n",
    "transformers_logger = logging.getLogger(\"transformers-last\")\n",
    "transformers_logger.setLevel(logging.WARNING)\n",
    "\n",
    "# Create a ClassificationModel\n",
    "args = {'reprocess_input_data': True, \n",
    "        'overwrite_output_dir': True,\n",
    "        'fp16': False,\n",
    "        'n_gpu': 4,\n",
    "        'save_model_every_epoch': False,\n",
    "        'evaluate_during_training': True,\n",
    "        'num_train_epochs': max_train_epochs,\n",
    "        'min_frequency': 5,\n",
    "        'train_batch_size': 128,\n",
    "        'output_dir': my_out_dir,\n",
    "        'cache_dir': my_out_dir,\n",
    "        'tensorboard_dir': my_out_dir,\n",
    "        'best_model_dir': best_model_dir,\n",
    "       }\n",
    "\n",
    "model = ClassificationModel('bert', model_name=model_name, num_labels=num_labels, args=args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model\n",
    "if do_trainining: \n",
    "    model.train_model(data_splits['train'], eval_df=data_splits['val'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_utils: Converting to features started. Cache is not used.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "597c5abffa554556a1e6dd15c99951f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=39850.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:simpletransformers.classification.classification_utils: Saving features into cached file /home/optas/DATA/OUT/artemis/neural_nets/txt_to_emotion/bert_based/cached_dev_bert_128_9_2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3caedfd39f8746e7b687306cc4b728eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=4982.0, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/optas/Tools/anaconda2/envs/artemis/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    }
   ],
   "source": [
    "# Evaluate the (last trained) model go below to use the per-validation optimal.\n",
    "result, model_outputs, wrong_predictions = model.eval_model(data_splits['test'],\n",
    "                                                            acc=sklearn.metrics.accuracy_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load best (trained) model  to do some evaluations\n",
    "model = ClassificationModel('bert', model_name=best_model_dir, num_labels=num_labels, args=args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate the model in terms of overall test accuracy\n",
    "result, model_outputs, wrong_predictions = model.eval_model(data_splits['test'], acc=sklearn.metrics.accuracy_score)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert Bert output to predicted maximizer\n",
    "predictions = model_outputs.argmax(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ternary prediction accuracy: 0.8759849435382685\n",
      "Pos-Neg (binary) prediction accuracy 0.91516403883715\n"
     ]
    }
   ],
   "source": [
    "## Test Accuracy for positive vs. negative emotions.\n",
    "dataset = data_splits['test']\n",
    "gt = dataset.labels\n",
    "\n",
    "gt_pne = gt.apply(lambda x: positive_negative_else(IDX_TO_EMOTION[x]))  # pos-neg-else conversion\n",
    "predictions_pne = pd.Series(predictions).apply(lambda x: positive_negative_else(IDX_TO_EMOTION[x]))\n",
    "print('Ternary prediction accuracy:', (gt_pne == predictions_pne).mean())\n",
    "\n",
    "# now, binary droping something-else\n",
    "se_label = positive_negative_else('something else')\n",
    "gt_pn = gt_pne[gt_pne != se_label]\n",
    "gt_pn.reset_index(drop=True, inplace=True)\n",
    "\n",
    "pred_pn = predictions_pne[(gt_pne != se_label).values]\n",
    "pred_pn.reset_index(drop=True, inplace=True)\n",
    "\n",
    "print('Pos-Neg (binary) prediction accuracy', (gt_pn == pred_pn).mean())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "artemis_dev",
   "language": "python",
   "name": "artemis_dev"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}