notebooks/Pre-Trained Model Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "\n",
    "from bella import helper\n",
    "from bella.models.tdlstm import LSTM, TDLSTM, TCLSTM\n",
    "from bella.models.target import TargetDep\n",
    "from bella.parsers import semeval_14"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How to download and use the models\n",
    "\n",
    "THis notebook gives a brief example of how to download the pre-trained model and how to apply it to your own data. We will also show how to apply them to the SemEval dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download the model that you want\n",
    "lstm = helper.download_model(LSTM, 'SemEval 14 Restaurant')\n",
    "tdlstm = helper.download_model(TDLSTM, 'SemEval 14 Restaurant')\n",
    "tclstm = helper.download_model(TCLSTM, 'SemEval 14 Restaurant')\n",
    "target_dep = helper.download_model(TargetDep, 'SemEval 14 Restaurant')\n",
    "models = [lstm, tdlstm, tclstm, target_dep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_example_pos = [{'text' : 'This bread is tasty', 'target': 'bread', \n",
    "                     'spans': [(5, 10)]}]\n",
    "test_example_neg = [{'text' : 'This bread is burnt', 'target': 'bread', \n",
    "                     'spans': [(5, 10)]}]\n",
    "test_example_multi = [{'text' : 'This bread is tasty but the sauce is too rich', 'target': 'sauce', \n",
    "                     'spans': [(28, 33)]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: LSTM\n",
      "\t Positive correct: True\n",
      "\t Negative correct: False\n",
      "\t Multi correct: False\n",
      "\n",
      "Model: TDLSTM\n",
      "\t Positive correct: True\n",
      "\t Negative correct: False\n",
      "\t Multi correct: False\n",
      "\n",
      "Model: TCLSTM\n",
      "\t Positive correct: True\n",
      "\t Negative correct: False\n",
      "\t Multi correct: False\n",
      "\n",
      "Model: Target Dependent\n",
      "\t Positive correct: True\n",
      "\t Negative correct: True\n",
      "\t Multi correct: True\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sentiment_mapper = {0: -1, 1: 0, 2: 1}\n",
    "for model in models:\n",
    "    pos_pred = model.predict(test_example_pos)[0]\n",
    "    neg_pred = model.predict(test_example_neg)[0]\n",
    "    multi_pred = model.predict(test_example_multi)[0]\n",
    "    if 'LSTM' in model.name():\n",
    "        pos_pred = sentiment_mapper[pos_pred]\n",
    "        neg_pred = sentiment_mapper[neg_pred]\n",
    "        multi_pred = sentiment_mapper[multi_pred]\n",
    "    print(f'Model: {model.name()}\\n\\t Positive correct: {pos_pred==1}\\n\\t'\n",
    "          f' Negative correct: {neg_pred==-1}\\n\\t Multi correct: {multi_pred==-1}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As shown above the LSTM models require a sentiment mapper to change the values to the -1, 0, and 1 scaling rather than 0, 1, 2.\n",
    "\n",
    "## How to apply them to a dataset\n",
    "\n",
    "We are going to apply the methods to the SemEval 2014 Resturant Test dataset that can be found [here](http://metashare.ilsp.gr:8080/repository/browse/semeval-2014-absa-test-data-gold-annotations/b98d11cec18211e38229842b2b6a04d77591d40acd7542b7af823a54fb03a155/). You do need to download the dataset to do this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to the SemEVal 2014 Resturant test data set\n",
    "test_data_path = Path('..', '..', 'ABSA_Gold_TestData', 'Restaurants_Test_Gold.xml')\n",
    "# Load the dataset\n",
    "test_data = semeval_14(test_data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/andrew/Envs/Bella/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: LSTM, Macro F1: 46.36 Accuracy: 71.07\n",
      "\n",
      "Model: TDLSTM, Macro F1: 57.68 Accuracy: 74.38\n",
      "\n",
      "Model: TCLSTM, Macro F1: 55.38 Accuracy: 72.41\n",
      "\n",
      "Model: Target Dependent, Macro F1: 56.16 Accuracy: 73.84\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sentiment_mapper = {0: -1, 1: 0, 2: 1}\n",
    "for model in models:\n",
    "    predictions = model.predict(test_data.data_dict())\n",
    "    if 'LSTM' in model.name():\n",
    "        predictions = [sentiment_mapper[pred] for pred in predictions]\n",
    "    y_test = test_data.sentiment_data()\n",
    "    f1 = f1_score(y_test, predictions, average='macro') * 100\n",
    "    acc = accuracy_score(y_test, predictions) * 100\n",
    "    print(f'Model: {model.name()}, Macro F1: {f1:.2f} '\n",
    "          f'Accuracy: {acc:.2f}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}