notebooks/models/Evaluate spacy models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate Spacy models for person names, orgs and locations using the Presidio Evaluator framework"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from copy import deepcopy\n",
    "from pprint import pprint\n",
    "from collections import Counter\n",
    "\n",
    "from presidio_evaluator import InputSample\n",
    "from presidio_evaluator.evaluation import Evaluator, ModelError\n",
    "from presidio_evaluator.models import SpacyModel\n",
    "from presidio_evaluator.experiment_tracking import get_experiment_tracker\n",
    "\n",
    "import pandas as pd\n",
    "import spacy\n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%% md\n"
    }
   },
   "source": [
    "Select data for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"synth_dataset_v2.json\"\n",
    "dataset = InputSample.read_dataset_json(\n",
    "    Path(Path.cwd().parent.parent, \"data\", dataset_name)\n",
    ")\n",
    "print(len(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_counter = Counter()\n",
    "for sample in dataset:\n",
    "    for tag in sample.tags:\n",
    "        entity_counter[tag] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Count per entity:\")\n",
    "pprint(entity_counter.most_common())\n",
    "\n",
    "print(\"\\nExample sentence:\")\n",
    "print(dataset[1])\n",
    "\n",
    "print(\"\\nMin and max number of tokens in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.tokens) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.tokens) for sample in dataset])}\"\n",
    ")\n",
    "\n",
    "print(\"\\nMin and max sentence length in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.full_text) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.full_text) for sample in dataset])}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select models for evaluation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\"en_core_web_sm\", \"en_core_web_lg\", \"en_core_web_trf\"]\n",
    "\n",
    "# If needed, install models using `python -m spacy download X` where x is the model name, or use spacy.cli.download:\n",
    "#spacy.cli.download(\"en_core_web_trf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run evaluation on all models:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "for model_name in models:\n",
    "    experiment = get_experiment_tracker()\n",
    "\n",
    "    print(\"-----------------------------------\")\n",
    "    print(f\"Evaluating model {model_name}\")\n",
    "\n",
    "    nlp = spacy.load(model_name)\n",
    "    model = SpacyModel(model=nlp, entities_to_keep=[\"PERSON\", \"GPE\", \"ORG\", \"NORP\"])\n",
    "    evaluator = Evaluator(model=model)\n",
    "    evaluation_results = evaluator.evaluate_all(deepcopy(dataset))\n",
    "    results = evaluator.calculate_score(evaluation_results)\n",
    "\n",
    "    # update params tracking\n",
    "    params = {\"dataset_name\": dataset_name, \"model_name\": model_name}\n",
    "    params.update(model.to_log())\n",
    "    experiment.log_parameters(params)\n",
    "    experiment.log_dataset_hash(dataset)\n",
    "    experiment.log_metrics(results.to_log())\n",
    "    entities, confmatrix = results.to_confusion_matrix()\n",
    "    experiment.log_confusion_matrix(matrix=confmatrix, labels=entities)\n",
    "\n",
    "    print(\"Confusion matrix:\")\n",
    "    print(pd.DataFrame(confmatrix, columns=entities, index=entities))\n",
    "\n",
    "    print(\"Precision and recall\")\n",
    "    print(results)\n",
    "\n",
    "    # end experiment\n",
    "    experiment.end()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Custom evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "# evaluate custom sentences (if exists)\n",
    "# nlp = spacy.load(spacy_ft_100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "sent = \"I am taiwanese but I live in Cambodia.\"\n",
    "# sent = input(\"Enter sentence: \")\n",
    "model.predict(InputSample(full_text=sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = results.model_errors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### False positives"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Most false positive tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "ModelError.most_common_fp_tokens(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "fps_df = ModelError.get_fps_dataframe(errors, entity=[\"GPE\"])\n",
    "fps_df[[\"full_text\", \"token\", \"prediction\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. False negative examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "ModelError.most_common_fn_tokens(errors, n=50, entity=[\"PERSON\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "More FN analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "fns_df = ModelError.get_fns_dataframe(errors, entity=[\"GPE\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "fns_df[[\"full_text\", \"token\", \"annotation\", \"prediction\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "print(\"All errors:\\n\")\n",
    "[print(error, \"\\n\") for error in errors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "presidio-evaluator",
   "language": "python",
   "name": "presidio-evaluator"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}