notebooks/models/Evaluate azure text analytics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate Azure Cognitive Services for languages to identify PII using the Presidio Evaluator framework\n",
    "\n",
    "Prerequisites: \n",
    " - Azure subscription - [Create one for free](https://azure.microsoft.com/en-us/free/cognitive-services/)\n",
    " - Once you have your Azure subscription, create a Language resource in the Azure portal to get your key and endpoint. After it deploys, click Go to resource.\n",
    " - You'll need the key and endpoint from the resource you create to connect your application to the API. You'll paste your key and endpoint into the code below later in the quickstart.\n",
    " - You can use the free pricing tier (Free F0) to try the service, and upgrade later to a paid tier for production.\n",
    " - To use the Analyze feature, you'll need a Language resource with the standard (S) pricing tier.\n",
    "\n",
    "Azure Cognitive Services for languages quickstart: https://learn.microsoft.com/en-us/azure/cognitive-services/language-service/personally-identifiable-information/quickstart?pivots=programming-language-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from copy import deepcopy\n",
    "from pprint import pprint\n",
    "from collections import Counter\n",
    "\n",
    "from presidio_evaluator import InputSample\n",
    "from presidio_evaluator.evaluation import Evaluator, ModelError\n",
    "from presidio_evaluator.models import TextAnalyticsWrapper\n",
    "from presidio_evaluator.experiment_tracking import get_experiment_tracker\n",
    "import pandas as pd\n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select data for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = \"synth_dataset_v2.json\"\n",
    "dataset = InputSample.read_dataset_json(Path(Path.cwd().parent.parent, \"data\", dataset_name))\n",
    "print(len(dataset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_counter = Counter()\n",
    "for sample in dataset:\n",
    "    for tag in sample.tags:\n",
    "        entity_counter[tag] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Count per entity:\")\n",
    "pprint(entity_counter.most_common())\n",
    "\n",
    "print(\"\\nExample sentence:\")\n",
    "print(dataset[1])\n",
    "\n",
    "print(\"\\nMin and max number of tokens in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.tokens) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.tokens) for sample in dataset])}\"\n",
    ")\n",
    "\n",
    "print(\"\\nMin and max sentence length in dataset:\")\n",
    "print(\n",
    "    f\"Min: {min([len(sample.full_text) for sample in dataset])}, \"\n",
    "    f\"Max: {max([len(sample.full_text) for sample in dataset])}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"Text analytics Analyzer\"\n",
    "# Paste your Azure Text Analytics key and endpoint here\n",
    "key = \"XXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n",
    "endpoint = \"https://xxxxxxxxxxx.cognitiveservices.azure.com/\"\n",
    "model = TextAnalyticsWrapper(ta_key=key, ta_endpoint=endpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Evaluating Azure Text Analytics.\")\n",
    "\n",
    "experiment = get_experiment_tracker()\n",
    "\n",
    "# Mapping from dataset Entities to Text Analytics Entities. \n",
    "# All supported PII entity categories in Text Analytics are listed in this link: https://learn.microsoft.com/en-us/azure/cognitive-services/language-service/personally-identifiable-information/concepts/conversations-entity-categories\n",
    "i2b2_entities_to_text_analytics =  {\"PERSON\":\"Person\",\n",
    "                                \"STREET_ADDRESS\":\"Address\",\n",
    "                                \"GPE\": \"O\",\n",
    "                                \"PHONE_NUMBER\":\"PhoneNumber\",\n",
    "                                \"ORGANIZATION\":\"Organization\",\n",
    "                                \"DATE_TIME\": \"DateTime\",\n",
    "                                \"TITLE\":\"O\",\n",
    "                                \"CREDIT_CARD\":\"CreditCardNumber\",\n",
    "                                \"US_SSN\":\"USSocialSecurityNumber\",\n",
    "                                \"AGE\": \"Age\",\n",
    "                                \"NRP\":\"O\",\n",
    "                                \"ZIP_CODE\":\"O\",\n",
    "                                \"EMAIL_ADDRESS\":\"Email\",\n",
    "                                \"DOMAIN_NAME\":\"URL\",\n",
    "                                \"IP_ADDRESS\":\"IPAddress\",\n",
    "                                \"IBAN_CODE\":\"InternationalBankingAccountNumber\",   \n",
    "                                \"US_DRIVER_LICENSE\":\"USDriversLicenseNumber\"\n",
    "                                }\n",
    "# List of entity names to focus the evaluator on (and ignore the rest) is defined with entities_to_keep parameter\n",
    "evaluator = Evaluator(model=model, entities_to_keep=[\"Person\", \"Address\"])\n",
    "dataset_ = Evaluator.align_entity_types(\n",
    "    deepcopy(dataset), entities_mapping=i2b2_entities_to_text_analytics\n",
    ")\n",
    "\n",
    "evaluation_results = evaluator.evaluate_all(dataset_)\n",
    "results = evaluator.calculate_score(evaluation_results)\n",
    "\n",
    "# update params tracking\n",
    "params = {\"dataset_name\": dataset_name, \"model_name\": model_name}\n",
    "params.update(model.to_log())\n",
    "experiment.log_parameters(params)\n",
    "experiment.log_dataset_hash(dataset)\n",
    "experiment.log_metrics(results.to_log())\n",
    "entities, confmatrix = results.to_confusion_matrix()\n",
    "experiment.log_confusion_matrix(matrix=confmatrix, labels=entities)\n",
    "\n",
    "# end experiment\n",
    "experiment.end()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Confusion matrix:\")\n",
    "print(pd.DataFrame(confmatrix, columns=entities, index=entities))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Precision and recall\")\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "vscode": {
   "interpreter": {
    "hash": "371968787ec79dd50357533864944a85029366968470cac36beb694745c2f7d6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}