correlation_compution_between_annotators.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "correlation compution between annotators.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPBz1lKlpjDno2dUSon5b05",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/petermr/CEVOpen/blob/master/correlation_compution_between_annotators.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#example notebook\n",
        "This is a demo notebook which provide a glimse of statistical compuation that we are going to do after we have the data from the annotaters.\n",
        "\n",
        "Here lets assume that we have 5 annotaters,namely annotater1 to annotater5. we create dataframe consisting the data from the annotators.\n",
        "\n",
        "The cell below shows how the df is generated. If you want to explore and learn more about pandas please check this out here\n",
        "\n",
        "In this notebook we are experimenting how to compute the correlation between two categorical columns by using:\n",
        "1. correlation coefficient(Pearson Method)\n",
        "2. confusion Matrix\n",
        "3. Cohens kappa"
      ],
      "metadata": {
        "id": "0AuH1Z7h59Xm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#### import pandas and other required packages\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import confusion_matrix, cohen_kappa_score\n",
        "##### initializing a dataframe\n",
        "data= pd.DataFrame()\n",
        "\n",
        "##### labels provided by the annotator\n",
        "anno_1=['yes', 'no', 'no', 'no','yes']\n",
        "anno_2=['yes', 'no','yes', 'no', 'no']\n",
        "anno_3=['yes', 'no','yes','yes','yes']\n",
        "anno_4=['yes', 'no', 'no', 'no','yes']    # same as anno_1\n",
        "anno_5=['no', 'yes', 'yes', 'yes','no']   # offosite of anno_1\n",
        "\n",
        "#### adding labels provided by the annotators to the empty pandas dataframe 'data'\n",
        "data['Annotator_1']= anno_1\n",
        "data['Annotator_2']= anno_2\n",
        "data['Annotator_3']= anno_3\n",
        "data['Annotator_4']=anno_4\n",
        "data['Annotator_5']=anno_5\n",
        "print('data:\\n', data)\n",
        "\n",
        "\n",
        "###### Replacing 'yes' and 'no' in the dataframe with 1 and 0 resoectively for \n",
        "######              further computing the correlation coefficient \n",
        "data.replace(('yes', 'no'), (1, 0), inplace=True)\n",
        "print('\\n converted_data:\\n', data)\n",
        "\n",
        "\n",
        "#### printing  the correlation coefficient computed using Pearson Method.\n",
        "print('\\n Correlation matrix Pearson:\\n', data.corr(method = 'pearson'))\n",
        "# from sklearn.metrics import matthews_corrcoef\n",
        "# print('Correlation matrix using sklearn  between Annotator 1 and Annotator 2:\\n',matthews_corrcoef(data['Annotator_1'].to_numpy(),data['Annotator_2'].to_numpy()) )\n",
        "# print('Correlation matrix using sklearn between Annotator 2 and Annotator 3:\\n',matthews_corrcoef(data['Annotator_2'].to_numpy(),data['Annotator_3'].to_numpy()) )\n",
        "# print('Correlation matrix kendall:\\n', data.corr(method = 'kendall'))\n",
        "# print('Correlation matrix spearman:\\n', data.corr(method = 'spearman'))\n",
        "\n",
        "\n",
        "#### printing  the confusion matrix:\n",
        "print('\\n confuion matrix between Annotator_1 and Annotator_2:\\n',pd.DataFrame(confusion_matrix(anno_1,anno_2, labels=['yes', 'no']), index=['yes', 'no'], columns=['yes', 'no']))\n",
        "\n",
        "\n",
        "#### printing  the cohen kappa  matrix:\n",
        "#sklearn.metrics.cohen_kappa_score(y1, y2, *, labels=None, weights=None, sample_weight=None)\n",
        "cohen_kappa_matrix = np.zeros((5,5))\n",
        "annotater_list = [anno_1,anno_2,anno_3,anno_4,anno_5]\n",
        "for i, a1 in enumerate(annotater_list):\n",
        "  for j, a2 in enumerate(annotater_list):\n",
        "    cohen_kappa_matrix[i][j] = cohen_kappa_score(a1, a2)\n",
        "    #print(i,j,cohen_kappa_score(a1, a2))\n",
        "\n",
        "print('\\n cohens kappa matrix:\\n',pd.DataFrame(cohen_kappa_matrix , index=['Annotator_1', 'Annotator_2','Annotator_3', 'Annotator_4','Annotator_5'], columns=['Annotator_1', 'Annotator_2','Annotator_3', 'Annotator_4','Annotator_5']))\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lL3ddSxs_Xmg",
        "outputId": "e60d8816-11b6-426d-fd60-6f713c96380e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "data:\n",
            "   Annotator_1 Annotator_2 Annotator_3 Annotator_4 Annotator_5\n",
            "0         yes         yes         yes         yes          no\n",
            "1          no          no          no          no         yes\n",
            "2          no         yes         yes          no         yes\n",
            "3          no          no         yes          no         yes\n",
            "4         yes          no         yes         yes          no\n",
            "\n",
            " converted_data:\n",
            "    Annotator_1  Annotator_2  Annotator_3  Annotator_4  Annotator_5\n",
            "0            1            1            1            1            0\n",
            "1            0            0            0            0            1\n",
            "2            0            1            1            0            1\n",
            "3            0            0            1            0            1\n",
            "4            1            0            1            1            0\n",
            "\n",
            " Correlation matrix Pearson:\n",
            "              Annotator_1  Annotator_2  Annotator_3  Annotator_4  Annotator_5\n",
            "Annotator_1     1.000000     0.166667     0.408248     1.000000    -1.000000\n",
            "Annotator_2     0.166667     1.000000     0.408248     0.166667    -0.166667\n",
            "Annotator_3     0.408248     0.408248     1.000000     0.408248    -0.408248\n",
            "Annotator_4     1.000000     0.166667     0.408248     1.000000    -1.000000\n",
            "Annotator_5    -1.000000    -0.166667    -0.408248    -1.000000     1.000000\n",
            "\n",
            " confuion matrix between Annotator_1 and Annotator_2:\n",
            "      yes  no\n",
            "yes    1   1\n",
            "no     1   2\n",
            "\n",
            " cohens kappa matrix:\n",
            "              Annotator_1  Annotator_2  Annotator_3  Annotator_4  Annotator_5\n",
            "Annotator_1     1.000000     0.166667     0.285714     1.000000    -0.923077\n",
            "Annotator_2     0.166667     1.000000     0.285714     0.166667    -0.153846\n",
            "Annotator_3     0.285714     0.285714     1.000000     0.285714    -0.363636\n",
            "Annotator_4     1.000000     0.166667     0.285714     1.000000    -0.923077\n",
            "Annotator_5    -0.923077    -0.153846    -0.363636    -0.923077     1.000000\n"
          ]
        }
      ]
    }
  ]
}