reading_annotator_label_and_correlation_computation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "reading annotator label and correlation computation.ipynb",
      "provenance": [],
      "mount_file_id": "1F3w-SyBE1XcHKqIH-HxQldv1aI_nKUy6",
      "authorship_tag": "ABX9TyO5VMJgJf65N7BZG55gbXoD",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/petermr/CEVOpen/blob/master/reading_annotator_label_and_correlation_computation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Roi53uKdT8Io"
      },
      "outputs": [],
      "source": [
        "#### import pandas and other required packages\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import os\n",
        "from sklearn.metrics import confusion_matrix, cohen_kappa_score\n",
        "import random"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Demostration of correlation Computation on Demo Data.\n",
        "This is a demo notebook which provide a glimse of statistical compuation that we are going to do after we have the data from the annotaters.\n",
        "\n",
        "#Description of processes in the cell below:\n",
        "Here lets assume that we have 5 annotaters,namely annotater1 to annotater5. we create dataframe consisting the data from the annotators.\n",
        "\n",
        "The cell below shows how the df is generated. If you want to explore and learn more about pandas please check this out [here](https://github.com/petermr/CEVOpen/blob/master/fun_with_pandas.ipynb)\n",
        "\n",
        "In this notebook we are experimenting how to compute the correlation between two categorical columns by using:\n",
        "1. [correlation coefficien ](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.corr.html)\n",
        "2. [Confusion Matrix](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html)\n",
        "3. [Cohens kappa](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.cohen_kappa_score.html)"
      ],
      "metadata": {
        "id": "kGfLiU__eCld"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#### import pandas and other required packages\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.metrics import confusion_matrix, cohen_kappa_score\n",
        "##### initializing a dataframe\n",
        "data= pd.DataFrame()\n",
        "\n",
        "##### labels provided by the annotator\n",
        "anno_1=['yes', 'no', 'no', 'no','yes']\n",
        "anno_2=['yes', 'no','yes', 'no', 'no']\n",
        "anno_3=['yes', 'no','yes','yes','yes']\n",
        "anno_4=['yes', 'no', 'no', 'no','yes']    # same as anno_1\n",
        "anno_5=['no', 'yes', 'yes', 'yes','no']   # offosite of anno_1\n",
        "\n",
        "#### adding labels provided by the annotators to the empty pandas dataframe 'data'\n",
        "data['Annotator_1']= anno_1\n",
        "data['Annotator_2']= anno_2\n",
        "data['Annotator_3']= anno_3\n",
        "data['Annotator_4']=anno_4\n",
        "data['Annotator_5']=anno_5\n",
        "print('data:\\n', data)\n",
        "\n",
        "\n",
        "###### Replacing 'yes' and 'no' in the dataframe with 1 and 0 resoectively for \n",
        "######              further computing the correlation coefficient \n",
        "data.replace(('yes', 'no'), (1, 0), inplace=True)\n",
        "print('\\n converted_data:\\n', data)\n",
        "\n",
        "\n",
        "#### printing  the correlation coefficient computed using Pearson Method.\n",
        "print('\\n Correlation matrix Pearson:\\n', data.corr(method = 'pearson'))\n",
        "# from sklearn.metrics import matthews_corrcoef\n",
        "# print('Correlation matrix using sklearn  between Annotator 1 and Annotator 2:\\n',matthews_corrcoef(data['Annotator_1'].to_numpy(),data['Annotator_2'].to_numpy()) )\n",
        "# print('Correlation matrix using sklearn between Annotator 2 and Annotator 3:\\n',matthews_corrcoef(data['Annotator_2'].to_numpy(),data['Annotator_3'].to_numpy()) )\n",
        "# print('Correlation matrix kendall:\\n', data.corr(method = 'kendall'))\n",
        "# print('Correlation matrix spearman:\\n', data.corr(method = 'spearman'))\n",
        "\n",
        "\n",
        "#### printing  the confusion matrix:\n",
        "print('\\n confuion matrix between Annotator_1 and Annotator_2:\\n',pd.DataFrame(confusion_matrix(anno_1,anno_2, labels=['yes', 'no']), index=['yes', 'no'], columns=['yes', 'no']))\n",
        "\n",
        "\n",
        "#### printing  the cohen kappa  matrix:\n",
        "#sklearn.metrics.cohen_kappa_score(y1, y2, *, labels=None, weights=None, sample_weight=None)\n",
        "cohen_kappa_matrix = np.zeros((5,5))\n",
        "annotater_list = [anno_1,anno_2,anno_3,anno_4,anno_5]\n",
        "for i, a1 in enumerate(annotater_list):\n",
        "  for j, a2 in enumerate(annotater_list):\n",
        "    cohen_kappa_matrix[i][j] = cohen_kappa_score(a1, a2)\n",
        "    #print(i,j,cohen_kappa_score(a1, a2))\n",
        "\n",
        "print('\\n cohens kappa matrix:\\n',pd.DataFrame(cohen_kappa_matrix , index=['Annotator_1', 'Annotator_2','Annotator_3', 'Annotator_4','Annotator_5'], columns=['Annotator_1', 'Annotator_2','Annotator_3', 'Annotator_4','Annotator_5']))\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lL3ddSxs_Xmg",
        "outputId": "268f66e6-b0b6-45b2-adb3-6a54689c5057"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "data:\n",
            "   Annotator_1 Annotator_2 Annotator_3 Annotator_4 Annotator_5\n",
            "0         yes         yes         yes         yes          no\n",
            "1          no          no          no          no         yes\n",
            "2          no         yes         yes          no         yes\n",
            "3          no          no         yes          no         yes\n",
            "4         yes          no         yes         yes          no\n",
            "\n",
            " converted_data:\n",
            "    Annotator_1  Annotator_2  Annotator_3  Annotator_4  Annotator_5\n",
            "0            1            1            1            1            0\n",
            "1            0            0            0            0            1\n",
            "2            0            1            1            0            1\n",
            "3            0            0            1            0            1\n",
            "4            1            0            1            1            0\n",
            "\n",
            " Correlation matrix Pearson:\n",
            "              Annotator_1  Annotator_2  Annotator_3  Annotator_4  Annotator_5\n",
            "Annotator_1     1.000000     0.166667     0.408248     1.000000    -1.000000\n",
            "Annotator_2     0.166667     1.000000     0.408248     0.166667    -0.166667\n",
            "Annotator_3     0.408248     0.408248     1.000000     0.408248    -0.408248\n",
            "Annotator_4     1.000000     0.166667     0.408248     1.000000    -1.000000\n",
            "Annotator_5    -1.000000    -0.166667    -0.408248    -1.000000     1.000000\n",
            "\n",
            " confuion matrix between Annotator_1 and Annotator_2:\n",
            "      yes  no\n",
            "yes    1   1\n",
            "no     1   2\n",
            "\n",
            " cohens kappa matrix:\n",
            "              Annotator_1  Annotator_2  Annotator_3  Annotator_4  Annotator_5\n",
            "Annotator_1     1.000000     0.166667     0.285714     1.000000    -0.923077\n",
            "Annotator_2     0.166667     1.000000     0.285714     0.166667    -0.153846\n",
            "Annotator_3     0.285714     0.285714     1.000000     0.285714    -0.363636\n",
            "Annotator_4     1.000000     0.166667     0.285714     1.000000    -0.923077\n",
            "Annotator_5    -0.923077    -0.153846    -0.363636    -0.923077     1.000000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Correlation Computation on LOOK-ALIKE of Original Data.\n",
        "Original Data is not yet made public. But we have used a similar looking data for the experimentation.\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "#### **Description of LOOK-ALIKE Data:**\n",
        "No of Annotators are 7\n",
        "\n",
        "No of fig labeled are 30\n",
        "\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "\n",
        "The methodolgy we are using is similar to the above cell. Though the data here is very similar to original data.\n",
        "\n",
        "The function **create_df **----> helps in genarating pandas dataframe from all the csv files provided by annotators of a concerned class_label(ex: Terpene)\n",
        "\n",
        "The cell below shows how you can extract labels from annotators csv for the **Terpene** class and do statistical computation to analyse the performance of each annotator in comparision to another.\n",
        "\n",
        "\n",
        "THe similar approach can be done to obtain information regarding other labels i.e. Pathway, Metabolic, Pre-Terpene.\n",
        "\n"
      ],
      "metadata": {
        "id": "et5pAJWmeLvg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#### downloading LOOK-ALIKE Data:\n",
        "import gdown\n",
        "url = 'https://drive.google.com/uc?id=119f8fhsX3bKyIII9fWoKVKkDoYDBMeSI'\n",
        "output = 'files.zip'\n",
        "gdown.download(url, output, quiet=False)\n",
        "!unzip /content/files.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CwJmd5_tondc",
        "outputId": "beab4283-75da-4946-941b-1790c20feb9b"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=119f8fhsX3bKyIII9fWoKVKkDoYDBMeSI\n",
            "To: /content/files.zip\n",
            "100%|██████████| 9.25k/9.25k [00:00<00:00, 1.35MB/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  /content/files.zip\n",
            "  inflating: look_alike_data/annotators_label/annotations_from_annotators5.csv  \n",
            "  inflating: look_alike_data/annotators_label/annotations_from_annotators4.csv  \n",
            "  inflating: look_alike_data/annotators_label/annotations_from_annotators2.csv  \n",
            "  inflating: look_alike_data/annotators_label/annotations_from_annotators3.csv  \n",
            "  inflating: look_alike_data/annotators_label/annotations_from_annotators6.csv  \n",
            "  inflating: look_alike_data/annotators_label/annotations_from_annotators7.csv  \n",
            "  inflating: look_alike_data/mapping_info/mapping_info_for_calss_label_Metabolic.csv  \n",
            "  inflating: look_alike_data/mapping_info/mapping_info_for_calss_label_Pre-Terpene.csv  \n",
            "  inflating: look_alike_data/mapping_info/mapping_info_for_calss_label_Terpene.csv  \n",
            "  inflating: look_alike_data/mapping_info/mapping_info_for_calss_label_Pathway.csv  \n",
            "  inflating: look_alike_data/annotators_label/annotations_from_annotators1.csv  \n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#### Function to generate the dataframe from the classification csv's provided by annotators \n",
        "def create_df(path, class_label):\n",
        "  \n",
        "  list_files = os.listdir(path)            # storing the files names from all the annotators available in directory provided by path\n",
        "  data= pd.DataFrame()                     # empty dataframe to store the accumulated data\n",
        "  map_list=[]                              # empty list to store mappings of dataframe column name to original csv names\n",
        "  \n",
        "  ## Iterating through all the csv provided by annotators and saving ut into the dataframe \"data\".\n",
        "  for i, name in enumerate(list_files):\n",
        "    sub_list=[]\n",
        "    df = pd.read_csv(path +name)                               # reading annotator file\n",
        "    sub_list.append('Annotator_'+ str(i))                      # appending dataframe column name(like:Annotator_1)  to the empty sub_list \n",
        "    sub_list.append(name)                                      # appending origibnal file name  to the empty sub_list \n",
        "    data['Annotator_'+ str(i)] = df[class_label].to_numpy()    # adding columns to data each having labels from respective annotator on the converned class label(ex: Terpene)\n",
        "    map_list.append(sub_list)                                  # appending the sub_kist to map_list\n",
        "  \n",
        "\n",
        "  ## saving the map_list\n",
        "  map_df = pd.DataFrame(map_list)\n",
        "  map_save_path = os.path.dirname(os.path.dirname(path)) +'/mapping_info/'           # path where mapping info is saved under the parent directory with folder name \"mapping_info\"\n",
        "  if not os.path.exists(map_save_path):                                              # checking id already a folder exists with path - map_save_path\n",
        "    os.makedirs(map_save_path)\n",
        "  map_df.to_csv(map_save_path +'mapping_info_for_calss_label_'+class_label+'.csv')   #  saving the mapping list into the direct path\n",
        "  return data, map_list "
      ],
      "metadata": {
        "id": "nqcbgSqPUgVT"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## For class label - Terpene\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "x6JOCupOlqUs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "####define path of your folder where the data from annotator sare stored\n",
        "path = '/content/look_alike_data/annotators_label/'\n",
        "\n",
        "#### define the class label you are currently interested in\n",
        "class_label = 'Terpene'\n",
        "data_terpene, mapping = create_df(path, class_label)\n",
        "data_terpene, mapping\n",
        "\n",
        "####printting dataframe generated\n",
        "#print('data_terpene:\\n',data_terpene)\n",
        "\n",
        "\n",
        "#### information about data_terpene\n",
        "print('\\nNo of Annotators are: ',data_terpene.shape[1])\n",
        "print('No of figures labeled by the annotators are: ',data_terpene.shape[0])\n",
        "print('Shape of the DataFrame \"data_terpene\" is : ',data_terpene.shape)\n",
        "print('\\nTop 10 entries of data_terpene are\\n: ',data_terpene.head(10))\n",
        "\n",
        "#### Converting data_terpene_converted to binary 0s and 1s\n",
        "data_terpene_converted = data_terpene.replace(('yes', 'no'), (1, 0), inplace=False)\n",
        "#print('\\n data_terpene_converted for terpene:\\n', data_terpene_converted)\n",
        "\n",
        "\n",
        "#### printing  the correlation coefficient computed using Pearson Method uding corr function from pandas\n",
        "print('\\n Correlation matrix Pearson for class label Terpene:\\n', data_terpene_converted.corr(method = 'pearson'))\n",
        "\n",
        "\n",
        "#### To print  the correlation coefficient computed using Kendal and Spearman Method follow the lines of code below.\n",
        "# print('Correlation matrix kendall:\\n', data.corr(method = 'kendall'))\n",
        "# print('Correlation matrix spearman:\\n', data.corr(method = 'spearman'))\n",
        "\n",
        "\n",
        "#### printing  the confusion matrix :\n",
        "print('\\nConfuion matrix between Annotator_1 and Annotator_2 for class label Terpene:\\n',\n",
        "      pd.DataFrame(confusion_matrix(data_terpene.Annotator_1.to_numpy(),\n",
        "      data_terpene.Annotator_2.to_numpy(), labels=['yes', 'no']), index=['yes', 'no'], columns=['yes', 'no']))\n",
        "'''      \n",
        "       Any combination of Annotators for confusion matrix can be chosen to print:\n",
        "            replace the pair ------> data_terpene.Annotator_1.to_numpy(),data_terpene.Annotator_2.to_numpy()\n",
        "            with whatever Annotator combination you wanna inspect. Lets say Annotator_2 and Annotator_6,\n",
        "                 Then new pair will be :\n",
        "                    data_terpene.Annotator_2.to_numpy(),data_terpene.Annotator_6.to_numpy()\n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "#### printing  the cohen kappa  matrix:\n",
        "\n",
        "total_annotators = data_terpene.shape[1]\n",
        "cohen_kappa_matrix = np.zeros((total_annotators,total_annotators))\n",
        "annotater_list = data_terpene.columns\n",
        "for i, a1 in enumerate(annotater_list):\n",
        "  for j, a2 in enumerate(annotater_list):\n",
        "    cohen_kappa_matrix[i][j] = cohen_kappa_score(data_terpene[a1], data_terpene[a2])\n",
        "    #print(i,j,cohen_kappa_score(a1, a2))\n",
        "\n",
        "print('\\nCohens kappa matrix for class label Terpene:\\n',pd.DataFrame(cohen_kappa_matrix , index= data_terpene.columns, columns=data_terpene.columns))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iByQIcTkUjsC",
        "outputId": "396ffdfe-c4a2-42ea-904a-e191b4318aa5"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "No of Annotators are:  7\n",
            "No of figures labeled by the annotators are:  30\n",
            "Shape of the DataFrame \"data_terpene\" is :  (30, 7)\n",
            "\n",
            "Top 10 entries of data_terpene are\n",
            ":    Annotator_0 Annotator_1 Annotator_2  ... Annotator_4 Annotator_5 Annotator_6\n",
            "0         yes          no         yes  ...          no          no         yes\n",
            "1          no          no         yes  ...         yes         yes         yes\n",
            "2         yes         yes         yes  ...         yes         yes         yes\n",
            "3          no          no          no  ...          no          no          no\n",
            "4         yes          no         yes  ...          no          no          no\n",
            "5         yes         yes         yes  ...          no         yes          no\n",
            "6          no          no         yes  ...          no         yes         yes\n",
            "7          no          no         yes  ...          no         yes         yes\n",
            "8          no          no          no  ...         yes          no         yes\n",
            "9         yes         yes          no  ...         yes          no          no\n",
            "\n",
            "[10 rows x 7 columns]\n",
            "\n",
            " Correlation matrix Pearson for class label Terpene:\n",
            "              Annotator_0  Annotator_1  ...  Annotator_5  Annotator_6\n",
            "Annotator_0     1.000000     0.157138  ...    -0.267261    -0.062500\n",
            "Annotator_1     0.157138     1.000000  ...     0.207514    -0.018487\n",
            "Annotator_2     0.120165    -0.138756  ...     0.484200     0.018487\n",
            "Annotator_3    -0.200100    -0.070869  ...    -0.078811     0.358073\n",
            "Annotator_4     0.042126     0.234413  ...     0.078811    -0.042126\n",
            "Annotator_5    -0.267261     0.207514  ...     1.000000     0.133631\n",
            "Annotator_6    -0.062500    -0.018487  ...     0.133631     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n",
            "\n",
            "Confuion matrix between Annotator_1 and Annotator_2 for class label Terpene:\n",
            "      yes  no\n",
            "yes    6   5\n",
            "no    13   6\n",
            "\n",
            "Cohens kappa matrix for class label Terpene:\n",
            "              Annotator_0  Annotator_1  ...  Annotator_5  Annotator_6\n",
            "Annotator_0     1.000000     0.148472  ...    -0.266667    -0.061947\n",
            "Annotator_1     0.148472     1.000000  ...     0.200000    -0.018100\n",
            "Annotator_2     0.117647    -0.120332  ...     0.466667     0.017467\n",
            "Annotator_3    -0.175115    -0.050584  ...    -0.066667     0.291845\n",
            "Annotator_4     0.034335     0.222798  ...     0.066667    -0.036866\n",
            "Annotator_5    -0.266667     0.200000  ...     1.000000     0.133333\n",
            "Annotator_6    -0.061947    -0.018100  ...     0.133333     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## For class label - Pathway"
      ],
      "metadata": {
        "id": "D2Otkwosmz-F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "####define path of your folder where the data from annotator sare stored\n",
        "path = '/content/look_alike_data/annotators_label/'\n",
        "\n",
        "#### define the class label you are currently interested in\n",
        "class_label = 'Pathway'\n",
        "data_pathway, mapping = create_df(path, class_label)\n",
        "data_pathway, mapping\n",
        "\n",
        "####printting dataframe generated\n",
        "#print('data_pathway:\\n',data_pathway)\n",
        "\n",
        "\n",
        "#### information about data_pathway\n",
        "print('\\nNo of Annotators are: ',data_pathway.shape[1])\n",
        "print('No of figures labeled by the annotators are: ',data_pathway.shape[0])\n",
        "print('Shape of the DataFrame \"data_pathway\" is : ',data_pathway.shape)\n",
        "print('\\nTop 10 entries of data_pathway are\\n: ',data_pathway.head(10))\n",
        "\n",
        "#### Converting data_pathway_converted to binary 0s and 1s\n",
        "data_pathway_converted = data_pathway.replace(('yes', 'no'), (1, 0), inplace=False)\n",
        "#print('\\n data_pathway_converted for pathway:\\n', data_pathway_converted)\n",
        "\n",
        "\n",
        "#### printing  the correlation coefficient computed using Pearson Method uding corr function from pandas\n",
        "print('\\n Correlation matrix Pearson for class label pathway:\\n', data_pathway_converted.corr(method = 'pearson'))\n",
        "\n",
        "\n",
        "#### To print  the correlation coefficient computed using Kendal and Spearman Method follow the lines of code below.\n",
        "# print('Correlation matrix kendall:\\n', data.corr(method = 'kendall'))\n",
        "# print('Correlation matrix spearman:\\n', data.corr(method = 'spearman'))\n",
        "\n",
        "\n",
        "#### printing  the confusion matrix :\n",
        "print('\\nConfuion matrix between Annotator_1 and Annotator_2 for class label pathway:\\n',\n",
        "      pd.DataFrame(confusion_matrix(data_pathway.Annotator_1.to_numpy(),\n",
        "      data_pathway.Annotator_2.to_numpy(), labels=['yes', 'no']), index=['yes', 'no'], columns=['yes', 'no']))\n",
        "'''      \n",
        "       Any combination of Annotators for confusion matrix can be chosen to print:\n",
        "            replace the pair ------> data_pathway.Annotator_1.to_numpy(),data_pathway.Annotator_2.to_numpy()\n",
        "            with whatever Annotator combination you wanna inspect. Lets say Annotator_2 and Annotator_6,\n",
        "                 Then new pair will be :\n",
        "                    data_pathway.Annotator_2.to_numpy(),data_pathway.Annotator_6.to_numpy()\n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "#### printing  the cohen kappa  matrix:\n",
        "\n",
        "total_annotators = data_pathway.shape[1]\n",
        "cohen_kappa_matrix = np.zeros((total_annotators,total_annotators))\n",
        "annotater_list = data_pathway.columns\n",
        "for i, a1 in enumerate(annotater_list):\n",
        "  for j, a2 in enumerate(annotater_list):\n",
        "    cohen_kappa_matrix[i][j] = cohen_kappa_score(data_pathway[a1], data_pathway[a2])\n",
        "    #print(i,j,cohen_kappa_score(a1, a2))\n",
        "\n",
        "print('\\nCohens kappa matrix for class label pathway:\\n',pd.DataFrame(cohen_kappa_matrix , index= data_pathway.columns, columns=data_pathway.columns))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "639e9763-49e1-4e7f-d685-81ffdef4e2a1",
        "id": "dNnjOj4Zl1CF"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "No of Annotators are:  7\n",
            "No of figures labeled by the annotators are:  30\n",
            "Shape of the DataFrame \"data_pathway\" is :  (30, 7)\n",
            "\n",
            "Top 10 entries of data_pathway are\n",
            ":    Annotator_0 Annotator_1 Annotator_2  ... Annotator_4 Annotator_5 Annotator_6\n",
            "0          no         yes         yes  ...         yes          no          no\n",
            "1          no         yes          no  ...          no          no         yes\n",
            "2         yes          no          no  ...          no         yes          no\n",
            "3          no         yes         yes  ...          no          no         yes\n",
            "4         yes          no         yes  ...          no         yes         yes\n",
            "5          no          no         yes  ...          no          no          no\n",
            "6          no          no         yes  ...          no         yes         yes\n",
            "7         yes          no         yes  ...          no         yes          no\n",
            "8         yes          no          no  ...          no         yes          no\n",
            "9          no          no          no  ...          no         yes          no\n",
            "\n",
            "[10 rows x 7 columns]\n",
            "\n",
            " Correlation matrix Pearson for class label pathway:\n",
            "               Annotator_0  Annotator_1  ...  Annotator_5  Annotator_6\n",
            "Annotator_0  1.000000e+00    -0.154303  ...     0.332956    -0.047246\n",
            "Annotator_1 -1.543033e-01     1.000000  ...    -0.161468     0.029161\n",
            "Annotator_2  3.854500e-17    -0.206349  ...    -0.278900     0.116642\n",
            "Annotator_3 -1.114556e-01     0.017198  ...     0.153741     0.273821\n",
            "Annotator_4 -2.445580e-01    -0.045283  ...    -0.172160     0.018487\n",
            "Annotator_5  3.329560e-01    -0.161468  ...     1.000000     0.125846\n",
            "Annotator_6 -4.724556e-02     0.029161  ...     0.125846     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n",
            "\n",
            "Confuion matrix between Annotator_1 and Annotator_2 for class label pathway:\n",
            "      yes  no\n",
            "yes    5   4\n",
            "no    16   5\n",
            "\n",
            "Cohens kappa matrix for class label pathway:\n",
            "              Annotator_0  Annotator_1  ...  Annotator_5  Annotator_6\n",
            "Annotator_0     1.000000    -0.153846  ...     0.297872    -0.043478\n",
            "Annotator_1    -0.153846     1.000000  ...    -0.139241     0.025974\n",
            "Annotator_2     0.000000    -0.149425  ...    -0.267606     0.109589\n",
            "Annotator_3    -0.075472     0.010989  ...     0.138756     0.239631\n",
            "Annotator_4    -0.243902    -0.044776  ...    -0.158798     0.017467\n",
            "Annotator_5     0.297872    -0.139241  ...     1.000000     0.125561\n",
            "Annotator_6    -0.043478     0.025974  ...     0.125561     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## For class label - Metabolic"
      ],
      "metadata": {
        "id": "iZssDDksm4Py"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "####define path of your folder where the data from annotator sare stored\n",
        "path = '/content/look_alike_data/annotators_label/'\n",
        "#### define the class label you are currently interested in\n",
        "class_label = 'Metabolic'\n",
        "data_metabolic, mapping = create_df(path, class_label)\n",
        "data_metabolic, mapping\n",
        "\n",
        "####printting dataframe generated\n",
        "#print('data_metabolic:\\n',data_metabolic)\n",
        "\n",
        "\n",
        "#### information about data_metabolic\n",
        "print('\\nNo of Annotators are: ',data_metabolic.shape[1])\n",
        "print('No of figures labeled by the annotators are: ',data_metabolic.shape[0])\n",
        "print('Shape of the DataFrame \"data_metabolic\" is : ',data_metabolic.shape)\n",
        "print('\\nTop 10 entries of data_metabolic are\\n: ',data_metabolic.head(10))\n",
        "\n",
        "#### Converting data_metabolic_converted to binary 0s and 1s\n",
        "data_metabolic_converted = data_metabolic.replace(('yes', 'no'), (1, 0), inplace=False)\n",
        "#print('\\n data_metabolic_converted for metabolic:\\n', data_metabolic_converted)\n",
        "\n",
        "\n",
        "#### printing  the correlation coefficient computed using Pearson Method uding corr function from pandas\n",
        "print('\\n Correlation matrix Pearson for class label metabolic:\\n', data_metabolic_converted.corr(method = 'pearson'))\n",
        "\n",
        "\n",
        "#### To print  the correlation coefficient computed using Kendal and Spearman Method follow the lines of code below.\n",
        "# print('Correlation matrix kendall:\\n', data.corr(method = 'kendall'))\n",
        "# print('Correlation matrix spearman:\\n', data.corr(method = 'spearman'))\n",
        "\n",
        "\n",
        "#### printing  the confusion matrix :\n",
        "print('\\nConfuion matrix between Annotator_1 and Annotator_2 for class label metabolic:\\n',\n",
        "      pd.DataFrame(confusion_matrix(data_metabolic.Annotator_1.to_numpy(),\n",
        "      data_metabolic.Annotator_2.to_numpy(), labels=['yes', 'no']), index=['yes', 'no'], columns=['yes', 'no']))\n",
        "'''      \n",
        "       Any combination of Annotators for confusion matrix can be chosen to print:\n",
        "            replace the pair ------> data_metabolic.Annotator_1.to_numpy(),data_metabolic.Annotator_2.to_numpy()\n",
        "            with whatever Annotator combination you wanna inspect. Lets say Annotator_2 and Annotator_6,\n",
        "                 Then new pair will be :\n",
        "                    data_metabolic.Annotator_2.to_numpy(),data_metabolic.Annotator_6.to_numpy()\n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "#### printing  the cohen kappa  matrix:\n",
        "\n",
        "total_annotators = data_metabolic.shape[1]\n",
        "cohen_kappa_matrix = np.zeros((total_annotators,total_annotators))\n",
        "annotater_list = data_metabolic.columns\n",
        "for i, a1 in enumerate(annotater_list):\n",
        "  for j, a2 in enumerate(annotater_list):\n",
        "    cohen_kappa_matrix[i][j] = cohen_kappa_score(data_metabolic[a1], data_metabolic[a2])\n",
        "    #print(i,j,cohen_kappa_score(a1, a2))\n",
        "\n",
        "print('\\nCohens kappa matrix for class label metabolic:\\n',pd.DataFrame(cohen_kappa_matrix , index= data_metabolic.columns, columns=data_metabolic.columns))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b225b509-b502-411c-892c-aeca12e0117f",
        "id": "8sOsQIY_mdUO"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "No of Annotators are:  7\n",
            "No of figures labeled by the annotators are:  30\n",
            "Shape of the DataFrame \"data_metabolic\" is :  (30, 7)\n",
            "\n",
            "Top 10 entries of data_metabolic are\n",
            ":    Annotator_0 Annotator_1 Annotator_2  ... Annotator_4 Annotator_5 Annotator_6\n",
            "0          no          no         yes  ...          no         yes         yes\n",
            "1         yes          no         yes  ...          no         yes         yes\n",
            "2          no          no         yes  ...          no         yes         yes\n",
            "3          no         yes         yes  ...          no          no         yes\n",
            "4          no          no         yes  ...          no          no          no\n",
            "5          no          no         yes  ...         yes          no         yes\n",
            "6         yes          no          no  ...         yes         yes         yes\n",
            "7          no         yes         yes  ...          no         yes          no\n",
            "8         yes          no         yes  ...          no          no          no\n",
            "9         yes         yes         yes  ...          no         yes         yes\n",
            "\n",
            "[10 rows x 7 columns]\n",
            "\n",
            " Correlation matrix Pearson for class label metabolic:\n",
            "              Annotator_0  Annotator_1  ...  Annotator_5  Annotator_6\n",
            "Annotator_0     1.000000     0.218218  ...     0.174964     0.262445\n",
            "Annotator_1     0.218218     1.000000  ...     0.133631     0.200446\n",
            "Annotator_2    -0.218218     0.041667  ...    -0.133631     0.133631\n",
            "Annotator_3     0.269841     0.145479  ...     0.262445     0.029161\n",
            "Annotator_4     0.047619    -0.327327  ...    -0.408248    -0.174964\n",
            "Annotator_5     0.174964     0.133631  ...     1.000000     0.205357\n",
            "Annotator_6     0.262445     0.200446  ...     0.205357     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n",
            "\n",
            "Confuion matrix between Annotator_1 and Annotator_2 for class label metabolic:\n",
            "      yes  no\n",
            "yes    5   1\n",
            "no    19   5\n",
            "\n",
            "Cohens kappa matrix for class label metabolic:\n",
            "              Annotator_0  Annotator_1  ...  Annotator_5  Annotator_6\n",
            "Annotator_0     1.000000     0.210526  ...     0.155844     0.246575\n",
            "Annotator_1     0.210526     1.000000  ...     0.102564     0.166667\n",
            "Annotator_2    -0.129032     0.019608  ...    -0.111111     0.102564\n",
            "Annotator_3     0.195402     0.086022  ...     0.246575     0.025974\n",
            "Annotator_4     0.047619    -0.315789  ...    -0.363636    -0.164384\n",
            "Annotator_5     0.155844     0.102564  ...     1.000000     0.203540\n",
            "Annotator_6     0.246575     0.166667  ...     0.203540     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## For class label - Pre-Terpene"
      ],
      "metadata": {
        "id": "gokXuHcGnyPi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "####define path of your folder where the data from annotator sare stored\n",
        "path = '/content/look_alike_data/annotators_label/'\n",
        "\n",
        "#### define the class label you are currently interested in\n",
        "class_label = 'Pre-Terpene'\n",
        "data_pre_Terpene, mapping = create_df(path, class_label)\n",
        "data_pre_Terpene, mapping\n",
        "\n",
        "####printting dataframe generated\n",
        "#print('data_pre_Terpene:\\n',data_pre_Terpene)\n",
        "\n",
        "\n",
        "#### information about data_pre_Terpene\n",
        "print('\\nNo of Annotators are: ',data_pre_Terpene.shape[1])\n",
        "print('No of figures labeled by the annotators are: ',data_pre_Terpene.shape[0])\n",
        "print('Shape of the DataFrame \"data_pre_Terpene\" is : ',data_pre_Terpene.shape)\n",
        "print('\\nTop 10 entries of data_pre_Terpene are\\n: ',data_pre_Terpene.head(10))\n",
        "\n",
        "#### Converting data_pre_Terpene_converted to binary 0s and 1s\n",
        "data_pre_Terpene_converted = data_pre_Terpene.replace(('yes', 'no'), (1, 0), inplace=False)\n",
        "#print('\\n data_pre_Terpene_converted for pre_Terpene:\\n', data_pre_Terpene_converted)\n",
        "\n",
        "\n",
        "#### printing  the correlation coefficient computed using Pearson Method uding corr function from pandas\n",
        "print('\\n Correlation matrix Pearson for class label pre_Terpene:\\n', data_pre_Terpene_converted.corr(method = 'pearson'))\n",
        "\n",
        "\n",
        "#### To print  the correlation coefficient computed using Kendal and Spearman Method follow the lines of code below.\n",
        "# print('Correlation matrix kendall:\\n', data.corr(method = 'kendall'))\n",
        "# print('Correlation matrix spearman:\\n', data.corr(method = 'spearman'))\n",
        "\n",
        "\n",
        "#### printing  the confusion matrix :\n",
        "print('\\nConfuion matrix between Annotator_1 and Annotator_2 for class label pre_Terpene:\\n',\n",
        "      pd.DataFrame(confusion_matrix(data_pre_Terpene.Annotator_1.to_numpy(),\n",
        "      data_pre_Terpene.Annotator_2.to_numpy(), labels=['yes', 'no']), index=['yes', 'no'], columns=['yes', 'no']))\n",
        "'''      \n",
        "       Any combination of Annotators for confusion matrix can be chosen to print:\n",
        "            replace the pair ------> data_pre_Terpene.Annotator_1.to_numpy(),data_pre_Terpene.Annotator_2.to_numpy()\n",
        "            with whatever Annotator combination you wanna inspect. Lets say Annotator_2 and Annotator_6,\n",
        "                 Then new pair will be :\n",
        "                    data_pre_Terpene.Annotator_2.to_numpy(),data_pre_Terpene.Annotator_6.to_numpy()\n",
        "\n",
        "\n",
        "'''\n",
        "\n",
        "#### printing  the cohen kappa  matrix:\n",
        "\n",
        "total_annotators = data_pre_Terpene.shape[1]\n",
        "cohen_kappa_matrix = np.zeros((total_annotators,total_annotators))\n",
        "annotater_list = data_pre_Terpene.columns\n",
        "for i, a1 in enumerate(annotater_list):\n",
        "  for j, a2 in enumerate(annotater_list):\n",
        "    cohen_kappa_matrix[i][j] = cohen_kappa_score(data_pre_Terpene[a1], data_pre_Terpene[a2])\n",
        "    #print(i,j,cohen_kappa_score(a1, a2))\n",
        "\n",
        "print('\\nCohens kappa matrix for class label pre_Terpene:\\n',pd.DataFrame(cohen_kappa_matrix , index= data_pre_Terpene.columns, columns=data_pre_Terpene.columns))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fc91c082-b9e1-49d1-a6c4-546c7f33d999",
        "id": "3Ovy8xi1nVDX"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "No of Annotators are:  7\n",
            "No of figures labeled by the annotators are:  30\n",
            "Shape of the DataFrame \"data_pre_Terpene\" is :  (30, 7)\n",
            "\n",
            "Top 10 entries of data_pre_Terpene are\n",
            ":    Annotator_0 Annotator_1 Annotator_2  ... Annotator_4 Annotator_5 Annotator_6\n",
            "0          no          no         yes  ...          no          no         yes\n",
            "1         yes         yes         yes  ...          no         yes         yes\n",
            "2          no         yes          no  ...         yes         yes          no\n",
            "3         yes         yes         yes  ...          no          no         yes\n",
            "4         yes          no         yes  ...          no          no         yes\n",
            "5          no         yes         yes  ...         yes         yes          no\n",
            "6          no          no         yes  ...         yes          no         yes\n",
            "7          no         yes         yes  ...          no         yes         yes\n",
            "8          no         yes         yes  ...          no          no         yes\n",
            "9          no         yes          no  ...         yes         yes         yes\n",
            "\n",
            "[10 rows x 7 columns]\n",
            "\n",
            " Correlation matrix Pearson for class label pre_Terpene:\n",
            "              Annotator_0   Annotator_1  ...   Annotator_5  Annotator_6\n",
            "Annotator_0     1.000000  1.443376e-01  ...  1.414214e-01     0.047565\n",
            "Annotator_1     0.144338  1.000000e+00  ...  2.266233e-17     0.109847\n",
            "Annotator_2     0.222911  1.930468e-01  ... -2.364331e-01    -0.005301\n",
            "Annotator_3     0.144338 -2.500000e-01  ... -1.360828e-01    -0.164771\n",
            "Annotator_4    -0.332956 -1.098470e-01  ...  6.726728e-02     0.085973\n",
            "Annotator_5     0.141421  2.266233e-17  ...  1.000000e+00    -0.201802\n",
            "Annotator_6     0.047565  1.098470e-01  ... -2.018018e-01     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n",
            "\n",
            "Confuion matrix between Annotator_1 and Annotator_2 for class label pre_Terpene:\n",
            "      yes  no\n",
            "yes   15   3\n",
            "no     8   4\n",
            "\n",
            "Cohens kappa matrix for class label pre_Terpene:\n",
            "              Annotator_0  Annotator_1  ...  Annotator_5  Annotator_6\n",
            "Annotator_0     1.000000     0.125000  ...     0.133333     0.042553\n",
            "Annotator_1     0.125000     1.000000  ...     0.000000     0.109589\n",
            "Annotator_2     0.150943     0.179104  ...    -0.200000    -0.004785\n",
            "Annotator_3     0.125000    -0.250000  ...    -0.133333    -0.164384\n",
            "Annotator_4    -0.325581    -0.103896  ...     0.066667     0.082969\n",
            "Annotator_5     0.133333     0.000000  ...     1.000000    -0.200000\n",
            "Annotator_6     0.042553     0.109589  ...    -0.200000     1.000000\n",
            "\n",
            "[7 rows x 7 columns]\n"
          ]
        }
      ]
    }
  ]
}