Table3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load relative prevalence for all individuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "RP = pd.read_csv(\"bootstrap/relative_prevalence.tsv\", sep=\"\\t\", index_col=[0], header=None)[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load relative prevalence without first person pronouns for all individuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "RP_no_FPP = pd.read_csv(\"bootstrap/relative_prevalence_without_FPP.tsv\", sep=\"\\t\", index_col=[0], header=None)[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load relative prevalence per category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "RP_category = pd.read_csv(\"bootstrap/relative_prevalence_category.tsv\", sep=\"\\t\", index_col=[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load relative prevalence without first person pronouns per category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "RP_category_no_FPP = pd.read_csv(\"bootstrap/relative_prevalence_category_without_FPP.tsv\", sep=\"\\t\", index_col=[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load relative prevalence for bootstrapped CDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "RP_CDS = pd.read_csv(\"bootstrap/relative_prevalence_CDS.tsv\", sep=\"\\t\", index_col=[0], header=None)[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load relative prevalence for bootstrapped CDS per category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "_CDS = pd.read_csv(\"data/list_of_CDS.tsv\", sep=\"\\t\", index_col=\"markers\")\n",
    "RP_category_CDS = pd.DataFrame(index=RP_CDS.index, columns=_CDS.categories.unique())\n",
    "for cat in _CDS.categories.unique():\n",
    "    fn = \"bootstrap/relative_prevalence_CDS_{}.tsv\".format(cat.replace(\" \", \"-\"))\n",
    "    RP_category_CDS.loc[:, cat] = pd.read_csv(fn, sep=\"\\t\", index_col=[0], header=None)[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define outputting function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def output_median(x):\n",
    "    return \"{:.3f}{}\".format(x.median(), \"*\" if x.quantile(q=0.025) > 1 else \"\")\n",
    "\n",
    "\n",
    "def output_CI(x):\n",
    "    return \"$[$\" + \"{:.3f}, {:.3f}\".format(x.quantile(q=0.025), x.quantile(q=0.975)) + r\"$]$\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "order = RP_category.median().sort_values(ascending=False).index\n",
    "\n",
    "idxvals = np.concatenate((np.array(['Total']), order), axis=None)\n",
    "table = pd.DataFrame(index=pd.Index(idxvals, name=\"CD Category\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Include relative prevalence values in table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "table.loc[\"Total\", r\"$PR$ median\"] = output_median(RP)\n",
    "table.loc[order, r\"$PR$ median\"] = RP_category.loc[:, order].apply(lambda x: output_median(x))\n",
    "\n",
    "table.loc[\"Total\", r\"$PR$ 95% CI\"] = output_CI(RP)\n",
    "table.loc[order, r\"$PR$ 95% CI\"] = RP_category.loc[:, order].apply(lambda x: output_CI(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Include relative prevalence without FPP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "table.loc[\"Total\", r\"$PR_1$ median\"] = output_median(RP_no_FPP)\n",
    "table.loc[order[1:], r\"$PR_1$ median\"] = RP_category_no_FPP.loc[:, order[1:]].apply(lambda x: output_median(x))\n",
    "table.loc[\"Personalizing\", r\"$PR_1$ median\"] = \"/\"\n",
    "\n",
    "table.loc[\"Total\", r\"$PR_1$ 95% CI\"] = output_CI(RP_no_FPP)\n",
    "table.loc[order[1:], r\"$PR_1$ 95% CI\"] = RP_category_no_FPP.loc[:, order[1:]].apply(lambda x: output_CI(x)).fillna(\"/\")\n",
    "table.loc[\"Personalizing\", r\"$PR_1$ 95% CI\"] = \"/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Include relative prevalence based on CDS bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "table.loc[\"Total\", r\"$PR_C$ median\"] = output_median(RP_CDS)\n",
    "table.loc[order, r\"$PR_C$ median\"] = RP_category_CDS.loc[:, order].apply(lambda x: output_median(x))\n",
    "\n",
    "table.loc[\"Total\", r\"$PR_C$ 95% CI\"] = output_CI(RP_CDS)\n",
    "table.loc[order, r\"$PR_C$ 95% CI\"] = RP_category_CDS.loc[:, order].apply(lambda x: output_CI(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>$PR$ median</th>\n",
       "      <th>$PR$ 95% CI</th>\n",
       "      <th>$PR_1$ median</th>\n",
       "      <th>$PR_1$ 95% CI</th>\n",
       "      <th>$PR_C$ median</th>\n",
       "      <th>$PR_C$ 95% CI</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CD Category</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>1.129*</td>\n",
       "      <td>$[$1.102, 1.157$]$</td>\n",
       "      <td>1.110*</td>\n",
       "      <td>$[$1.082, 1.137$]$</td>\n",
       "      <td>1.231*</td>\n",
       "      <td>$[$1.168, 1.320$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Personalizing</th>\n",
       "      <td>2.084*</td>\n",
       "      <td>$[$1.940, 2.239$]$</td>\n",
       "      <td>/</td>\n",
       "      <td>/</td>\n",
       "      <td>2.403*</td>\n",
       "      <td>$[$1.676, 3.043$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Emotional Reasoning</th>\n",
       "      <td>1.983*</td>\n",
       "      <td>$[$1.759, 2.228$]$</td>\n",
       "      <td>1.815*</td>\n",
       "      <td>$[$1.467, 2.217$]$</td>\n",
       "      <td>2.316*</td>\n",
       "      <td>$[$2.013, 3.158$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Overgeneralizing</th>\n",
       "      <td>1.441*</td>\n",
       "      <td>$[$1.367, 1.518$]$</td>\n",
       "      <td>1.344*</td>\n",
       "      <td>$[$1.271, 1.420$]$</td>\n",
       "      <td>1.605*</td>\n",
       "      <td>$[$1.414, 1.776$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mental Filtering</th>\n",
       "      <td>1.296*</td>\n",
       "      <td>$[$1.129, 1.471$]$</td>\n",
       "      <td>1.191</td>\n",
       "      <td>$[$0.931, 1.491$]$</td>\n",
       "      <td>1.466*</td>\n",
       "      <td>$[$1.171, 1.924$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Disqualifying the Positive</th>\n",
       "      <td>1.229*</td>\n",
       "      <td>$[$1.142, 1.320$]$</td>\n",
       "      <td>1.229*</td>\n",
       "      <td>$[$1.142, 1.320$]$</td>\n",
       "      <td>1.401*</td>\n",
       "      <td>$[$1.203, 1.536$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Labeling and mislabeling</th>\n",
       "      <td>1.207*</td>\n",
       "      <td>$[$1.159, 1.256$]$</td>\n",
       "      <td>1.090*</td>\n",
       "      <td>$[$1.041, 1.139$]$</td>\n",
       "      <td>1.336*</td>\n",
       "      <td>$[$1.176, 1.554$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Dichotomous Reasoning</th>\n",
       "      <td>1.131*</td>\n",
       "      <td>$[$1.101, 1.162$]$</td>\n",
       "      <td>1.131*</td>\n",
       "      <td>$[$1.101, 1.162$]$</td>\n",
       "      <td>1.217*</td>\n",
       "      <td>$[$1.159, 1.305$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fortune-telling</th>\n",
       "      <td>1.110</td>\n",
       "      <td>$[$0.955, 1.219$]$</td>\n",
       "      <td>0.908</td>\n",
       "      <td>$[$0.735, 1.037$]$</td>\n",
       "      <td>1.177</td>\n",
       "      <td>$[$0.855, 1.506$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Magnification and Minimization</th>\n",
       "      <td>1.084*</td>\n",
       "      <td>$[$1.039, 1.130$]$</td>\n",
       "      <td>1.084*</td>\n",
       "      <td>$[$1.039, 1.130$]$</td>\n",
       "      <td>1.085*</td>\n",
       "      <td>$[$1.020, 1.412$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Should statements</th>\n",
       "      <td>1.056*</td>\n",
       "      <td>$[$1.013, 1.100$]$</td>\n",
       "      <td>1.056*</td>\n",
       "      <td>$[$1.013, 1.100$]$</td>\n",
       "      <td>1.116</td>\n",
       "      <td>$[$0.837, 1.409$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mindreading</th>\n",
       "      <td>1.052</td>\n",
       "      <td>$[$0.984, 1.117$]$</td>\n",
       "      <td>1.052</td>\n",
       "      <td>$[$0.984, 1.117$]$</td>\n",
       "      <td>1.127</td>\n",
       "      <td>$[$0.894, 1.259$]$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Catastrophizing</th>\n",
       "      <td>0.920</td>\n",
       "      <td>$[$0.763, 1.077$]$</td>\n",
       "      <td>0.920</td>\n",
       "      <td>$[$0.763, 1.077$]$</td>\n",
       "      <td>0.979</td>\n",
       "      <td>$[$0.859, 1.046$]$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               $PR$ median         $PR$ 95% CI $PR_1$ median  \\\n",
       "CD Category                                                                    \n",
       "Total                               1.129*  $[$1.102, 1.157$]$        1.110*   \n",
       "Personalizing                       2.084*  $[$1.940, 2.239$]$             /   \n",
       "Emotional Reasoning                 1.983*  $[$1.759, 2.228$]$        1.815*   \n",
       "Overgeneralizing                    1.441*  $[$1.367, 1.518$]$        1.344*   \n",
       "Mental Filtering                    1.296*  $[$1.129, 1.471$]$         1.191   \n",
       "Disqualifying the Positive          1.229*  $[$1.142, 1.320$]$        1.229*   \n",
       "Labeling and mislabeling            1.207*  $[$1.159, 1.256$]$        1.090*   \n",
       "Dichotomous Reasoning               1.131*  $[$1.101, 1.162$]$        1.131*   \n",
       "Fortune-telling                      1.110  $[$0.955, 1.219$]$         0.908   \n",
       "Magnification and Minimization      1.084*  $[$1.039, 1.130$]$        1.084*   \n",
       "Should statements                   1.056*  $[$1.013, 1.100$]$        1.056*   \n",
       "Mindreading                          1.052  $[$0.984, 1.117$]$         1.052   \n",
       "Catastrophizing                      0.920  $[$0.763, 1.077$]$         0.920   \n",
       "\n",
       "                                     $PR_1$ 95% CI $PR_C$ median  \\\n",
       "CD Category                                                        \n",
       "Total                           $[$1.082, 1.137$]$        1.231*   \n",
       "Personalizing                                    /        2.403*   \n",
       "Emotional Reasoning             $[$1.467, 2.217$]$        2.316*   \n",
       "Overgeneralizing                $[$1.271, 1.420$]$        1.605*   \n",
       "Mental Filtering                $[$0.931, 1.491$]$        1.466*   \n",
       "Disqualifying the Positive      $[$1.142, 1.320$]$        1.401*   \n",
       "Labeling and mislabeling        $[$1.041, 1.139$]$        1.336*   \n",
       "Dichotomous Reasoning           $[$1.101, 1.162$]$        1.217*   \n",
       "Fortune-telling                 $[$0.735, 1.037$]$         1.177   \n",
       "Magnification and Minimization  $[$1.039, 1.130$]$        1.085*   \n",
       "Should statements               $[$1.013, 1.100$]$         1.116   \n",
       "Mindreading                     $[$0.984, 1.117$]$         1.127   \n",
       "Catastrophizing                 $[$0.763, 1.077$]$         0.979   \n",
       "\n",
       "                                     $PR_C$ 95% CI  \n",
       "CD Category                                         \n",
       "Total                           $[$1.168, 1.320$]$  \n",
       "Personalizing                   $[$1.676, 3.043$]$  \n",
       "Emotional Reasoning             $[$2.013, 3.158$]$  \n",
       "Overgeneralizing                $[$1.414, 1.776$]$  \n",
       "Mental Filtering                $[$1.171, 1.924$]$  \n",
       "Disqualifying the Positive      $[$1.203, 1.536$]$  \n",
       "Labeling and mislabeling        $[$1.176, 1.554$]$  \n",
       "Dichotomous Reasoning           $[$1.159, 1.305$]$  \n",
       "Fortune-telling                 $[$0.855, 1.506$]$  \n",
       "Magnification and Minimization  $[$1.020, 1.412$]$  \n",
       "Should statements               $[$0.837, 1.409$]$  \n",
       "Mindreading                     $[$0.894, 1.259$]$  \n",
       "Catastrophizing                 $[$0.859, 1.046$]$  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "table.to_csv(\"figures/Table3.tex\", sep=\"&\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}