clean_pd_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import pathlib\n",
    "import ntpath\n",
    "from commons.data_processing import *\n",
    "from commons.DataProcessors.pd_processor import PDProcessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all pd data files\n",
    "std_files = get_files(r\".\\N_Glycosylation_Results\\StandardProteins\", exts=['.xlsx'])\n",
    "\n",
    "mt10_files = get_files(r\".\\N_Glycosylation_Results\\BCaP_MT10\", exts=['.xlsx'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read each data file and concatenate contents\n",
    "\n",
    "# instantiate main object\n",
    "read_data = None\n",
    "\n",
    "# create map for dilution factor\n",
    "dilutions = {\"1\": \"1x\", \"05\": \"2x\", \"025\": \"4x\", \"0125\": \"8x\", \"00625\": \"16x\"}\n",
    "\n",
    "for file in std_files:\n",
    "    # get filename from full path\n",
    "    base_name, _ = ntpath.splitext(ntpath.basename(file))\n",
    "    \n",
    "    # extract dilution, temperature and run #\n",
    "    div = base_name.split(\"-\")[0]\n",
    "    div = div.split(\"_\")\n",
    "    dil, temp, run = div[-3:]\n",
    "\n",
    "    # get sample group name\n",
    "    group = ntpath.dirname(file)\n",
    "    group = group.split('\\\\')[-1]\n",
    "    \n",
    "    # use PDProcessor class to read data\n",
    "    data = PDProcessor([file], sample_name=\"_\".join(div[-3:]))\n",
    "\n",
    "    # create special columns with our extracted info\n",
    "    data.add_special_column(\"dilution\", dilutions[dil])\n",
    "    data.add_special_column(\"temperature\", temp)\n",
    "    data.add_special_column(\"run\", run.lower())\n",
    "    data.add_special_column(\"sample\", group)\n",
    "\n",
    "    # concatenate data\n",
    "    if read_data is None:\n",
    "        read_data = data\n",
    "    else:\n",
    "        # joining handled by PDProcessor class\n",
    "        read_data.join_processors(data)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in mt10_files:\n",
    "    # get filename from full path\n",
    "    base_name, _ = ntpath.splitext(ntpath.basename(file))\n",
    "    \n",
    "    # extract dilution, temperature and run #\n",
    "    div = base_name.split(\"-\")[0]\n",
    "    div = div.split(\"_\")\n",
    "    temp, dil, run = div[-3:]\n",
    "\n",
    "    # get sample group name\n",
    "    group = ntpath.dirname(file)\n",
    "    group = group.split('\\\\')[-1]\n",
    "    \n",
    "    # use PDProcessor class to read data\n",
    "    data = PDProcessor([file], sample_name=\"_\".join(div[-3:]))\n",
    "\n",
    "    # create special columns with our extracted info\n",
    "    data.add_special_column(\"dilution\", dil)\n",
    "    data.add_special_column(\"temperature\", temp)\n",
    "    data.add_special_column(\"run\", run.lower())\n",
    "    data.add_special_column(\"sample\", group)\n",
    "\n",
    "    # concatenate data\n",
    "    if read_data is None:\n",
    "        read_data = data\n",
    "    else:\n",
    "        # joining handled by PDProcessor class\n",
    "        read_data.join_processors(data)\n",
    "\n",
    "read_data.alias_engine(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define functions to match glycan composition to type\n",
    "def categorize_glycan(glycan):\n",
    "\n",
    "    glycan = glycan.replace(\")\", \",\")\n",
    "    glycan = glycan.replace(\"(\", \" \")\n",
    "    glycan = glycan.split(\",\")[:-1]\n",
    "    d = {k: int(v) for [k, v] in [i.split(\" \") for i in glycan]}\n",
    "\n",
    "    if \"NeuAc\" in d or \"NeuGc\" in d:\n",
    "        return \"Sialylated\"\n",
    "    elif \"Fuc\" in d:\n",
    "        if d[\"HexNAc\"] > 2:\n",
    "            return \"Fucosylated\"\n",
    "        elif d[\"HexNAc\"] == 2:\n",
    "            if \"Hex\" in d:\n",
    "                if d[\"Hex\"] > 4:\n",
    "                    return \"Complex\"\n",
    "                else:\n",
    "                    return \"Paucimannose\"\n",
    "            else:\n",
    "                return \"Paucimannose\"\n",
    "        else:\n",
    "            return \"Paucimannose\"\n",
    "    elif d[\"HexNAc\"] > 2:\n",
    "        return \"Complex\"\n",
    "    elif d[\"HexNAc\"] <= 2:\n",
    "        if \"Hex\" in d:\n",
    "            if d[\"Hex\"] <= 9 and d[\"Hex\"] > 4:\n",
    "                return \"High Mannose\"\n",
    "\n",
    "        return \"Paucimannose\"\n",
    "\n",
    "def determine_degree_sial(glycan):\n",
    "\n",
    "    degree_map = {\n",
    "        1: 'Monosialylated',\n",
    "        2: 'Disialylated',\n",
    "        3: 'Trisialylated',\n",
    "        4: 'Tetrasialylated',\n",
    "        5: 'Pentasialylated',\n",
    "    }\n",
    "\n",
    "    glycan = glycan.replace(')', ',')\n",
    "    glycan = glycan.replace('(', ' ')\n",
    "    glycan = glycan.split(',')[:-1]\n",
    "    d = {k:int(v) for [k, v] in [i.split(' ') for i in glycan]}\n",
    "\n",
    "    if not 'NeuAc' in d and 'NeuGc' not in d:\n",
    "        return 0\n",
    "    else:\n",
    "        total = int(d.get('NeuAc', 0)) + int(d.get('NeuGc', 0))\n",
    "        return degree_map[total]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13016, 47)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract peptide contents from PDProcessor object\n",
    "peptides = read_data.peptides\n",
    "\n",
    "# extract only rows containing glycans\n",
    "peptides = peptides[~peptides.glycan_composition.isna()] \n",
    "\n",
    "# get high scoring peptides\n",
    "peptides = peptides.loc[\n",
    "    (peptides.byonic_score >= 150) &\n",
    "    (peptides.delta_mod_score >= 10), :]\n",
    "\n",
    "# add new column\n",
    "peptides['glycan'] = peptides.glycan_composition\n",
    "\n",
    "# map glycan type and sialylation info\n",
    "peptides.loc[:, \"glycan_type\"] = peptides.glycan.map(categorize_glycan)\n",
    "peptides.loc[:, \"degree_sial\"] = peptides.glycan.map(determine_degree_sial)\n",
    "\n",
    "\n",
    "# map unique glycopeptide id\n",
    "peptides.loc[:, \"pep_mods\"] = peptides.apply(\n",
    "    lambda x: x[\"sequence\"] + \"_\" + x[\"glycan\"], axis=1\n",
    ")\n",
    "\n",
    "# sort according to concentration\n",
    "peptides = peptides.sort_values('dilution', ascending=False)\n",
    "\n",
    "# reset index\n",
    "peptides = peptides.reset_index(drop=True)\n",
    "\n",
    "peptides.head(3)\n",
    "peptides.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original dataframe has been reduced from 13016 rows to 11420 rows\n"
     ]
    }
   ],
   "source": [
    "# further reduce the dataset to include only those glycopeptides found in 2/3 of runs\n",
    "\n",
    "final_results = pd.DataFrame()\n",
    "\n",
    "# for each temperature\n",
    "for t, temperature in iterate_contents('temperature', peptides, get_item=True):\n",
    "    # for each concentration\n",
    "    for c, concentration in iterate_contents('dilution', temperature, get_item=True):\n",
    "        # count occurrences of peptides\n",
    "        counts = concentration.pep_mods.value_counts()\n",
    "\n",
    "        # keep only the ones with occurence >= 2\n",
    "        valid_number = counts[counts.values >= 2].keys()\n",
    "        valid_ids = concentration[concentration.pep_mods.isin(valid_number)]\n",
    "        \n",
    "        # append final results\n",
    "        final_results = pd.concat([final_results, valid_ids])\n",
    "\n",
    "final_results.reset_index(inplace=True, drop=True)\n",
    "\n",
    "print(f'Original dataframe has been reduced from {len(peptides)} rows to {len(final_results)} rows')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create an output location\n",
    "export_loc = pathlib.PurePath('.', 'Working_Datafiles')\n",
    "pathlib.Path(export_loc).mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# export final dataframe as working document\n",
    "export_filename = pathlib.PurePath(export_loc, 'All_N_Glycopeptides.csv')\n",
    "final_results.to_csv(str(export_filename), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accession</th>\n",
       "      <th>description</th>\n",
       "      <th>checked</th>\n",
       "      <th>confidence</th>\n",
       "      <th>annotated_sequence</th>\n",
       "      <th>modifications</th>\n",
       "      <th>master_protein_accessions</th>\n",
       "      <th>rt_min</th>\n",
       "      <th>mz_da</th>\n",
       "      <th>charge</th>\n",
       "      <th>...</th>\n",
       "      <th>sequence</th>\n",
       "      <th>data_source</th>\n",
       "      <th>temperature</th>\n",
       "      <th>run</th>\n",
       "      <th>concentration</th>\n",
       "      <th>dilution</th>\n",
       "      <th>glycan</th>\n",
       "      <th>glycan_type</th>\n",
       "      <th>degree_sial</th>\n",
       "      <th>pep_mods</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>Alpha-1-acid glycoprotein OS=Bos taurus OX=991...</td>\n",
       "      <td>True</td>\n",
       "      <td>High</td>\n",
       "      <td>[R].QNGTLSK.[V]</td>\n",
       "      <td>1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>26.5127</td>\n",
       "      <td>1513.61</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>QNGTLSK</td>\n",
       "      <td>1_45C_Run1</td>\n",
       "      <td>45C</td>\n",
       "      <td>Run1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1x</td>\n",
       "      <td>HexNAc(5)Hex(6)NeuAc(1)</td>\n",
       "      <td>Sialylated</td>\n",
       "      <td>Monosialylated</td>\n",
       "      <td>QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>Alpha-1-acid glycoprotein OS=Bos taurus OX=991...</td>\n",
       "      <td>True</td>\n",
       "      <td>High</td>\n",
       "      <td>[R].QNGTLSK.[V]</td>\n",
       "      <td>1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>26.2705</td>\n",
       "      <td>1513.61</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>QNGTLSK</td>\n",
       "      <td>1_45C_Run2</td>\n",
       "      <td>45C</td>\n",
       "      <td>Run2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1x</td>\n",
       "      <td>HexNAc(5)Hex(6)NeuAc(1)</td>\n",
       "      <td>Sialylated</td>\n",
       "      <td>Monosialylated</td>\n",
       "      <td>QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>Alpha-1-acid glycoprotein OS=Bos taurus OX=991...</td>\n",
       "      <td>True</td>\n",
       "      <td>High</td>\n",
       "      <td>[R].QNGTLSK.[V]</td>\n",
       "      <td>1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>24.5519</td>\n",
       "      <td>1513.61</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>QNGTLSK</td>\n",
       "      <td>1_30C_Run3</td>\n",
       "      <td>30C</td>\n",
       "      <td>Run3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1x</td>\n",
       "      <td>HexNAc(5)Hex(6)NeuAc(1)</td>\n",
       "      <td>Sialylated</td>\n",
       "      <td>Monosialylated</td>\n",
       "      <td>QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>Alpha-1-acid glycoprotein OS=Bos taurus OX=991...</td>\n",
       "      <td>True</td>\n",
       "      <td>High</td>\n",
       "      <td>[R].QNGTLSK.[V]</td>\n",
       "      <td>1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>24.9925</td>\n",
       "      <td>1513.61</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>QNGTLSK</td>\n",
       "      <td>1_30C_Run1</td>\n",
       "      <td>30C</td>\n",
       "      <td>Run1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1x</td>\n",
       "      <td>HexNAc(5)Hex(6)NeuAc(1)</td>\n",
       "      <td>Sialylated</td>\n",
       "      <td>Monosialylated</td>\n",
       "      <td>QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533</th>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>Alpha-1-acid glycoprotein OS=Bos taurus OX=991...</td>\n",
       "      <td>True</td>\n",
       "      <td>High</td>\n",
       "      <td>[R].QNGTLSK.[V]</td>\n",
       "      <td>1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "      <td>Q3SZR3</td>\n",
       "      <td>26.1591</td>\n",
       "      <td>1513.61</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>QNGTLSK</td>\n",
       "      <td>1_45C_Run3</td>\n",
       "      <td>45C</td>\n",
       "      <td>Run3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1x</td>\n",
       "      <td>HexNAc(5)Hex(6)NeuAc(1)</td>\n",
       "      <td>Sialylated</td>\n",
       "      <td>Monosialylated</td>\n",
       "      <td>QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    accession                                        description checked  \\\n",
       "30     Q3SZR3  Alpha-1-acid glycoprotein OS=Bos taurus OX=991...    True   \n",
       "124    Q3SZR3  Alpha-1-acid glycoprotein OS=Bos taurus OX=991...    True   \n",
       "174    Q3SZR3  Alpha-1-acid glycoprotein OS=Bos taurus OX=991...    True   \n",
       "234    Q3SZR3  Alpha-1-acid glycoprotein OS=Bos taurus OX=991...    True   \n",
       "533    Q3SZR3  Alpha-1-acid glycoprotein OS=Bos taurus OX=991...    True   \n",
       "\n",
       "    confidence annotated_sequence                   modifications  \\\n",
       "30        High    [R].QNGTLSK.[V]  1xHexNAc(5)Hex(6)NeuAc(1) [N2]   \n",
       "124       High    [R].QNGTLSK.[V]  1xHexNAc(5)Hex(6)NeuAc(1) [N2]   \n",
       "174       High    [R].QNGTLSK.[V]  1xHexNAc(5)Hex(6)NeuAc(1) [N2]   \n",
       "234       High    [R].QNGTLSK.[V]  1xHexNAc(5)Hex(6)NeuAc(1) [N2]   \n",
       "533       High    [R].QNGTLSK.[V]  1xHexNAc(5)Hex(6)NeuAc(1) [N2]   \n",
       "\n",
       "    master_protein_accessions   rt_min    mz_da charge  ... sequence  \\\n",
       "30                     Q3SZR3  26.5127  1513.61      2  ...  QNGTLSK   \n",
       "124                    Q3SZR3  26.2705  1513.61      2  ...  QNGTLSK   \n",
       "174                    Q3SZR3  24.5519  1513.61      2  ...  QNGTLSK   \n",
       "234                    Q3SZR3  24.9925  1513.61      2  ...  QNGTLSK   \n",
       "533                    Q3SZR3  26.1591  1513.61      2  ...  QNGTLSK   \n",
       "\n",
       "    data_source temperature   run concentration dilution  \\\n",
       "30   1_45C_Run1         45C  Run1           1.0       1x   \n",
       "124  1_45C_Run2         45C  Run2           1.0       1x   \n",
       "174  1_30C_Run3         30C  Run3           1.0       1x   \n",
       "234  1_30C_Run1         30C  Run1           1.0       1x   \n",
       "533  1_45C_Run3         45C  Run3           1.0       1x   \n",
       "\n",
       "                      glycan glycan_type     degree_sial  \\\n",
       "30   HexNAc(5)Hex(6)NeuAc(1)  Sialylated  Monosialylated   \n",
       "124  HexNAc(5)Hex(6)NeuAc(1)  Sialylated  Monosialylated   \n",
       "174  HexNAc(5)Hex(6)NeuAc(1)  Sialylated  Monosialylated   \n",
       "234  HexNAc(5)Hex(6)NeuAc(1)  Sialylated  Monosialylated   \n",
       "533  HexNAc(5)Hex(6)NeuAc(1)  Sialylated  Monosialylated   \n",
       "\n",
       "                                   pep_mods  \n",
       "30   QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]  \n",
       "124  QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]  \n",
       "174  QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]  \n",
       "234  QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]  \n",
       "533  QNGTLSK_1xHexNAc(5)Hex(6)NeuAc(1) [N2]  \n",
       "\n",
       "[5 rows x 47 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "peptides.loc[\n",
    "    (peptides.concentration==1.0) &\n",
    "    (peptides.glycan_type=='Sialylated') &\n",
    "    (peptides.degree_sial=='Monosialylated') &\n",
    "    (peptides.sequence=='QNGTLSK') &\n",
    "    (peptides.glycan == 'HexNAc(5)Hex(6)NeuAc(1)'), :\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "f2234c425d7a2224a10e1bb13348da236963434a2b4832174bcd474428c5b2f6"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}