prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Datasets used\n",
    "1. Synthetic data\n",
    "2. [Titanic survivors](https://www.kaggle.com/c/titanic/data)\n",
    "3. [Breast Cancer Wisconsin (Diagnostic) Data Set](https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+(Diagnostic))\n",
    "4. Harmonizome\n",
    "  + ProteomicsDB Cell Type and Tissue Protein Expression Profiles ([Gene-Attribute Matrix Cleaned](http://amp.pharm.mssm.edu/static/hdfs/harmonizome/data/proteomicsdb/gene_attribute_matrix_cleaned.txt.gz))\n",
    "  + [HGNC gene family](http://www.genenames.org/cgi-bin/genefamilies/download-all/tsv)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utils import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 20) (1000,)\n"
     ]
    }
   ],
   "source": [
    "## Synthetic data\n",
    "X, y = make_classification(n_classes=2, class_sep=2, weights=[0.1, 0.9],\n",
    "\tn_informative=3, n_redundant=1, flip_y=0,\n",
    "\tn_features=20, n_clusters_per_class=1,\n",
    "\tn_samples=1000, random_state=10)\n",
    "print X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(891, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex  Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male   22      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female   38      1   \n",
       "2                             Heikkinen, Miss. Laina  female   26      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female   35      1   \n",
       "4                           Allen, Mr. William Henry    male   35      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Load Titanic\n",
    "titanic = pd.read_csv('datasets/Titanic/train.csv')\n",
    "print titanic.shape\n",
    "titanic.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "342\n",
      "0.383838383838\n"
     ]
    }
   ],
   "source": [
    "## Examine class ratio\n",
    "print titanic['Survived'].sum()\n",
    "print titanic['Survived'].sum()/float(titanic.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(891, 8) (891,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Python/2.7/site-packages/numpy-1.10.4-py2.7-macosx-10.10-intel.egg/numpy/lib/arraysetops.py:200: FutureWarning: numpy not_equal will not check object identity in the future. The comparison did not return the same result as suggested by the identity (`is`)) and will change.\n",
      "  flag = np.concatenate(([True], aux[1:] != aux[:-1]))\n"
     ]
    }
   ],
   "source": [
    "## Process features\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "for col in ['Sex', 'Cabin', 'Embarked']:\n",
    "    titanic[col] = le.fit_transform(titanic[col])\n",
    "\n",
    "## Split df into X and y and convert to numpy.array\n",
    "X = titanic.drop(['PassengerId', 'Survived', 'Name', 'Ticket'], axis=1).values\n",
    "y = titanic['Survived'].values\n",
    "print X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(198, 35)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>outcome</th>\n",
       "      <th>Attr0</th>\n",
       "      <th>Attr1</th>\n",
       "      <th>Attr2</th>\n",
       "      <th>Attr3</th>\n",
       "      <th>Attr4</th>\n",
       "      <th>Attr5</th>\n",
       "      <th>Attr6</th>\n",
       "      <th>Attr7</th>\n",
       "      <th>...</th>\n",
       "      <th>Attr23</th>\n",
       "      <th>Attr24</th>\n",
       "      <th>Attr25</th>\n",
       "      <th>Attr26</th>\n",
       "      <th>Attr27</th>\n",
       "      <th>Attr28</th>\n",
       "      <th>Attr29</th>\n",
       "      <th>Attr30</th>\n",
       "      <th>Attr31</th>\n",
       "      <th>Attr32</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>119513</td>\n",
       "      <td>N</td>\n",
       "      <td>31</td>\n",
       "      <td>18.02</td>\n",
       "      <td>27.60</td>\n",
       "      <td>117.50</td>\n",
       "      <td>1013.0</td>\n",
       "      <td>0.09489</td>\n",
       "      <td>0.1036</td>\n",
       "      <td>0.1086</td>\n",
       "      <td>...</td>\n",
       "      <td>139.70</td>\n",
       "      <td>1436.0</td>\n",
       "      <td>0.1195</td>\n",
       "      <td>0.1926</td>\n",
       "      <td>0.3140</td>\n",
       "      <td>0.1170</td>\n",
       "      <td>0.2677</td>\n",
       "      <td>0.08113</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8423</td>\n",
       "      <td>N</td>\n",
       "      <td>61</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.2776</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>...</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>842517</td>\n",
       "      <td>N</td>\n",
       "      <td>116</td>\n",
       "      <td>21.37</td>\n",
       "      <td>17.44</td>\n",
       "      <td>137.50</td>\n",
       "      <td>1373.0</td>\n",
       "      <td>0.08836</td>\n",
       "      <td>0.1189</td>\n",
       "      <td>0.1255</td>\n",
       "      <td>...</td>\n",
       "      <td>159.10</td>\n",
       "      <td>1949.0</td>\n",
       "      <td>0.1188</td>\n",
       "      <td>0.3449</td>\n",
       "      <td>0.3414</td>\n",
       "      <td>0.2032</td>\n",
       "      <td>0.4334</td>\n",
       "      <td>0.09067</td>\n",
       "      <td>2.5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>843483</td>\n",
       "      <td>N</td>\n",
       "      <td>123</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.2839</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>...</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>843584</td>\n",
       "      <td>R</td>\n",
       "      <td>27</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.1328</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>...</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       ID outcome  Attr0  Attr1  Attr2   Attr3   Attr4    Attr5   Attr6  \\\n",
       "0  119513       N     31  18.02  27.60  117.50  1013.0  0.09489  0.1036   \n",
       "1    8423       N     61  17.99  10.38  122.80  1001.0  0.11840  0.2776   \n",
       "2  842517       N    116  21.37  17.44  137.50  1373.0  0.08836  0.1189   \n",
       "3  843483       N    123  11.42  20.38   77.58   386.1  0.14250  0.2839   \n",
       "4  843584       R     27  20.29  14.34  135.10  1297.0  0.10030  0.1328   \n",
       "\n",
       "    Attr7   ...    Attr23  Attr24  Attr25  Attr26  Attr27  Attr28  Attr29  \\\n",
       "0  0.1086   ...    139.70  1436.0  0.1195  0.1926  0.3140  0.1170  0.2677   \n",
       "1  0.3001   ...    184.60  2019.0  0.1622  0.6656  0.7119  0.2654  0.4601   \n",
       "2  0.1255   ...    159.10  1949.0  0.1188  0.3449  0.3414  0.2032  0.4334   \n",
       "3  0.2414   ...     98.87   567.7  0.2098  0.8663  0.6869  0.2575  0.6638   \n",
       "4  0.1980   ...    152.20  1575.0  0.1374  0.2050  0.4000  0.1625  0.2364   \n",
       "\n",
       "    Attr30  Attr31  Attr32  \n",
       "0  0.08113     5.0       5  \n",
       "1  0.11890     3.0       2  \n",
       "2  0.09067     2.5       0  \n",
       "3  0.17300     2.0       0  \n",
       "4  0.07678     3.5       0  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Load breast cancer data\n",
    "bc = pd.read_csv('datasets/wpbc.data', \n",
    "                    names=['ID', 'outcome'] + ['Attr%s'%i for i in range(33)])\n",
    "print bc.shape\n",
    "bc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N    151\n",
      "R     47\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print bc['outcome'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(198, 33) (198,)\n",
      "0.237373737374\n"
     ]
    }
   ],
   "source": [
    "## Split df into X and y and convert to numpy.array\n",
    "X = bc.drop(['ID', 'outcome'], axis=1)\n",
    "y = bc['outcome'].map({'N':0, 'R':1})\n",
    "\n",
    "X = X.values\n",
    "y = y.values\n",
    "\n",
    "print X.shape, y.shape\n",
    "print y.sum()/float(y.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2776, 56)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GeneSym</th>\n",
       "      <th>UniprotAcc</th>\n",
       "      <th>GeneID/Brenda Tissue Ontology BTO:</th>\n",
       "      <th>38</th>\n",
       "      <th>675</th>\n",
       "      <th>1615</th>\n",
       "      <th>568</th>\n",
       "      <th>2806</th>\n",
       "      <th>1938</th>\n",
       "      <th>18</th>\n",
       "      <th>...</th>\n",
       "      <th>1158</th>\n",
       "      <th>1363</th>\n",
       "      <th>1078</th>\n",
       "      <th>763</th>\n",
       "      <th>2553</th>\n",
       "      <th>1129</th>\n",
       "      <th>776</th>\n",
       "      <th>289</th>\n",
       "      <th>2417</th>\n",
       "      <th>914</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LOC102724985</td>\n",
       "      <td>Q6P996</td>\n",
       "      <td>102724985</td>\n",
       "      <td>5.017355</td>\n",
       "      <td>4.830592</td>\n",
       "      <td>5.783336</td>\n",
       "      <td>5.049440</td>\n",
       "      <td>4.858041</td>\n",
       "      <td>4.605708</td>\n",
       "      <td>4.868083</td>\n",
       "      <td>...</td>\n",
       "      <td>5.395317</td>\n",
       "      <td>4.734884</td>\n",
       "      <td>4.835634</td>\n",
       "      <td>4.659831</td>\n",
       "      <td>4.643150</td>\n",
       "      <td>4.376222</td>\n",
       "      <td>4.026439</td>\n",
       "      <td>4.659961</td>\n",
       "      <td>3.929821</td>\n",
       "      <td>4.854218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PDXDC1</td>\n",
       "      <td>H3BU11</td>\n",
       "      <td>23042</td>\n",
       "      <td>4.744159</td>\n",
       "      <td>4.601433</td>\n",
       "      <td>5.267198</td>\n",
       "      <td>4.741168</td>\n",
       "      <td>4.571314</td>\n",
       "      <td>4.518610</td>\n",
       "      <td>4.565993</td>\n",
       "      <td>...</td>\n",
       "      <td>4.894014</td>\n",
       "      <td>4.403376</td>\n",
       "      <td>4.440597</td>\n",
       "      <td>4.472297</td>\n",
       "      <td>4.479050</td>\n",
       "      <td>4.216782</td>\n",
       "      <td>3.893532</td>\n",
       "      <td>4.520178</td>\n",
       "      <td>3.725262</td>\n",
       "      <td>4.458436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>COG5</td>\n",
       "      <td>Q9UP83</td>\n",
       "      <td>10466</td>\n",
       "      <td>4.213385</td>\n",
       "      <td>4.213385</td>\n",
       "      <td>5.759206</td>\n",
       "      <td>4.528063</td>\n",
       "      <td>4.035077</td>\n",
       "      <td>4.359530</td>\n",
       "      <td>4.309750</td>\n",
       "      <td>...</td>\n",
       "      <td>4.634878</td>\n",
       "      <td>4.799939</td>\n",
       "      <td>4.510710</td>\n",
       "      <td>4.048240</td>\n",
       "      <td>3.924763</td>\n",
       "      <td>4.041728</td>\n",
       "      <td>4.135629</td>\n",
       "      <td>4.715816</td>\n",
       "      <td>4.092942</td>\n",
       "      <td>4.545271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HEXA</td>\n",
       "      <td>H3BU85</td>\n",
       "      <td>3073</td>\n",
       "      <td>4.535638</td>\n",
       "      <td>4.740634</td>\n",
       "      <td>5.118528</td>\n",
       "      <td>4.181380</td>\n",
       "      <td>4.731774</td>\n",
       "      <td>4.665979</td>\n",
       "      <td>4.865457</td>\n",
       "      <td>...</td>\n",
       "      <td>4.708274</td>\n",
       "      <td>4.937703</td>\n",
       "      <td>4.598400</td>\n",
       "      <td>5.290727</td>\n",
       "      <td>5.423842</td>\n",
       "      <td>4.154158</td>\n",
       "      <td>3.857923</td>\n",
       "      <td>5.115823</td>\n",
       "      <td>4.369458</td>\n",
       "      <td>5.008221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HEXB</td>\n",
       "      <td>H0Y9B6</td>\n",
       "      <td>3074</td>\n",
       "      <td>4.605520</td>\n",
       "      <td>4.591239</td>\n",
       "      <td>6.294882</td>\n",
       "      <td>4.139686</td>\n",
       "      <td>5.471996</td>\n",
       "      <td>4.442832</td>\n",
       "      <td>5.214406</td>\n",
       "      <td>...</td>\n",
       "      <td>5.139610</td>\n",
       "      <td>4.979131</td>\n",
       "      <td>5.654624</td>\n",
       "      <td>5.278758</td>\n",
       "      <td>5.283525</td>\n",
       "      <td>4.972013</td>\n",
       "      <td>4.536124</td>\n",
       "      <td>5.388605</td>\n",
       "      <td>4.576318</td>\n",
       "      <td>5.425797</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        GeneSym UniprotAcc  GeneID/Brenda Tissue Ontology BTO:        38  \\\n",
       "0  LOC102724985     Q6P996                           102724985  5.017355   \n",
       "1        PDXDC1     H3BU11                               23042  4.744159   \n",
       "2          COG5     Q9UP83                               10466  4.213385   \n",
       "3          HEXA     H3BU85                                3073  4.535638   \n",
       "4          HEXB     H0Y9B6                                3074  4.605520   \n",
       "\n",
       "        675      1615       568      2806      1938        18    ...     \\\n",
       "0  4.830592  5.783336  5.049440  4.858041  4.605708  4.868083    ...      \n",
       "1  4.601433  5.267198  4.741168  4.571314  4.518610  4.565993    ...      \n",
       "2  4.213385  5.759206  4.528063  4.035077  4.359530  4.309750    ...      \n",
       "3  4.740634  5.118528  4.181380  4.731774  4.665979  4.865457    ...      \n",
       "4  4.591239  6.294882  4.139686  5.471996  4.442832  5.214406    ...      \n",
       "\n",
       "       1158      1363      1078       763      2553      1129       776  \\\n",
       "0  5.395317  4.734884  4.835634  4.659831  4.643150  4.376222  4.026439   \n",
       "1  4.894014  4.403376  4.440597  4.472297  4.479050  4.216782  3.893532   \n",
       "2  4.634878  4.799939  4.510710  4.048240  3.924763  4.041728  4.135629   \n",
       "3  4.708274  4.937703  4.598400  5.290727  5.423842  4.154158  3.857923   \n",
       "4  5.139610  4.979131  5.654624  5.278758  5.283525  4.972013  4.536124   \n",
       "\n",
       "        289      2417       914  \n",
       "0  4.659961  3.929821  4.854218  \n",
       "1  4.520178  3.725262  4.458436  \n",
       "2  4.715816  4.092942  4.545271  \n",
       "3  5.115823  4.369458  5.008221  \n",
       "4  5.388605  4.576318  5.425797  \n",
       "\n",
       "[5 rows x 56 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Load ProteomicsDB data\n",
    "proteome = pd.read_csv('datasets/Harmonizome/gene_attribute_matrix_cleaned.txt.gz', \n",
    "                       sep='\\t', compression='gzip', skiprows=2)\n",
    "print proteome.shape\n",
    "proteome.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20086, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HGNC ID</th>\n",
       "      <th>Approved Symbol</th>\n",
       "      <th>Approved Name</th>\n",
       "      <th>Status</th>\n",
       "      <th>Previous Symbols</th>\n",
       "      <th>Synonyms</th>\n",
       "      <th>Chromosome</th>\n",
       "      <th>Accession Numbers</th>\n",
       "      <th>RefSeq IDs</th>\n",
       "      <th>Gene Family Tag</th>\n",
       "      <th>Gene family description</th>\n",
       "      <th>Gene family ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>324</td>\n",
       "      <td>AGPAT1</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferase 1</td>\n",
       "      <td>Approved</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LPAAT-alpha</td>\n",
       "      <td>6p21.3</td>\n",
       "      <td>U56417</td>\n",
       "      <td>NM_006411</td>\n",
       "      <td>AGPAT</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferases</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>325</td>\n",
       "      <td>AGPAT2</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferase 2</td>\n",
       "      <td>Approved</td>\n",
       "      <td>BSCL</td>\n",
       "      <td>LPAAT-beta</td>\n",
       "      <td>9q34.3</td>\n",
       "      <td>AF000237</td>\n",
       "      <td>NM_006412</td>\n",
       "      <td>AGPAT</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferases</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>326</td>\n",
       "      <td>AGPAT3</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferase 3</td>\n",
       "      <td>Approved</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LPAAT-gamma</td>\n",
       "      <td>21q22.3</td>\n",
       "      <td>AF156774</td>\n",
       "      <td>NM_020132</td>\n",
       "      <td>AGPAT</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferases</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20885</td>\n",
       "      <td>AGPAT4</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferase 4</td>\n",
       "      <td>Approved</td>\n",
       "      <td>NaN</td>\n",
       "      <td>LPAAT-delta, dJ473J16.2</td>\n",
       "      <td>6q25.3</td>\n",
       "      <td>AF156776</td>\n",
       "      <td>NM_020133</td>\n",
       "      <td>AGPAT</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferases</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20886</td>\n",
       "      <td>AGPAT5</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferase 5</td>\n",
       "      <td>Approved</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FLJ11210, LPAAT-e, LPAAT-epsilon</td>\n",
       "      <td>8p23.1</td>\n",
       "      <td>AF375789</td>\n",
       "      <td>NM_018361</td>\n",
       "      <td>AGPAT</td>\n",
       "      <td>1-acylglycerol-3-phosphate O-acyltransferases</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   HGNC ID Approved Symbol                                   Approved Name  \\\n",
       "0      324          AGPAT1  1-acylglycerol-3-phosphate O-acyltransferase 1   \n",
       "1      325          AGPAT2  1-acylglycerol-3-phosphate O-acyltransferase 2   \n",
       "2      326          AGPAT3  1-acylglycerol-3-phosphate O-acyltransferase 3   \n",
       "3    20885          AGPAT4  1-acylglycerol-3-phosphate O-acyltransferase 4   \n",
       "4    20886          AGPAT5  1-acylglycerol-3-phosphate O-acyltransferase 5   \n",
       "\n",
       "     Status Previous Symbols                          Synonyms Chromosome  \\\n",
       "0  Approved              NaN                       LPAAT-alpha     6p21.3   \n",
       "1  Approved             BSCL                        LPAAT-beta     9q34.3   \n",
       "2  Approved              NaN                       LPAAT-gamma    21q22.3   \n",
       "3  Approved              NaN           LPAAT-delta, dJ473J16.2     6q25.3   \n",
       "4  Approved              NaN  FLJ11210, LPAAT-e, LPAAT-epsilon     8p23.1   \n",
       "\n",
       "  Accession Numbers RefSeq IDs Gene Family Tag  \\\n",
       "0            U56417  NM_006411           AGPAT   \n",
       "1          AF000237  NM_006412           AGPAT   \n",
       "2          AF156774  NM_020132           AGPAT   \n",
       "3          AF156776  NM_020133           AGPAT   \n",
       "4          AF375789  NM_018361           AGPAT   \n",
       "\n",
       "                         Gene family description  Gene family ID  \n",
       "0  1-acylglycerol-3-phosphate O-acyltransferases              46  \n",
       "1  1-acylglycerol-3-phosphate O-acyltransferases              46  \n",
       "2  1-acylglycerol-3-phosphate O-acyltransferases              46  \n",
       "3  1-acylglycerol-3-phosphate O-acyltransferases              46  \n",
       "4  1-acylglycerol-3-phosphate O-acyltransferases              46  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Load HGNC gene family \n",
    "gene_family = pd.read_csv('datasets/Harmonizome/HGNC_gene_family.txt',sep='\\t')\n",
    "print gene_family.shape\n",
    "gene_family.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MicroRNAs                                            1777\n",
      "Zinc fingers C2H2-type                                720\n",
      "RNAs, 7SL, cytoplasmic                                689\n",
      "Cytoplasmic transfer RNAs                             588\n",
      "Solute carriers                                       395\n",
      "CD molecules                                          394\n",
      "Small nucleolar RNAs, C/D box                         326\n",
      "Ring finger proteins                                  275\n",
      "WD repeat domain containing                           262\n",
      "Immunoglobulin-like domain containing                 245\n",
      "Ankyrin repeat domain containing                      242\n",
      "Endogenous ligands                                    236\n",
      "EF-hand domain containing                             222\n",
      "RNA binding motif containing                          213\n",
      "Long non-coding RNAs                                  211\n",
      "Pleckstrin homology domain containing                 206\n",
      "Immunoglobulin heavy locus at 14q32.33                185\n",
      "Protein phosphatase 1 regulatory subunits             181\n",
      "V-set domain containing                               163\n",
      "I-set domain containing                               161\n",
      "Fibronectin type III domain containing                153\n",
      "PDZ domain containing                                 152\n",
      "Small nucleolar RNAs, H/ACA box                       142\n",
      "BTB domain containing                                 134\n",
      "Olfactory receptors, family 4                         129\n",
      "Vomeronasal receptors                                 129\n",
      "T-cell receptor alpha locus at 14q11.2                117\n",
      "Histones                                              116\n",
      "Tetratricopeptide repeat domain containing            115\n",
      "Piwi-interacting RNA clusters                         114\n",
      "                                                     ... \n",
      "Platelet-activating factor receptor                     1\n",
      "ATP binding cassette subfamily E                        1\n",
      "Cytochrome P450 family 17                               1\n",
      "Cytochrome P450 family 24                               1\n",
      "Cystatins, type 3                                       1\n",
      "UDP-glucose ceramide glucosyltransferases               1\n",
      "Myosins, class XIX                                      1\n",
      "Olfactory receptors, family 55                          1\n",
      "Succinate receptor                                      1\n",
      "Chloride channels, ATP-gated CFTR                       1\n",
      "Histone deacetylases, class IV                          1\n",
      "Zinc activated channels                                 1\n",
      "RAN family GTPases                                      1\n",
      "Myosins, class XVI                                      1\n",
      "Sodium leak channels, non selective                     1\n",
      "G protein-coupled estrogen receptor                     1\n",
      "Myosins, class VI                                       1\n",
      "Cytochrome P450 family 39                               1\n",
      "G protein-coupled bile acid receptor                    1\n",
      "Hydrogen voltage gated channels                         1\n",
      "Chemokine (X-C motif) receptor                          1\n",
      "Cytochrome P450 family 5                                1\n",
      "Chemokine (C-X-3-C motif) receptor                      1\n",
      "Oxoglutarate receptor                                   1\n",
      "Class II Cys-based phosphatases                         1\n",
      "Ubiquitin specific peptidase like                       1\n",
      "Lipid phosphatases                                      1\n",
      "Myosins, class X                                        1\n",
      "Cytochrome P450 family 19                               1\n",
      "Adhesion G protein-coupled receptors, subfamily V       1\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print gene_family['Gene family description'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['6-phosphofructo-2-kinases/fructose-2,6-biphosphatases', 'A-kinase anchoring proteins', 'Adenylate kinases', 'C2 domain containing protein kinases', 'CDC like kinases', 'Cyclin-dependent kinases', 'Death-associated protein kinases', 'Diacylglycerol kinases', 'Erb-b2 receptor tyrosine kinases', 'Glycerol kinases', 'MAP kinase phosphatases', 'Membrane-associated guanylate kinases', 'Mitogen-activated protein kinase kinase kinase kinases', 'Mitogen-activated protein kinase kinase kinases', 'Mitogen-activated protein kinase kinases', 'mitogen-activated protein kinase-activated protein kinases', 'Mitogen-activated protein kinases', 'MOB kinase activators', 'Phosphatidylinositol 3-kinase subunits', 'Receptor Tyrosine Kinases', 'SCY1-like, kinase-like proteins', 'Type I receptor serine/threonine kinases', 'Type II receptor serine/threonine kinases']\n",
      "273\n"
     ]
    }
   ],
   "source": [
    "# Count kinases\n",
    "print filter(lambda x: 'kinase' in x.lower(), gene_family['Gene family description'].unique())\n",
    "print len(filter(lambda x: 'kinase' in x.lower(), gene_family['Gene family description']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['E2F transcription factors', 'General transcription factors', 'Kruppel-like transcription factors', 'Sp transcription factors', 'TEA domain transcription factors', 'Transcription factor Dp family']\n",
      "66\n"
     ]
    }
   ],
   "source": [
    "# Count TF\n",
    "print filter(lambda x: 'transcription factor' in x.lower(), gene_family['Gene family description'].unique())\n",
    "print len(filter(lambda x: 'transcription factor' in x.lower(), gene_family['Gene family description']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20086, 1)\n",
      "(2776, 53)\n",
      "(1863, 54)\n"
     ]
    }
   ],
   "source": [
    "# Inner join with proteome data\n",
    "gene_family = gene_family[['Approved Symbol', 'Gene family description']]\n",
    "gene_family.set_index('Approved Symbol', inplace=True)\n",
    "print gene_family.shape\n",
    "\n",
    "proteome = proteome.drop(['UniprotAcc', 'GeneID/Brenda Tissue Ontology BTO:'], axis=1)\n",
    "proteome.set_index('GeneSym', inplace=True)\n",
    "print proteome.shape\n",
    "\n",
    "proteome = proteome.merge(gene_family, left_index=True, right_index=True, how='inner')\n",
    "print proteome.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([u'38', u'675', u'1615', u'568', u'2806', u'1938', u'18', u'4860', u'200001', u'200005', u'3884', u'3323', u'1939', u'2178', u'200004', u'1961', u'200002', u'1908', u'200003', u'179', u'4189', u'2696', u'599', u'1321', u'7', u'93', u'975', u'773', u'200043', u'1008', u'661', u'567', u'1890', u'269', u'200034', u'200045', u'2805', u'1061', u'664', u'562', u'1175', u'759', u'988', u'1158', u'1363', u'1078', u'763', u'2553', u'1129', u'776', u'289', u'2417', u'914'], dtype='object')\n",
      "34 (1863,) (1863, 53)\n"
     ]
    }
   ],
   "source": [
    "# Split X and y\n",
    "X = proteome.drop(['Gene family description'], axis=1)\n",
    "print X.columns\n",
    "X = X.values\n",
    "y = proteome['Gene family description']\n",
    "y = map(lambda x: 'kinase' in x.lower(), y)\n",
    "y = np.array(y, dtype=np.int64)\n",
    "print y.sum(), y.shape, X.shape\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}