main_PC_tfomics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RCNN + PC model\n",
    "Scripts for setting up our RCNN + PC model using tfomics (https://github.com/p-koo/tfomics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_curve, auc, precision_recall_curve, accuracy_score, roc_auc_score\n",
    "import sys\n",
    "import h5py\n",
    "import conutils\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "from __future__ import print_function \n",
    "import os, sys\n",
    "from six.moves import cPickle\n",
    "from collections import OrderedDict\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "sys.path.append('../Tensor/tfomics')\n",
    "from tfomics import neuralnetwork as nn\n",
    "from tfomics import utils, learn\n",
    "\n",
    "# import models\n",
    "from model_zoo import fourthplace_connectomics_model\n",
    "from model_zoo import simple_connectomics_model, simple_connectomics_model2\n",
    "from model_zoo import residual_connectomics_model, residual_connectomics_model2,residual_connectomics_model4\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load data -- from https://www.kaggle.com/c/connectomics/data\n",
    "#\n",
    "filename = '../Tensor/kaggle_connect_data/normal_dataset.hdf5'\n",
    "group_name = ['normal_data']\n",
    "dataset = h5py.File(filename,'r')\n",
    "%time F_1 = np.array(dataset['/'+group_name[0]+'/F_1'])\n",
    "scores_1 = np.array(dataset['/'+group_name[0]+'/scores_1'])\n",
    "F_2 = np.array(dataset['/'+group_name[0]+'/F_2'])\n",
    "scores_2 = np.array(dataset['/'+group_name[0]+'/scores_2'])\n",
    "F_3 = np.array(dataset['/'+group_name[0]+'/F_3'])\n",
    "scores_3 = np.array(dataset['/'+group_name[0]+'/scores_3'])\n",
    "F_4 = np.array(dataset['/'+group_name[0]+'/F_4'])\n",
    "scores_4 = np.array(dataset['/'+group_name[0]+'/scores_4'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load network positions for removing light scattering effects\n",
    "#\n",
    "pos = '../Tensor/kaggle_connect_data/normal-1/networkPositions_normal-1.txt'\n",
    "pos_1 = np.loadtxt(pos,delimiter=',')\n",
    "F_1ls = conutils.unscatter(F_1.T,pos_1)\n",
    "\n",
    "pos = '../Tensor/kaggle_connect_data/normal-2/networkPositions_normal-2.txt'\n",
    "pos_2 = np.loadtxt(pos,delimiter=',')\n",
    "F_2ls = conutils.unscatter(F_2.T,pos_2)\n",
    "\n",
    "pos = '../Tensor/kaggle_connect_data/normal-3/networkPositions_normal-3.txt'\n",
    "pos_3 = np.loadtxt(pos,delimiter=',')\n",
    "F_3ls = conutils.unscatter(F_3.T,pos_3)\n",
    "\n",
    "pos = '../Tensor/kaggle_connect_data/normal-4/networkPositions_normal-4.txt'\n",
    "pos_4 = np.loadtxt(pos,delimiter=',')\n",
    "F_4ls = conutils.unscatter(F_4.T,pos_4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Calculate partial correlation metrics for all datasets\n",
    "#\n",
    "pred_out_1 = conutils.get_partial_corr_scores(F_1ls)\n",
    "pred_out_2 = conutils.get_partial_corr_scores(F_2ls)\n",
    "pred_out_3 = conutils.get_partial_corr_scores(F_3ls)\n",
    "pred_out_4 = conutils.get_partial_corr_scores(F_4ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Downsample data\n",
    "#\n",
    "ds_1 = conutils.roma_ds(F_1ls)\n",
    "ds_2 = conutils.roma_ds(F_2ls)\n",
    "ds_3 = conutils.roma_ds(F_3ls)\n",
    "ds_valid = conutils.roma_ds(F_4ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Z-score data\n",
    "#\n",
    "vs_1 = conutils.standardize_rows(ds_1)\n",
    "vs_2 = conutils.standardize_rows(ds_2)\n",
    "vs_3 = conutils.standardize_rows(ds_3)\n",
    "vs_valid = conutils.standardize_rows(ds_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Standardize partial correlation coefficients\n",
    "#\n",
    "p1 = pred_out_1.copy()\n",
    "p2 = pred_out_2.copy()\n",
    "p3 = pred_out_3.copy()\n",
    "p4 = pred_out_4.copy()\n",
    "\n",
    "p1[p1==0] = np.min(p1[p1!=0])\n",
    "p2[p2==0] = np.min(p2[p2!=0])\n",
    "p3[p3==0] = np.min(p3[p3!=0])\n",
    "p4[p4==0] = np.min(p4[p4!=0])\n",
    "\n",
    "p1 = p1 - np.mean(p1)\n",
    "p1 = p1/np.std(p1)\n",
    "\n",
    "p2 = p2 - np.mean(p2)\n",
    "p2 = p2/np.std(p2)\n",
    "\n",
    "p3 = p3 - np.mean(p3)\n",
    "p3 = p3/np.std(p3)\n",
    "\n",
    "p4 = p4 - np.mean(p4)\n",
    "p4 = p4/np.std(p4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create data structure for model input\n",
    "#\n",
    "dtf, ltf = conutils.pairwise_prep_tuple_partialcorr((vs_1,vs_2,vs_3), \n",
    "                                                    (scores_1,scores_2,scores_3), \n",
    "                                                    (p1, p2, p3),\n",
    "                                                   represent=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Separate data into training and cross-validation sets\n",
    "#\n",
    "inds = np.random.choice(dtf.shape[0],replace=False,size=dtf.shape[0])\n",
    "dtf = dtf[inds,:,:,:]\n",
    "ltf = ltf[inds]\n",
    "\n",
    "crossval = dtf.shape[0]//4\n",
    "dtf_crossval = dtf[:crossval,:,:,:]\n",
    "ltf_crossval = ltf[:crossval,:]\n",
    "dtf = dtf[crossval:,:,:,:]\n",
    "ltf = ltf[crossval:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train = dtf\n",
    "y_train = ltf\n",
    "X_valid = dtf_crossval\n",
    "y_valid = ltf_crossval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# get shapes\n",
    "num_data, height, width, dim = X_train.shape\n",
    "input_shape=[None, height, width, dim]\n",
    "num_labels = y_train.shape[1]  \n",
    "\n",
    "# load model\n",
    "net, placeholders, optimization = residual_connectomics_model4.model(input_shape, num_labels)\n",
    "\n",
    "# build neural network class\n",
    "nnmodel = nn.NeuralNet(net, placeholders)\n",
    "nnmodel.inspect_layers()\n",
    "\n",
    "data_path = './'\n",
    "\n",
    "# set output file paths\n",
    "results_path = utils.make_directory(data_path, 'results')\n",
    "output_name = 'dataset1_residual4'\n",
    "filepath = os.path.join(results_path, output_name)\n",
    "\n",
    "# compile neural trainer\n",
    "nntrainer = nn.NeuralTrainer(nnmodel, optimization, save='best', filepath=filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Train model\n",
    "#\n",
    "train = {'inputs': X_train, 'targets': y_train, 'keep_prob_conv': 0.8, 'keep_prob_dense': 0.5, 'is_training': True}\n",
    "valid = {'inputs': X_valid, 'targets': y_valid, 'keep_prob_conv': 1.0, 'keep_prob_dense': 1.0, 'is_training': False}\n",
    "data = {'train': train, 'valid': valid}\n",
    "learn.train_minibatch(nntrainer, data, batch_size=100, num_epochs=200, \n",
    "                      patience=20, verbose=2, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "val_dat = vs_valid\n",
    "val_lbl = scores_4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Evaluate model on validation data\n",
    "#\n",
    "pred_lbl =  conutils.valid_eval_tfomics_partialcorr(nntrainer,val_dat,p4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get ROC-AUC metric\n",
    "#\n",
    "fpr, tpr, thresholds = roc_curve(np.reshape(val_lbl,(1e6,)), pred_lbl)\n",
    "auc(fpr, tpr)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}