01-Extract-Timeseries-Features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GAP_TIME          = 6  # In hours\n",
    "WINDOW_SIZE       = 24 # In hours\n",
    "SEED              = 10\n",
    "ID_COLS           = ['subject_id', 'hadm_id', 'icustay_id']\n",
    "GPU               = '2'\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = GPU\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MIMIC_EXTRACT_DATA = \"data/all_hourly_data.h5\"\n",
    "\n",
    "data_full_lvl2 = pd.read_hdf(MIMIC_EXTRACT_DATA, \"vitals_labs\")\n",
    "data_full_raw  = pd.read_hdf(MIMIC_EXTRACT_DATA, \"vitals_labs\")\n",
    "statics = pd.read_hdf(MIMIC_EXTRACT_DATA, 'patients')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_imputer(df):\n",
    "    idx = pd.IndexSlice\n",
    "    df = df.copy()\n",
    "    if len(df.columns.names) > 2: df.columns = df.columns.droplevel(('label', 'LEVEL1', 'LEVEL2'))\n",
    "    \n",
    "    df_out = df.loc[:, idx[:, ['mean', 'count']]]\n",
    "    icustay_means = df_out.loc[:, idx[:, 'mean']].groupby(ID_COLS).mean()\n",
    "    \n",
    "    df_out.loc[:,idx[:,'mean']] = df_out.loc[:,idx[:,'mean']].groupby(ID_COLS).fillna(\n",
    "        method='ffill'\n",
    "    ).groupby(ID_COLS).fillna(icustay_means).fillna(0)\n",
    "    \n",
    "    df_out.loc[:, idx[:, 'count']] = (df.loc[:, idx[:, 'count']] > 0).astype(float)\n",
    "    df_out.rename(columns={'count': 'mask'}, level='Aggregation Function', inplace=True)\n",
    "    \n",
    "    is_absent = (1 - df_out.loc[:, idx[:, 'mask']])\n",
    "    hours_of_absence = is_absent.cumsum()\n",
    "    time_since_measured = hours_of_absence - hours_of_absence[is_absent==0].fillna(method='ffill')\n",
    "    time_since_measured.rename(columns={'mask': 'time_since_measured'}, level='Aggregation Function', inplace=True)\n",
    "\n",
    "    df_out = pd.concat((df_out, time_since_measured), axis=1)\n",
    "    df_out.loc[:, idx[:, 'time_since_measured']] = df_out.loc[:, idx[:, 'time_since_measured']].fillna(100)\n",
    "    \n",
    "    df_out.sort_index(axis=1, inplace=True)\n",
    "    return df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ys = statics[statics.max_hours > WINDOW_SIZE + GAP_TIME][['mort_hosp', 'mort_icu', 'los_icu']]\n",
    "Ys['los_3'] = Ys['los_icu'] > 3\n",
    "Ys['los_7'] = Ys['los_icu'] > 7\n",
    "Ys.drop(columns=['los_icu'], inplace=True)\n",
    "Ys.astype(float)\n",
    "\n",
    "lvl2, raw = [df[\n",
    "    (df.index.get_level_values('icustay_id').isin(set(Ys.index.get_level_values('icustay_id')))) &\n",
    "    (df.index.get_level_values('hours_in') < WINDOW_SIZE)\n",
    "] for df in (data_full_lvl2, data_full_raw)]\n",
    "\n",
    "raw.columns = raw.columns.droplevel(level=['LEVEL2'])\n",
    "\n",
    "train_frac, dev_frac, test_frac = 0.7, 0.1, 0.2\n",
    "lvl2_subj_idx, raw_subj_idx, Ys_subj_idx = [df.index.get_level_values('subject_id') for df in (lvl2, raw, Ys)]\n",
    "lvl2_subjects = set(lvl2_subj_idx)\n",
    "assert lvl2_subjects == set(Ys_subj_idx), \"Subject ID pools differ!\"\n",
    "assert lvl2_subjects == set(raw_subj_idx), \"Subject ID pools differ!\"\n",
    "\n",
    "np.random.seed(SEED)\n",
    "subjects, N = np.random.permutation(list(lvl2_subjects)), len(lvl2_subjects)\n",
    "N_train, N_dev, N_test = int(train_frac * N), int(dev_frac * N), int(test_frac * N)\n",
    "train_subj = subjects[:N_train]\n",
    "dev_subj   = subjects[N_train:N_train + N_dev]\n",
    "test_subj  = subjects[N_train+N_dev:]\n",
    "\n",
    "[(lvl2_train, lvl2_dev, lvl2_test), (raw_train, raw_dev, raw_test), (Ys_train, Ys_dev, Ys_test)] = [\n",
    "    [df[df.index.get_level_values('subject_id').isin(s)] for s in (train_subj, dev_subj, test_subj)] \\\n",
    "    for df in (lvl2, raw, Ys)\n",
    "]\n",
    "\n",
    "idx = pd.IndexSlice\n",
    "lvl2_means, lvl2_stds = lvl2_train.loc[:, idx[:,'mean']].mean(axis=0), lvl2_train.loc[:, idx[:,'mean']].std(axis=0)\n",
    "\n",
    "lvl2_train.loc[:, idx[:,'mean']] = (lvl2_train.loc[:, idx[:,'mean']] - lvl2_means)/lvl2_stds\n",
    "lvl2_dev.loc[:, idx[:,'mean']] = (lvl2_dev.loc[:, idx[:,'mean']] - lvl2_means)/lvl2_stds\n",
    "lvl2_test.loc[:, idx[:,'mean']] = (lvl2_test.loc[:, idx[:,'mean']] - lvl2_means)/lvl2_stds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lvl2_train, lvl2_dev, lvl2_test = [\n",
    "    simple_imputer(df) for df in (lvl2_train, lvl2_dev, lvl2_test)\n",
    "]\n",
    "lvl2_flat_train, lvl2_flat_dev, lvl2_flat_test = [\n",
    "    df.pivot_table(index=['subject_id', 'hadm_id', 'icustay_id'], columns=['hours_in']) for df in (\n",
    "       lvl2_train, lvl2_dev, lvl2_test\n",
    "    )\n",
    "]\n",
    "\n",
    "for df in lvl2_train, lvl2_dev, lvl2_test: assert not df.isnull().any().any()\n",
    "    \n",
    "[(Ys_train, Ys_dev, Ys_test)] = [\n",
    "[df[df.index.get_level_values('subject_id').isin(s)] for s in (train_subj, dev_subj, test_subj)] \\\n",
    "for df in (Ys,)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(lvl2_train, \"data/lvl2_imputer_train.pkl\")\n",
    "pd.to_pickle(lvl2_dev, \"data/lvl2_imputer_dev.pkl\")\n",
    "pd.to_pickle(lvl2_test, \"data/lvl2_imputer_test.pkl\")\n",
    "\n",
    "pd.to_pickle(Ys, \"data/Ys.pkl\")\n",
    "pd.to_pickle(Ys_train, \"data/Ys_train.pkl\")\n",
    "pd.to_pickle(Ys_dev, \"data/Ys_dev.pkl\")\n",
    "pd.to_pickle(Ys_test, \"data/Ys_test.pkl\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}