tf_cubic_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dlituiev/repos/scikit-learn/sklearn/cross_validation.py:43: DeprecationWarning: This module has been deprecated in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import pandas as pd\n",
    "import numpy  as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "#######################\n",
    "def batchgen(batchsize):\n",
    "\n",
    "    def getbatch(x,y):\n",
    "        assert (len(x) == len(y)), \"dimension mismatch\"\n",
    "        for i in range(0, len(y), batchsize):\n",
    "            yield x[i:i+batchsize], y[i:i+batchsize],\n",
    "    return getbatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def summary_dict(summary_str, summary_proto = None):\n",
    "    \"convert summary string to a dictionary\"\n",
    "    if summary_proto is None:\n",
    "        summary_proto = tf.Summary()\n",
    "    summary_proto.ParseFromString(summary_str)\n",
    "    summaries = {}\n",
    "    for val in summary_proto.value:\n",
    "        # Assuming all summaries are scalars.\n",
    "        summaries[val.tag] = val.simple_value\n",
    "    return summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tflearn import tflearn, vardict\n",
    "\n",
    "class tflasso(tflearn):\n",
    "    def _create_network(self):\n",
    "        self.vars = vardict()\n",
    "        self.vars.x = tf.placeholder(\"float\", shape=[None, self.xlen])\n",
    "        self.vars.y = tf.placeholder(\"float\", shape=[None, 1])\n",
    "\n",
    "        #def fully_connected():\n",
    "            \n",
    "        # Create Model\n",
    "        self.parameters[\"W1\"] = tf.Variable(tf.truncated_normal([1, self.xlen], stddev=0.1), name=\"weight\")\n",
    "        self.parameters[\"b1\"] = tf.Variable(tf.constant(0.1, shape=[1, 1]), name=\"bias\")\n",
    "        \n",
    "        \n",
    "        self.vars.y_predicted = tf.matmul( self.vars.x, tf.transpose(self.W1)) + self.b1\n",
    "        self.saver = tf.train.Saver()\n",
    "        \n",
    "    def _create_loss(self):\n",
    "        # Minimize the squared errors\n",
    "        l2_loss = tf.reduce_mean(tf.pow( self.vars.y_predicted - self.vars.y, 2))\n",
    "        l2_sy = tf.scalar_summary( \"L2_loss\", l2_loss )\n",
    "        \"Lasso penalty\"\n",
    "        l1_penalty = tf.reduce_sum((tf.abs(tf.concat(1, [self.W1,self.b1], name = \"l1\" ) )) )\n",
    "        l1p_sy =  tf.scalar_summary( \"L1_penalty\" , l1_penalty )\n",
    "        \"total\"\n",
    "        tot_loss = l2_loss + self.ALPHA * l1_penalty\n",
    "        tot_loss_sy =  tf.scalar_summary( \"loss\" , tot_loss )\n",
    "        \"R2\"\n",
    "        _, y_var = tf.nn.moments(self.vars.y, [0,1])\n",
    "        rsq =  1 - l2_loss / y_var\n",
    "        rsq_sy = tf.scalar_summary( \"R2\", rsq)\n",
    "        return tot_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../../data/atac_tss_800_1.h5\r\n"
     ]
    }
   ],
   "source": [
    "%ls ../../data/*.h5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'No object named y in the file'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-dff5763bad26>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHDFStore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatafile\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0my_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"y\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mX_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstore\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"X\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/pandas-0.18.0+109.g614fb2f-py3.5-macosx-10.6-intel.egg/pandas/io/pytables.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    415\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    416\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 417\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    418\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    419\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/pandas-0.18.0+109.g614fb2f-py3.5-macosx-10.6-intel.egg/pandas/io/pytables.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    631\u001b[0m         \u001b[0mgroup\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_node\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mgroup\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 633\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'No object named %s in the file'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    634\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'No object named y in the file'"
     ]
    }
   ],
   "source": [
    "datafile = \"../../data/atac_tss_800_1.h5\"\n",
    "\n",
    "with pd.HDFStore(datafile) as store:\n",
    "    print(store.groups())\n",
    "    y_ = store[\"y\"]\n",
    "    X_ = store[\"X\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\" transform data \"\"\"\n",
    "sys.path.append(\"..\")\n",
    "from transform_tss import safelog, sumstrands, groupcolumns\n",
    "\n",
    "feature_step = 100\n",
    "select = list(feature_step * np.arange(-2,3,1))\n",
    "\n",
    "Xgr = groupcolumns(X_, step = feature_step, select = select)\n",
    "\n",
    "X, y = safelog(Xgr, y_)\n",
    "\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "pf3 = PolynomialFeatures(degree=3)\n",
    "X3 = pf3.fit_transform(X)\n",
    "trainsamples = 4000\n",
    "train_X, train_Y = X3[:trainsamples], y[:trainsamples].as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_labels(pf):\n",
    "    return list(pf._combinations(10, degree=pf.degree,\n",
    "                          interaction_only=pf.interaction_only,\n",
    "                          include_bias = pf.include_bias))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tfl = tflasso(ALPHA = 2e-1, checkpoint_dir = \"./cubiclasso/\", dropout = None)\n",
    "tfl.fit( train_X, train_Y , load = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "load = 0\n",
    "g = tf.Graph()\n",
    "with g.as_default():\n",
    "    tfl._create_network()\n",
    "\n",
    "    tot_loss = tfl._create_loss()\n",
    "    summary_op = tf.merge_all_summaries()\n",
    "    sess_config = tf.ConfigProto(inter_op_parallelism_threads=tfl.NUM_CORES,\n",
    "                               intra_op_parallelism_threads= tfl.NUM_CORES)\n",
    "    # Initializing the variables\n",
    "    init = tf.initialize_all_variables()\n",
    "\n",
    "    with tf.Session(config = sess_config) as sess:\n",
    "        if load:\n",
    "            tfl._load_(sess)\n",
    "        else:\n",
    "            sess.run(init)\n",
    "\n",
    "        feed_dict={ tfl.vars.x: train_X, }\n",
    "        if tfl.dropout:\n",
    "            feed_dict[ tfl.vars.keep_prob] = 0.5\n",
    "\n",
    "        y_predicted = sess.run( tfl.vars.y_predicted,\n",
    "                        feed_dict = feed_dict )\n",
    "        if y is not None:\n",
    "            feed_dict[ tfl.vars.y ] = np.reshape(y, [-1, 1])\n",
    "            tfl.summary_proto = tf.Summary()\n",
    "            print(tfl.summary_proto.value)\n",
    "            #summary_str = sess.run(summary_op, feed_dict=feed_dict)\n",
    "            #summary_d = summary_dict(summary_str, tfl.summary_proto)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tfl.summary_proto.DESCRIPTOR.fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ts, r2s = list(zip( *tfl.r2_progress ))\n",
    "plt.plot(ts, r2s)\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.ylabel(\"R^2\")\n",
    "plt.ylim([0, 0.1* np.ceil(10*max(r2s))])\n",
    "pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_X, test_Y = X3[trainsamples:], y[trainsamples:].as_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tfl = tflasso(checkpoint_dir = \"./cubiclasso/\", dropout = False)\n",
    "tfl.transform( test_X, test_Y, load = True)\n",
    "\n",
    "print( tfl.loss )\n",
    "r2 = 1- tfl.loss/test_Y.var()\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pf3.powers_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "W1 = tfl.get_params()[\"W1\"][0]\n",
    "ncoef = len(W1)\n",
    "xlabels = np.array( get_labels(pf3) )\n",
    "\n",
    "forder = np.array([len(x) for x in xlabels])\n",
    "\n",
    "\n",
    "fig, axs = plt.subplots(3,figsize = (14, 5))\n",
    "fig.subplots_adjust(hspace=.5)\n",
    "for nn in range(3):\n",
    "    valid =( forder == (nn+1))\n",
    "    print(sum(valid))\n",
    "    x_ =  np.arange(ncoef)[valid]\n",
    "    y_ = np.log10( abs(W1[valid]) )\n",
    "    axs[nn].scatter(x_, y_ )\n",
    "    axs[nn].scatter( x_[y_>-3], y_[y_>-3], 25, \"r\" )\n",
    "    #axs[nn].stem( x_[y_>-3], y_[y_>-3], markerfmt = \"ro\" )\n",
    "    if nn < 2:\n",
    "        axs[nn].set_xticks(x_ )\n",
    "        axs[nn].set_xticklabels([repr(x) for x in xlabels[valid]], rotation = 90)\n",
    "    else:\n",
    "        axs[nn].set_xticks(x_[::4] )\n",
    "        axs[nn].set_xticklabels([repr(x) for x in xlabels[valid][::4]], rotation = 90)\n",
    "pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.stem?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "W1 = tfl.get_params()[\"W1\"][0]\n",
    "print(len(W1))\n",
    "plt.stem( np.arange(len(W1)), np.log10( abs(W1)) )\n",
    "plt.stem( np.arange(len(W1))[np.log10(W1)>-3], np.log10(W1)[np.log10(W1)>-3], markerfmt = \"ro\" )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}