aistats data munging.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "BASEDIR = \"/home/dmoore/python/sparsegp/experiments/\"\n",
      "import os,sys\n",
      "import numpy as np\n",
      "\n",
      "if '/home/dmoore/python/' not in sys.path:\n",
      "    sys.path.append('/home/dmoore/python/')\n",
      "\n",
      "from sparsegp.experiments.code.datasets import mkdir_p\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "def standardizeX(dfX, ignore_cols=[]):\n",
      "    for col in dfX.columns:\n",
      "        if col in ignore_cols: continue\n",
      "                \n",
      "        mx = dfX[col].max()\n",
      "        mn = dfX[col].min()\n",
      "        print col, mx, mn\n",
      "        dfX[col] = (dfX[col] - mn) / ((mx-mn)/100.0 )\n",
      "        print col, mx, mn, (mx-mn)\n",
      "        \n",
      "def save_test_train_split(Xframe, yframe, dataset, seed=0, n_train=None):\n",
      "\n",
      "    n = Xframe.shape[0]\n",
      "    \n",
      "    n_train = int(n*.7) if n_train is None else n_train\n",
      "    \n",
      "    np.random.seed(seed)\n",
      "    p = np.random.permutation(n)\n",
      "        \n",
      "    train_idx = p[:n_train]\n",
      "    test_idx = p[n_train:]\n",
      "\n",
      "    X_train = Xframe.iloc[train_idx,:]\n",
      "    y_train = yframe.iloc[train_idx,:]\n",
      "    \n",
      "\n",
      "    \n",
      "    X_test = Xframe.iloc[test_idx,:]\n",
      "    y_test = yframe.iloc[test_idx,:]\n",
      "    \n",
      "    mu = np.mean(y_train)\n",
      "    sigma = np.std(y_train)\n",
      "    y_train = (y_train - mu)/sigma\n",
      "    y_test = (y_test - mu)/sigma\n",
      "    \n",
      "    savedir = os.path.join(BASEDIR, 'datasets', dataset)\n",
      "    mkdir_p(savedir)\n",
      "    \n",
      "    X_train.to_csv(os.path.join(savedir, 'X_train.txt'), index=False)\n",
      "    y_train.to_csv(os.path.join(savedir, 'y_train.txt'), index=False)\n",
      "    \n",
      "    X_test.to_csv(os.path.join(savedir, 'X_test.txt'), index=False)\n",
      "    y_test.to_csv(os.path.join(savedir, 'y_test.txt'), index=False)\n",
      "\n",
      "    \n",
      "    "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "\n",
      "h1 = pd.read_csv(os.path.join(BASEDIR, 'raw_data/cal_housing_data.csv'))\n",
      "h2 = pd.read_csv(os.path.join(BASEDIR, 'raw_data/cal_housing_dbf.csv'))\n",
      "h2 = h2[np.isfinite(h2['BLCKGR'])]\n",
      "h2 = h2[h2['H061A001'] > 0]\n",
      "print h1\n",
      "print h2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "       lon    lat  age  income  units  population  households   rooms   value\n",
        "0  -122.23  37.88   41     880    129         322         126  8.3252  452600\n",
        "1  -122.22  37.86   21    7099   1106        2401        1138  8.3014  358500\n",
        "2  -122.24  37.85   52    1467    190         496         177  7.2574  352100\n",
        "3  -122.25  37.85   52    1274    235         558         219  5.6431  341300\n",
        "4  -122.25  37.85   52    1627    280         565         259  3.8462  342200\n",
        "5  -122.25  37.85   52     919    213         413         193  4.0368  269700\n",
        "6  -122.25  37.84   52    2535    489        1094         514  3.6591  299200\n",
        "7  -122.25  37.84   52    3104    687        1157         647  3.1200  241400\n",
        "8  -122.26  37.84   42    2555    665        1206         595  2.0804  226700\n",
        "9  -122.25  37.84   52    3549    707        1551         714  3.6912  261100\n",
        "10 -122.26  37.85   52    2202    434         910         402  3.2031  281500\n",
        "11 -122.26  37.85   52    3503    752        1504         734  3.2705  241800\n",
        "12 -122.26  37.85   52    2491    474        1098         468  3.0750  213500\n",
        "13 -122.26  37.84   52     696    191         345         174  2.6736  191300\n",
        "14 -122.26  37.85   52    2643    626        1212         620  1.9167  159200\n",
        "15 -122.26  37.85   50    1120    283         697         264  2.1250  140000\n",
        "16 -122.27  37.85   52    1966    347         793         331  2.7750  152500\n",
        "17 -122.27  37.85   52    1228    293         648         303  2.1202  155500\n",
        "18 -122.26  37.84   50    2239    455         990         419  1.9911  158700\n",
        "19 -122.27  37.84   52    1503    298         690         275  2.6033  162900\n",
        "20 -122.27  37.85   40     751    184         409         166  1.3578  147500\n",
        "21 -122.27  37.85   42    1639    367         929         366  1.7135  159800\n",
        "22 -122.27  37.84   52    2436    541        1015         478  1.7250  113900\n",
        "23 -122.27  37.84   52    1688    337         853         325  2.1806   99700\n",
        "24 -122.27  37.84   52    2224    437        1006         422  2.6000  132600\n",
        "25 -122.28  37.85   41     535    123         317         119  2.4038  107500\n",
        "26 -122.28  37.85   49    1130    244         607         239  2.4597   93800\n",
        "27 -122.28  37.85   52    1898    421        1102         397  1.8080  105500\n",
        "28 -122.28  37.84   50    2082    492        1131         473  1.6424  108900\n",
        "29 -122.28  37.84   52     729    160         395         155  1.6875  132000\n",
        "30 -122.28  37.84   49    1916    447         863         378  1.9274  122300\n",
        "31 -122.28  37.84   52    2153    481        1168         441  1.9615  115200\n",
        "32 -122.27  37.84   48    1922    409        1026         335  1.7969  110400\n",
        "33 -122.27  37.83   49    1655    366         754         329  1.3750  104900\n",
        "34 -122.27  37.83   51    2665    574        1258         536  2.7303  109700\n",
        "35 -122.27  37.83   49    1215    282         570         264  1.4861   97200\n",
        "36 -122.27  37.83   48    1798    432         987         374  1.0972  104500\n",
        "37 -122.28  37.83   52    1511    390         901         403  1.4103  103900\n",
        "38 -122.26  37.83   52    1470    330         689         309  3.4800  191400\n",
        "39 -122.26  37.83   52    2432    715        1377         696  2.5898  176000\n",
        "40 -122.26  37.83   52    1665    419         946         395  2.0978  155400\n",
        "41 -122.26  37.83   51     936    311         517         249  1.2852  150000\n",
        "42 -122.26  37.84   49     713    202         462         189  1.0250  118800\n",
        "43 -122.26  37.84   52     950    202         467         198  3.9643  188800\n",
        "44 -122.26  37.83   52    1443    311         660         292  3.0125  184400\n",
        "45 -122.26  37.83   52    1656    420         718         382  2.6768  182300\n",
        "46 -122.26  37.83   50    1125    322         616         304  2.0260  142500\n",
        "47 -122.27  37.82   43    1007    312         558         253  1.7348  137500\n",
        "48 -122.26  37.82   40     624    195         423         160  0.9506  187500\n",
        "49 -122.27  37.82   40     946    375         700         352  1.7750  112500\n",
        "50 -122.27  37.82   21     896    453         735         438  0.9218  171900\n",
        "51 -122.27  37.82   43    1868    456        1061         407  1.5045   93800\n",
        "52 -122.27  37.82   41    3221    853        1959         720  1.1108   97500\n",
        "53 -122.27  37.82   52    1630    456        1162         400  1.2475  104200\n",
        "54 -122.28  37.82   52    1170    235         701         233  1.6098   87500\n",
        "55 -122.28  37.82   52     945    243         576         220  1.4113   83100\n",
        "56 -122.28  37.82   52    1238    288         622         259  1.5057   87500\n",
        "57 -122.28  37.82   52    1489    335         728         244  0.8172   85300\n",
        "58 -122.28  37.82   52    1387    341        1074         304  1.2171   80300\n",
        "59 -122.29  37.82    2     158     43          94          57  2.5625   60000\n",
        "       ...    ...  ...     ...    ...         ...         ...     ...     ...\n",
        "\n",
        "[20640 rows x 9 columns]\n",
        "     FIPS     TRACT         BLOCKGR  SUMLEV  STATEFP  CNTY  COUSUBFP  PLACEFP  \\\n",
        "901  6001  60014001  060014001.00:1     150        6     1       NaN      NaN   \n",
        "902  6001  60014001  060014001.00:2     150        6     1       NaN      NaN   \n",
        "904  6001  60014002  060014002.00:1     150        6     1       NaN      NaN   \n",
        "905  6001  60014002  060014002.00:2     150        6     1       NaN      NaN   \n",
        "906  6001  60014002  060014002.00:3     150        6     1       NaN      NaN   \n",
        "907  6001  60014002  060014002.00:4     150        6     1       NaN      NaN   \n",
        "909  6001  60014003  060014003.00:1     150        6     1       NaN      NaN   \n",
        "910  6001  60014003  060014003.00:2     150        6     1       NaN      NaN   \n",
        "911  6001  60014003  060014003.00:3     150        6     1       NaN      NaN   \n",
        "912  6001  60014003  060014003.00:4     150        6     1       NaN      NaN   \n",
        "914  6001  60014004  060014004.00:1     150        6     1       NaN      NaN   \n",
        "915  6001  60014004  060014004.00:2     150        6     1       NaN      NaN   \n",
        "916  6001  60014004  060014004.00:3     150        6     1       NaN      NaN   \n",
        "917  6001  60014004  060014004.00:4     150        6     1       NaN      NaN   \n",
        "919  6001  60014005  060014005.00:1     150        6     1       NaN      NaN   \n",
        "920  6001  60014005  060014005.00:2     150        6     1       NaN      NaN   \n",
        "921  6001  60014005  060014005.00:3     150        6     1       NaN      NaN   \n",
        "922  6001  60014005  060014005.00:4     150        6     1       NaN      NaN   \n",
        "924  6001  60014006  060014006.00:1     150        6     1       NaN      NaN   \n",
        "925  6001  60014006  060014006.00:2     150        6     1       NaN      NaN   \n",
        "927  6001  60014007  060014007.00:1     150        6     1       NaN      NaN   \n",
        "928  6001  60014007  060014007.00:2     150        6     1       NaN      NaN   \n",
        "929  6001  60014007  060014007.00:3     150        6     1       NaN      NaN   \n",
        "930  6001  60014007  060014007.00:4     150        6     1       NaN      NaN   \n",
        "931  6001  60014007  060014007.00:5     150        6     1       NaN      NaN   \n",
        "933  6001  60014008  060014008.00:1     150        6     1       NaN      NaN   \n",
        "934  6001  60014008  060014008.00:2     150        6     1       NaN      NaN   \n",
        "935  6001  60014008  060014008.00:3     150        6     1       NaN      NaN   \n",
        "936  6001  60014008  060014008.00:4     150        6     1       NaN      NaN   \n",
        "938  6001  60014009  060014009.00:1     150        6     1       NaN      NaN   \n",
        "939  6001  60014009  060014009.00:2     150        6     1       NaN      NaN   \n",
        "940  6001  60014009  060014009.00:3     150        6     1       NaN      NaN   \n",
        "942  6001  60014010  060014010.00:1     150        6     1       NaN      NaN   \n",
        "943  6001  60014010  060014010.00:2     150        6     1       NaN      NaN   \n",
        "944  6001  60014010  060014010.00:3     150        6     1       NaN      NaN   \n",
        "945  6001  60014010  060014010.00:4     150        6     1       NaN      NaN   \n",
        "946  6001  60014010  060014010.00:5     150        6     1       NaN      NaN   \n",
        "947  6001  60014010  060014010.00:6     150        6     1       NaN      NaN   \n",
        "949  6001  60014011  060014011.00:1     150        6     1       NaN      NaN   \n",
        "950  6001  60014011  060014011.00:2     150        6     1       NaN      NaN   \n",
        "951  6001  60014011  060014011.00:3     150        6     1       NaN      NaN   \n",
        "952  6001  60014011  060014011.00:4     150        6     1       NaN      NaN   \n",
        "955  6001  60014012  060014012.00:1     150        6     1       NaN      NaN   \n",
        "956  6001  60014012  060014012.00:2     150        6     1       NaN      NaN   \n",
        "957  6001  60014012  060014012.00:3     150        6     1       NaN      NaN   \n",
        "958  6001  60014012  060014012.00:4     150        6     1       NaN      NaN   \n",
        "960  6001  60014013  060014013.00:1     150        6     1       NaN      NaN   \n",
        "961  6001  60014013  060014013.00:2     150        6     1       NaN      NaN   \n",
        "963  6001  60014013  060014013.00:4     150        6     1       NaN      NaN   \n",
        "965  6001  60014014  060014014.00:1     150        6     1       NaN      NaN   \n",
        "966  6001  60014014  060014014.00:2     150        6     1       NaN      NaN   \n",
        "967  6001  60014014  060014014.00:3     150        6     1       NaN      NaN   \n",
        "969  6001  60014015  060014015.00:1     150        6     1       NaN      NaN   \n",
        "970  6001  60014015  060014015.00:2     150        6     1       NaN      NaN   \n",
        "971  6001  60014015  060014015.00:3     150        6     1       NaN      NaN   \n",
        "973  6001  60014016  060014016.00:1     150        6     1       NaN      NaN   \n",
        "974  6001  60014016  060014016.00:2     150        6     1       NaN      NaN   \n",
        "977  6001  60014017  060014017.00:1     150        6     1       NaN      NaN   \n",
        "978  6001  60014017  060014017.00:2     150        6     1       NaN      NaN   \n",
        "980  6001  60014017  060014017.00:4     150        6     1       NaN      NaN   \n",
        "      ...       ...             ...     ...      ...   ...       ...      ...   \n",
        "\n",
        "     TRACTBNA  BLCKGR  LOGRECNU  P0010001  P0050001  P0060001  P0060002  \\\n",
        "901      4001       1     46052       322       134       322         0   \n",
        "902      4001       2     46053      2401      1110      2401         0   \n",
        "904      4002       1     46054       496       188       496         0   \n",
        "905      4002       2     46055       558       221       558         0   \n",
        "906      4002       3     46056       565       240       565         0   \n",
        "907      4002       4     46057       413       227       413         0   \n",
        "909      4003       1     46058      1094       492      1094         0   \n",
        "910      4003       2     46059      1157       578      1157         0   \n",
        "911      4003       3     46060      1206       652      1206         0   \n",
        "912      4003       4     46061      1551       740      1551         0   \n",
        "914      4004       1     46062       910       415       910         0   \n",
        "915      4004       2     46063      1504       757      1504         0   \n",
        "916      4004       3     46064      1098       446      1098         0   \n",
        "917      4004       4     46065       345       165       345         0   \n",
        "919      4005       1     46066      1212       584      1212         0   \n",
        "920      4005       2     46067       697       266       697         0   \n",
        "921      4005       3     46068       793       330       793         0   \n",
        "922      4005       4     46069       648       301       648         0   \n",
        "924      4006       1     46070       990       477       990         0   \n",
        "925      4006       2     46071       690       233       690         0   \n",
        "927      4007       1     46072       409       195       409         0   \n",
        "928      4007       2     46073       929       363       929         0   \n",
        "929      4007       3     46074      1015       464      1015         0   \n",
        "930      4007       4     46075       853       305       853         0   \n",
        "931      4007       5     46076      1006       394      1006         0   \n",
        "933      4008       1     46077       317       110       317         0   \n",
        "934      4008       2     46078       607       186       607         0   \n",
        "935      4008       3     46079      1102       391      1102         0   \n",
        "936      4008       4     46080      1131       487      1131         0   \n",
        "938      4009       1     46081       395       139       395         0   \n",
        "939      4009       2     46082       863       362       863         0   \n",
        "940      4009       3     46083      1168       448      1168         0   \n",
        "942      4010       1     46084      1026       327      1026         0   \n",
        "943      4010       2     46085       754       287       754         0   \n",
        "944      4010       3     46086      1258       516      1258         0   \n",
        "945      4010       4     46087       570       249       570         0   \n",
        "946      4010       5     46088       987       385       987         0   \n",
        "947      4010       6     46089       901       444       901         0   \n",
        "949      4011       1     46090       689       294       689         0   \n",
        "950      4011       2     46091      1377       697      1377         0   \n",
        "951      4011       3     46092       946       344       946         0   \n",
        "952      4011       4     46093       517       241       517         0   \n",
        "955      4012       1     46095       467       184       467         0   \n",
        "956      4012       2     46096       660       269       660         0   \n",
        "957      4012       3     46097       718       416       718         0   \n",
        "958      4012       4     46098       616       311       616         0   \n",
        "960      4013       1     46099       558       251       558         0   \n",
        "961      4013       2     46100       423       186       423         0   \n",
        "963      4013       4     46102       735       469       735         0   \n",
        "965      4014       1     46103      1061       460      1061         0   \n",
        "966      4014       2     46104      1959       666      1959         0   \n",
        "967      4014       3     46105      1162       379      1162         0   \n",
        "969      4015       1     46106       701       223       701         0   \n",
        "970      4015       2     46107       576       192       576         0   \n",
        "971      4015       3     46108       622       259       622         0   \n",
        "973      4016       1     46109       728       323       728         0   \n",
        "974      4016       2     46110      1074       287      1074         0   \n",
        "977      4017       1     46112       554       136       554         0   \n",
        "978      4017       2     46113        86        13        86         0   \n",
        "980      4017       4     46115       377       161       377         0   \n",
        "          ...     ...       ...       ...       ...       ...       ...   \n",
        "\n",
        "     P0060003  P0060004  P0100001  P0120001  P0120002      \n",
        "901         0         0         0       298         0 ...  \n",
        "902         0         0        78      2020       111 ...  \n",
        "904         0         0         0       404        47 ...  \n",
        "905         0         0        19       496        19 ...  \n",
        "906         0         0        54       454         0 ...  \n",
        "907         0         0        39       343        23 ...  \n",
        "909         0         0        48       977        23 ...  \n",
        "910         0         0        54       873        84 ...  \n",
        "911         0         0       142       546       454 ...  \n",
        "912         0         0       127      1201       148 ...  \n",
        "914         0         0        42       722        70 ...  \n",
        "915         0         0        60      1004       322 ...  \n",
        "916         0         0        87       561       383 ...  \n",
        "917         0         0        15       181        85 ...  \n",
        "919         0         0        73       574       465 ...  \n",
        "920         0         0        50       155       446 ...  \n",
        "921         0         0        16       100       621 ...  \n",
        "922         0         0        18       259       326 ...  \n",
        "924         0         0        19       281       642 ...  \n",
        "925         0         0        87       102       455 ...  \n",
        "927         0         0        36        35       334 ...  \n",
        "928         0         0        16       123       739 ...  \n",
        "929         0         0        97       157       733 ...  \n",
        "930         0         0        23        67       753 ...  \n",
        "931         0         0         0       121       832 ...  \n",
        "933         0         0         0        49       261 ...  \n",
        "934         0         0         0        71       440 ...  \n",
        "935         0         0        52        64       923 ...  \n",
        "936         0         0        34       315       697 ...  \n",
        "938         0         0        18        48       329 ...  \n",
        "939         0         0        24       124       666 ...  \n",
        "940         0         0        21       276       822 ...  \n",
        "942         0         0         8        38       980 ...  \n",
        "943         0         0        61       165       528 ...  \n",
        "944         0         0       104       121       938 ...  \n",
        "945         0         0        20        75       441 ...  \n",
        "946         0         0        73        12       747 ...  \n",
        "947         0         0        17        18       866 ...  \n",
        "949         0         0        13       388       178 ...  \n",
        "950         0         0       209       515       573 ...  \n",
        "951         0         0        94       159       380 ...  \n",
        "952         0         0        32       152       218 ...  \n",
        "955         0         0        18       269       168 ...  \n",
        "956         0         0        23       370       146 ...  \n",
        "957         0         0        19       486       179 ...  \n",
        "958         0         0        25       296       195 ...  \n",
        "960         0         0        20       167       290 ...  \n",
        "961         0         0        35       102       248 ...  \n",
        "963         0         0        21       245       309 ...  \n",
        "965         0         0         0        31       878 ...  \n",
        "966         0         0       102       179      1606 ...  \n",
        "967         0         0       125        66       839 ...  \n",
        "969         0         0        76        42       583 ...  \n",
        "970         0         0        29        51       496 ...  \n",
        "971         0         0        23        56       504 ...  \n",
        "973         0         0        30        22       654 ...  \n",
        "974         0         0        54        99       668 ...  \n",
        "977         0         0       192        68       254 ...  \n",
        "978         0         0        45         0        41 ...  \n",
        "980         0         0        13        10       354 ...  \n",
        "          ...       ...       ...       ...       ...      \n",
        "\n",
        "[20077 rows x 99 columns]\n"
       ]
      }
     ],
     "prompt_number": 61
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "(h2[h2['H061A001'] == 14999])['P0050001']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 62,
       "text": [
        "3788     114\n",
        "4600      50\n",
        "4682     117\n",
        "5241     213\n",
        "15615    505\n",
        "18885    616\n",
        "26229    331\n",
        "Name: P0050001, dtype: float64"
       ]
      }
     ],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "h1[h1['value'] == 14999]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>lon</th>\n",
        "      <th>lat</th>\n",
        "      <th>age</th>\n",
        "      <th>income</th>\n",
        "      <th>units</th>\n",
        "      <th>population</th>\n",
        "      <th>households</th>\n",
        "      <th>rooms</th>\n",
        "      <th>value</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>2521 </th>\n",
        "      <td>-122.74</td>\n",
        "      <td> 39.71</td>\n",
        "      <td> 16</td>\n",
        "      <td> 255</td>\n",
        "      <td>  73</td>\n",
        "      <td>  85</td>\n",
        "      <td>  38</td>\n",
        "      <td> 1.6607</td>\n",
        "      <td> 14999</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2799 </th>\n",
        "      <td>-117.02</td>\n",
        "      <td> 36.40</td>\n",
        "      <td> 19</td>\n",
        "      <td> 619</td>\n",
        "      <td> 239</td>\n",
        "      <td> 490</td>\n",
        "      <td> 164</td>\n",
        "      <td> 2.1000</td>\n",
        "      <td> 14999</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9188 </th>\n",
        "      <td>-117.86</td>\n",
        "      <td> 34.24</td>\n",
        "      <td> 52</td>\n",
        "      <td> 803</td>\n",
        "      <td> 267</td>\n",
        "      <td> 628</td>\n",
        "      <td> 225</td>\n",
        "      <td> 4.1932</td>\n",
        "      <td> 14999</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19802</th>\n",
        "      <td>-123.17</td>\n",
        "      <td> 40.31</td>\n",
        "      <td> 36</td>\n",
        "      <td>  98</td>\n",
        "      <td>  28</td>\n",
        "      <td>  18</td>\n",
        "      <td>   8</td>\n",
        "      <td> 0.5360</td>\n",
        "      <td> 14999</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>4 rows \u00d7 9 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 63,
       "text": [
        "          lon    lat  age  income  units  population  households   rooms  \\\n",
        "2521  -122.74  39.71   16     255     73          85          38  1.6607   \n",
        "2799  -117.02  36.40   19     619    239         490         164  2.1000   \n",
        "9188  -117.86  34.24   52     803    267         628         225  4.1932   \n",
        "19802 -123.17  40.31   36      98     28          18           8  0.5360   \n",
        "\n",
        "       value  \n",
        "2521   14999  \n",
        "2799   14999  \n",
        "9188   14999  \n",
        "19802  14999  \n",
        "\n",
        "[4 rows x 9 columns]"
       ]
      }
     ],
     "prompt_number": 63
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 63
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print (h2['H061A001'] > 0).sum()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "20077\n"
       ]
      }
     ],
     "prompt_number": 64
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "precip = pd.read_csv(os.path.join(BASEDIR, 'raw_data/NCAR_pinfill/ppt.complete.Y101'),sep=r'\\s*', names=['sta', 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec', 'missing'], index_col=0)\n",
      "precip_full = precip[precip['missing'] == 000000000000]\n",
      "print precip_full"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "        jan  feb  mar  apr  may  jun  jul  aug  sep  oct  nov  dec  missing\n",
        "sta                                                                        \n",
        "010008   94  174  126   78   58  113  151  109   61  198  131   99        0\n",
        "010063  118  139  131  132  109  119   75   67   81  110  162  126        0\n",
        "010140  123   88  125  116   50  105   55  151   71  236  164  202        0\n",
        "010184   86   99  155  177  102   73   51  150   49  202  211  103        0\n",
        "010252  196  203  143  124  168   96  229  184   34  424  162  150        0\n",
        "010369  107  267  151   79   36  139   55  119   82  301  185  139        0\n",
        "010390  107   81  109  146   81   60  102  106   89  203  138   77        0\n",
        "010402  169  216  180  160  170   55  233  272   36  254  199  168        0\n",
        "010440  128  117  105  113   24   28  131   85  103  223  109  167        0\n",
        "010505  122   98  131  121   48  160   69   60  101  232  181  107        0\n",
        "010583  158  113  188  193  353  108  179  339   70  272  217  184        0\n",
        "010655   96  122   92  122   46   69   76   94  193  177  139   80        0\n",
        "010764  106  212  101  105  163   84   22   53  105  326  202   82        0\n",
        "010831   98  111   92  112   80   78   46   38  140  302  177  125        0\n",
        "010957  139  137  110  108   33   97  115  114  104  264  137   84        0\n",
        "011080  186  224  211  190  211   73  199  258   50  507  188  136        0\n",
        "011084  165  193  171  224  237  116  167  292   23  380  194   81        0\n",
        "011099  148  115  102   68   81  101  132   93  145  200  205   88        0\n",
        "011225   75  176   90  120   90   93   27   72  113  167  210   46        0\n",
        "011301  127   84  150  130   94   30   93  143   26  251  151  201        0\n",
        "011324   95  155  136   59   23   54   75  150   53  245   86  132        0\n",
        "011377  122  160  145  150   51  144   77   44   85  209  140  135        0\n",
        "011525  114  197  139  186   94   49   46  123   81  242  203  123        0\n",
        "011566  216  126  189  179   88   64  120  195   42  166  119  193        0\n",
        "011620  103  344  127  162   39   58   44  139   89  250  170  118        0\n",
        "011690  157  122  109  127   71   56  115  120   34  210  148  208        0\n",
        "011803  176   47  299  191  136  155  232  105   74  211  237  114        0\n",
        "012245  113  149  187  128  103   54   61   88   60  248  146  125        0\n",
        "012632   95  125  110  135   64   95   96  115   55  183  156  114        0\n",
        "012758  183  188  165  131  130   73  126  300   62  303  151  141        0\n",
        "012813  138   72  255  170  153  115  220  244   77  308  295  123        0\n",
        "012883  120  132  203  150   45  131   67   95   81  164  149  110        0\n",
        "013160   79  129  165  145   59   45   54  109  116  146  162  121        0\n",
        "013271  135  130  147  135   77  132   99  165    0  316  129   72        0\n",
        "013511   79  155  130  188  122   65   54  131   64  254   95  118        0\n",
        "013519  109  109  134  114   52   48  139  103   15  327  115  159        0\n",
        "013620  155  205  138  238  131  142  158   83   95  142  171  129        0\n",
        "013761  111  216  133  114   74   97  134  131   43  154  113   61        0\n",
        "013775  100  213  142  112   77   72   57   57  149  255  193   97        0\n",
        "013816  143  132  132   90   69  161   51  207   36  308  111  108        0\n",
        "013842   85  167  116  128   21   86   46   71   90  278  152  116        0\n",
        "013899  165  166  136  242  132   93  131   66   63  129  123   96        0\n",
        "013930   91  132  132   97   69   90  105  111   51  170  159   78        0\n",
        "014064   89  178   84  114   86  117  105   35  117  173  131  105        0\n",
        "014209   83  118  102  123   71   66   99   66  141  291  168   89        0\n",
        "014226  140  150  147  161   64  130   63   33  106  204  195  121        0\n",
        "014431  165  181  160  145  105   95  108  172   39  320  189   71        0\n",
        "014619   92  100   96  117   89  112   12  114   94  239  178  119        0\n",
        "014798   64  123  140  173  109   50   59   49   38  146  106   82        0\n",
        "015112  125  230  139  146   69   68   62  160   83  242  143  128        0\n",
        "015121  127  107  150  137   33   69   35  135  161  224  120  183        0\n",
        "015420  123  110  143   70   89   15   92  166   35  220  148  192        0\n",
        "015478  180   76  217  189  307   84  118  193   29  221  204  225        0\n",
        "015635  117  178  113  162  184   83   66   50  113  200  124  125        0\n",
        "015658   90  178  171  133   44   48  198   99   81  216  154   91        0\n",
        "016121  133  153  121  104   90  106   91  111  163  313  144   90        0\n",
        "016129   94  134  122   63   32   52   52   99   51  391  118  113        0\n",
        "016246  105  129  151  135   58   89   41  111  113  314  172  105        0\n",
        "016334  127  101  178  186  170   58  130  103   45  208  144  147        0\n",
        "016478   98  130  119  103   59   85   56   55  108  271  161   95        0\n",
        "        ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...      ...\n",
        "\n",
        "[5776 rows x 13 columns]\n"
       ]
      }
     ],
     "prompt_number": 65
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "precip_sta = pd.read_csv(os.path.join(BASEDIR, 'raw_data/NCAR_pinfill/METAinfo'),sep=r'\\s*', names=['sta', 'lon', 'lat', 'elev'], index_col=0, header=1)\n",
      "precip_joined = precip_full.join(precip_sta, how='inner')\n",
      "precip_joined['annual'] = precip_joined['jan']+precip_joined['feb']+precip_joined['mar']+precip_joined['apr']+precip_joined['may']+precip_joined['jun']+precip_joined['jul']+precip_joined['aug']+precip_joined['sep']+precip_joined['oct']+precip_joined['nov']+precip_joined['dec']\n",
      "print precip_joined"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "        jan  feb  mar  apr  may  jun  jul  aug  sep  oct  nov  dec  missing  \\\n",
        "sta                                                                           \n",
        "010063  118  139  131  132  109  119   75   67   81  110  162  126        0   \n",
        "010140  123   88  125  116   50  105   55  151   71  236  164  202        0   \n",
        "010184   86   99  155  177  102   73   51  150   49  202  211  103        0   \n",
        "010252  196  203  143  124  168   96  229  184   34  424  162  150        0   \n",
        "010369  107  267  151   79   36  139   55  119   82  301  185  139        0   \n",
        "010390  107   81  109  146   81   60  102  106   89  203  138   77        0   \n",
        "010402  169  216  180  160  170   55  233  272   36  254  199  168        0   \n",
        "010440  128  117  105  113   24   28  131   85  103  223  109  167        0   \n",
        "010505  122   98  131  121   48  160   69   60  101  232  181  107        0   \n",
        "010583  158  113  188  193  353  108  179  339   70  272  217  184        0   \n",
        "010655   96  122   92  122   46   69   76   94  193  177  139   80        0   \n",
        "010764  106  212  101  105  163   84   22   53  105  326  202   82        0   \n",
        "010831   98  111   92  112   80   78   46   38  140  302  177  125        0   \n",
        "010957  139  137  110  108   33   97  115  114  104  264  137   84        0   \n",
        "011080  186  224  211  190  211   73  199  258   50  507  188  136        0   \n",
        "011084  165  193  171  224  237  116  167  292   23  380  194   81        0   \n",
        "011099  148  115  102   68   81  101  132   93  145  200  205   88        0   \n",
        "011225   75  176   90  120   90   93   27   72  113  167  210   46        0   \n",
        "011301  127   84  150  130   94   30   93  143   26  251  151  201        0   \n",
        "011324   95  155  136   59   23   54   75  150   53  245   86  132        0   \n",
        "011377  122  160  145  150   51  144   77   44   85  209  140  135        0   \n",
        "011525  114  197  139  186   94   49   46  123   81  242  203  123        0   \n",
        "011566  216  126  189  179   88   64  120  195   42  166  119  193        0   \n",
        "011620  103  344  127  162   39   58   44  139   89  250  170  118        0   \n",
        "011690  157  122  109  127   71   56  115  120   34  210  148  208        0   \n",
        "011803  176   47  299  191  136  155  232  105   74  211  237  114        0   \n",
        "012245  113  149  187  128  103   54   61   88   60  248  146  125        0   \n",
        "012632   95  125  110  135   64   95   96  115   55  183  156  114        0   \n",
        "012758  183  188  165  131  130   73  126  300   62  303  151  141        0   \n",
        "012813  138   72  255  170  153  115  220  244   77  308  295  123        0   \n",
        "012883  120  132  203  150   45  131   67   95   81  164  149  110        0   \n",
        "013160   79  129  165  145   59   45   54  109  116  146  162  121        0   \n",
        "013271  135  130  147  135   77  132   99  165    0  316  129   72        0   \n",
        "013511   79  155  130  188  122   65   54  131   64  254   95  118        0   \n",
        "013519  109  109  134  114   52   48  139  103   15  327  115  159        0   \n",
        "013620  155  205  138  238  131  142  158   83   95  142  171  129        0   \n",
        "013761  111  216  133  114   74   97  134  131   43  154  113   61        0   \n",
        "013775  100  213  142  112   77   72   57   57  149  255  193   97        0   \n",
        "013816  143  132  132   90   69  161   51  207   36  308  111  108        0   \n",
        "013842   85  167  116  128   21   86   46   71   90  278  152  116        0   \n",
        "013899  165  166  136  242  132   93  131   66   63  129  123   96        0   \n",
        "013930   91  132  132   97   69   90  105  111   51  170  159   78        0   \n",
        "014064   89  178   84  114   86  117  105   35  117  173  131  105        0   \n",
        "014209   83  118  102  123   71   66   99   66  141  291  168   89        0   \n",
        "014226  140  150  147  161   64  130   63   33  106  204  195  121        0   \n",
        "014431  165  181  160  145  105   95  108  172   39  320  189   71        0   \n",
        "014619   92  100   96  117   89  112   12  114   94  239  178  119        0   \n",
        "014798   64  123  140  173  109   50   59   49   38  146  106   82        0   \n",
        "015112  125  230  139  146   69   68   62  160   83  242  143  128        0   \n",
        "015121  127  107  150  137   33   69   35  135  161  224  120  183        0   \n",
        "015420  123  110  143   70   89   15   92  166   35  220  148  192        0   \n",
        "015478  180   76  217  189  307   84  118  193   29  221  204  225        0   \n",
        "015635  117  178  113  162  184   83   66   50  113  200  124  125        0   \n",
        "015658   90  178  171  133   44   48  198   99   81  216  154   91        0   \n",
        "016121  133  153  121  104   90  106   91  111  163  313  144   90        0   \n",
        "016129   94  134  122   63   32   52   52   99   51  391  118  113        0   \n",
        "016246  105  129  151  135   58   89   41  111  113  314  172  105        0   \n",
        "016334  127  101  178  186  170   58  130  103   45  208  144  147        0   \n",
        "016478   98  130  119  103   59   85   56   55  108  271  161   95        0   \n",
        "016508  140  132  132  126   88  108   60   90   95  214  127  168        0   \n",
        "        ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...  ...      ...   \n",
        "\n",
        "          lon    lat  elev  annual  \n",
        "sta                                 \n",
        "010063 -87.18  34.22   244    1369  \n",
        "010140 -87.42  32.23    55    1486  \n",
        "010184 -88.28  33.23    50    1458  \n",
        "010252 -86.50  31.32   107    2113  \n",
        "010369 -85.83  33.27   308    1660  \n",
        "010390 -86.98  34.80   217    1299  \n",
        "010402 -87.48  31.17    92    2112  \n",
        "010440 -86.68  32.47    61    1333  \n",
        "010505 -87.35  33.45    85    1430  \n",
        "010583 -87.78  30.88    82    2374  \n",
        "010655 -86.88  34.70   183    1306  \n",
        "010764 -87.00  33.40   137    1561  \n",
        "010831 -86.75  33.57   192    1399  \n",
        "010957 -86.17  34.22   329    1442  \n",
        "011080 -87.03  31.12    49    2433  \n",
        "011084 -87.05  31.07    26    2243  \n",
        "011099 -85.73  34.93   201    1478  \n",
        "011225 -86.72  33.45   140    1279  \n",
        "011301 -87.32  32.03    72    1480  \n",
        "011324 -85.68  32.83   207    1263  \n",
        "011377 -87.53  33.90   131    1462  \n",
        "011525 -87.25  32.90   139    1597  \n",
        "011566 -88.25  31.47    61    1697  \n",
        "011620 -86.33  33.28   127    1643  \n",
        "011690 -87.52  31.53    55    1477  \n",
        "011803 -88.22  30.38     3    1977  \n",
        "012245 -87.83  32.52    31    1462  \n",
        "012632 -87.80  33.25    77    1343  \n",
        "012758 -86.97  31.43   104    1953  \n",
        "012813 -87.88  30.55     7    2170  \n",
        "012883 -87.83  33.68   110    1447  \n",
        "013160 -88.13  32.83    38    1330  \n",
        "013271 -86.73  31.65    85    1537  \n",
        "013511 -87.58  32.70    67    1455  \n",
        "013519 -86.63  31.83   134    1424  \n",
        "013620 -87.62  34.23   290    1787  \n",
        "013761 -85.33  31.35   113    1381  \n",
        "013775 -85.60  33.65   259    1524  \n",
        "013816 -86.32  31.95   181    1548  \n",
        "013842 -85.40  33.53   360    1356  \n",
        "013899 -87.93  34.33   259    1542  \n",
        "013930 -87.45  33.25    59    1285  \n",
        "014064 -86.58  34.70   185    1334  \n",
        "014209 -85.75  33.82   217    1417  \n",
        "014226 -87.27  33.90   162    1514  \n",
        "014431 -86.18  31.23    82    1750  \n",
        "014619 -86.55  33.55   194    1362  \n",
        "014798 -88.18  32.58    49    1139  \n",
        "015112 -87.27  32.68    52    1595  \n",
        "015121 -87.22  32.47    61    1481  \n",
        "015420 -87.42  32.10    35    1403  \n",
        "015478 -88.23  30.68    66    2043  \n",
        "015635 -87.30  34.48   197    1515  \n",
        "015658 -87.63  33.00    36    1503  \n",
        "016121 -86.48  33.95   262    1619  \n",
        "016129 -85.38  32.63   226    1321  \n",
        "016246 -86.67  33.77   244    1523  \n",
        "016334 -88.02  32.23    26    1597  \n",
        "016478 -86.70  33.68   185    1340  \n",
        "016508 -86.93  32.65    73    1480  \n",
        "          ...    ...   ...     ...  \n",
        "\n",
        "[5775 rows x 17 columns]\n"
       ]
      }
     ],
     "prompt_number": 66
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 66
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "precip_X = precip_joined[['lon', 'lat', 'elev']]\n",
      "standardizeX(precip_X, ignore_cols=['lon', 'lat'])\n",
      "mkdir_p(os.path.join(BASEDIR, 'datasets/precip_all/'))\n",
      "precip_annual = precip_joined[['annual']]\n",
      "precip_annual = (precip_annual - precip_annual.mean())/precip_annual.std()\n",
      "\n",
      "save_test_train_split(precip_X, precip_annual, \"precip_all\", 0, n_train=5000)\n",
      "\n",
      "\n",
      "for month in ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']:\n",
      "    mkdir_p(os.path.join(BASEDIR, 'datasets/precip_%s/' % month))\n",
      "    precip_month = precip_joined[[month]]\n",
      "    precip_month = (precip_month - precip_month.mean())/precip_month.std()\n",
      "    save_test_train_split(precip_X, precip_month, \"precip_%s\" % month, 0, n_train=5000)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "elev 3537 -59\n",
        "elev 3537 -59 3596\n"
       ]
      }
     ],
     "prompt_number": 67
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "            \n",
      "\n",
      "precip_X.head()\n",
      "#precip_X['lon'].astype('float')\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>lon</th>\n",
        "      <th>lat</th>\n",
        "      <th>elev</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>sta</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>010063</th>\n",
        "      <td>-87.18</td>\n",
        "      <td> 34.22</td>\n",
        "      <td>  8.426029</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>010140</th>\n",
        "      <td>-87.42</td>\n",
        "      <td> 32.23</td>\n",
        "      <td>  3.170189</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>010184</th>\n",
        "      <td>-88.28</td>\n",
        "      <td> 33.23</td>\n",
        "      <td>  3.031146</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>010252</th>\n",
        "      <td>-86.50</td>\n",
        "      <td> 31.32</td>\n",
        "      <td>  4.616240</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>010369</th>\n",
        "      <td>-85.83</td>\n",
        "      <td> 33.27</td>\n",
        "      <td> 10.205784</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "<p>5 rows \u00d7 3 columns</p>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 68,
       "text": [
        "          lon    lat       elev\n",
        "sta                            \n",
        "010063 -87.18  34.22   8.426029\n",
        "010140 -87.42  32.23   3.170189\n",
        "010184 -88.28  33.23   3.031146\n",
        "010252 -86.50  31.32   4.616240\n",
        "010369 -85.83  33.27  10.205784\n",
        "\n",
        "[5 rows x 3 columns]"
       ]
      }
     ],
     "prompt_number": 68
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tco = pd.read_csv(os.path.join(BASEDIR, 'raw_data/tco.csv'), names=['lon', 'lat', 'elev', 'tco'])\n",
      "print tco"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "        lon   lat  elev  tco\n",
        "0  -179.375 -89.5     0    0\n",
        "1  -178.125 -89.5     0    0\n",
        "2  -176.875 -89.5     0    0\n",
        "3  -175.625 -89.5     0    0\n",
        "4  -174.375 -89.5     0    0\n",
        "5  -173.125 -89.5     0    0\n",
        "6  -171.875 -89.5     0    0\n",
        "7  -170.625 -89.5     0    0\n",
        "8  -169.375 -89.5     0    0\n",
        "9  -168.125 -89.5     0    0\n",
        "10 -166.875 -89.5     0    0\n",
        "11 -165.625 -89.5     0    0\n",
        "12 -164.375 -89.5     0    0\n",
        "13 -163.125 -89.5     0    0\n",
        "14 -161.875 -89.5     0    0\n",
        "15 -160.625 -89.5     0    0\n",
        "16 -159.375 -89.5     0    0\n",
        "17 -158.125 -89.5     0    0\n",
        "18 -156.875 -89.5     0    0\n",
        "19 -155.625 -89.5     0    0\n",
        "20 -154.375 -89.5     0    0\n",
        "21 -153.125 -89.5     0    0\n",
        "22 -151.875 -89.5     0    0\n",
        "23 -150.625 -89.5     0    0\n",
        "24 -149.375 -89.5     0    0\n",
        "25 -148.125 -89.5     0    0\n",
        "26 -146.875 -89.5     0    0\n",
        "27 -145.625 -89.5     0    0\n",
        "28 -144.375 -89.5     0    0\n",
        "29 -143.125 -89.5     0    0\n",
        "30 -141.875 -89.5     0    0\n",
        "31 -140.625 -89.5     0    0\n",
        "32 -139.375 -89.5     0    0\n",
        "33 -138.125 -89.5     0    0\n",
        "34 -136.875 -89.5     0    0\n",
        "35 -135.625 -89.5     0    0\n",
        "36 -134.375 -89.5     0    0\n",
        "37 -133.125 -89.5     0    0\n",
        "38 -131.875 -89.5     0    0\n",
        "39 -130.625 -89.5     0    0\n",
        "40 -129.375 -89.5     0    0\n",
        "41 -128.125 -89.5     0    0\n",
        "42 -126.875 -89.5     0    0\n",
        "43 -125.625 -89.5     0    0\n",
        "44 -124.375 -89.5     0    0\n",
        "45 -123.125 -89.5     0    0\n",
        "46 -121.875 -89.5     0    0\n",
        "47 -120.625 -89.5     0    0\n",
        "48 -119.375 -89.5     0    0\n",
        "49 -118.125 -89.5     0    0\n",
        "50 -116.875 -89.5     0    0\n",
        "51 -115.625 -89.5     0    0\n",
        "52 -114.375 -89.5     0    0\n",
        "53 -113.125 -89.5     0    0\n",
        "54 -111.875 -89.5     0    0\n",
        "55 -110.625 -89.5     0    0\n",
        "56 -109.375 -89.5     0    0\n",
        "57 -108.125 -89.5     0    0\n",
        "58 -106.875 -89.5     0    0\n",
        "59 -105.625 -89.5     0    0\n",
        "        ...   ...   ...  ...\n",
        "\n",
        "[51840 rows x 4 columns]\n"
       ]
      }
     ],
     "prompt_number": 69
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tco_X = tco[['lon', 'lat']]\n",
      "tco_y = tco[['tco']]\n",
      "tco_y = (tco_y - tco_y.mean()) / tco_y.std()\n",
      "\n",
      "save_test_train_split(tco_X, tco_y, \"tco\", n_train=15000)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 70
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tco_y.hist()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "TclError",
       "evalue": "no display name and no $DISPLAY environment variable",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mTclError\u001b[0m                                  Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-71-f907b5411566>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtco_y\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[1;32m/home/dmoore/.virtualenvs/sparsegp/local/lib/python2.7/site-packages/pandas/tools/plotting.pyc\u001b[0m in \u001b[0;36mhist_frame\u001b[1;34m(data, column, by, grid, xlabelsize, xrot, ylabelsize, yrot, ax, sharex, sharey, figsize, layout, **kwds)\u001b[0m\n\u001b[0;32m   2067\u001b[0m                 \u001b[0mcols\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2068\u001b[0m     fig, axes = _subplots(nrows=rows, ncols=cols, ax=ax, squeeze=False,\n\u001b[1;32m-> 2069\u001b[1;33m                           sharex=sharex, sharey=sharey, figsize=figsize)\n\u001b[0m\u001b[0;32m   2070\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2071\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcol\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_try_sort\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/dmoore/.virtualenvs/sparsegp/local/lib/python2.7/site-packages/pandas/tools/plotting.pyc\u001b[0m in \u001b[0;36m_subplots\u001b[1;34m(nrows, ncols, sharex, sharey, squeeze, subplot_kw, ax, secondary_y, data, **fig_kw)\u001b[0m\n\u001b[0;32m   2431\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2432\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0max\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2433\u001b[1;33m         \u001b[0mfig\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mfig_kw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2434\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2435\u001b[0m         \u001b[0mfig\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_figure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/dmoore/.virtualenvs/sparsegp/local/lib/python2.7/site-packages/matplotlib/pyplot.pyc\u001b[0m in \u001b[0;36mfigure\u001b[1;34m(num, figsize, dpi, facecolor, edgecolor, frameon, FigureClass, **kwargs)\u001b[0m\n\u001b[0;32m    421\u001b[0m                                         \u001b[0mframeon\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mframeon\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    422\u001b[0m                                         \u001b[0mFigureClass\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFigureClass\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 423\u001b[1;33m                                         **kwargs)\n\u001b[0m\u001b[0;32m    424\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    425\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfigLabel\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/dmoore/.virtualenvs/sparsegp/local/lib/python2.7/site-packages/matplotlib/backends/backend_tkagg.pyc\u001b[0m in \u001b[0;36mnew_figure_manager\u001b[1;34m(num, *args, **kwargs)\u001b[0m\n\u001b[0;32m     77\u001b[0m     \u001b[0mFigureClass\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'FigureClass'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mFigure\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[0mfigure\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFigureClass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 79\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mnew_figure_manager_given_figure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfigure\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     80\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     81\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/dmoore/.virtualenvs/sparsegp/local/lib/python2.7/site-packages/matplotlib/backends/backend_tkagg.pyc\u001b[0m in \u001b[0;36mnew_figure_manager_given_figure\u001b[1;34m(num, figure)\u001b[0m\n\u001b[0;32m     85\u001b[0m     \"\"\"\n\u001b[0;32m     86\u001b[0m     \u001b[0m_focus\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mwindowing\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFocusManager\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 87\u001b[1;33m     \u001b[0mwindow\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTk\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m     \u001b[0mwindow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwithdraw\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/lib/python2.7/lib-tk/Tkinter.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, screenName, baseName, className, useTk, sync, use)\u001b[0m\n\u001b[0;32m   1686\u001b[0m                 \u001b[0mbaseName\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbaseName\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mext\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1687\u001b[0m         \u001b[0minteractive\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1688\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_tkinter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscreenName\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbaseName\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mclassName\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minteractive\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwantobjects\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0museTk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msync\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1689\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0museTk\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1690\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_loadtk\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mTclError\u001b[0m: no display name and no $DISPLAY environment variable"
       ]
      }
     ],
     "prompt_number": 71
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "h1 = pd.read_csv(os.path.join(BASEDIR, 'raw_data/cal_housing_data.csv'))\n",
      "h1_norm = (h1 - h1.mean()) / h1.std()\n",
      "print h1_norm"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "         lon       lat       age    income     units  population  households  \\\n",
        "0  -1.327803  1.052523  0.982119 -0.804800 -0.970683   -0.974405   -0.977009   \n",
        "1  -1.322812  1.043159 -0.607004  2.045841  1.348617    0.861418    1.669921   \n",
        "2  -1.332794  1.038478  1.856137 -0.535733 -0.825875   -0.820757   -0.843616   \n",
        "3  -1.337785  1.038478  1.856137 -0.624199 -0.719049   -0.766010   -0.733764   \n",
        "4  -1.337785  1.038478  1.856137 -0.462393 -0.612224   -0.759828   -0.629142   \n",
        "5  -1.337785  1.038478  1.856137 -0.786923 -0.771275   -0.894049   -0.801768   \n",
        "6  -1.337785  1.033796  1.856137 -0.046187 -0.116079   -0.292704    0.037822   \n",
        "7  -1.337785  1.033796  1.856137  0.214629  0.353953   -0.237073    0.385689   \n",
        "8  -1.342777  1.033796  1.061575 -0.037020  0.301727   -0.193805    0.249681   \n",
        "9  -1.337785  1.033796  1.856137  0.418606  0.401431    0.110841    0.560930   \n",
        "10 -1.342777  1.038478  1.856137 -0.198827 -0.246643   -0.455182   -0.255119   \n",
        "11 -1.342777  1.038478  1.856137  0.397521  0.508257    0.069339    0.613241   \n",
        "12 -1.342777  1.038478  1.856137 -0.066356 -0.151687   -0.289172   -0.082493   \n",
        "13 -1.342777  1.033796  1.856137 -0.889141 -0.823501   -0.954095   -0.851463   \n",
        "14 -1.342777  1.038478  1.856137  0.003317  0.209145   -0.188507    0.315069   \n",
        "15 -1.342777  1.038478  1.697224 -0.694789 -0.605102   -0.643268   -0.616064   \n",
        "16 -1.347768  1.038478  1.856137 -0.307003 -0.453173   -0.558497   -0.440823   \n",
        "17 -1.347768  1.038478  1.856137 -0.645285 -0.581363   -0.686537   -0.514058   \n",
        "18 -1.342777  1.033796  1.697224 -0.181867 -0.196792   -0.384540   -0.210655   \n",
        "19 -1.347768  1.033796  1.856137 -0.519231 -0.569494   -0.649449   -0.587293   \n",
        "20 -1.347768  1.038478  0.902663 -0.863930 -0.840118   -0.897581   -0.872387   \n",
        "21 -1.347768  1.038478  1.061575 -0.456892 -0.405695   -0.438405   -0.349279   \n",
        "22 -1.347768  1.033796  1.856137 -0.091567  0.007364   -0.362464   -0.056338   \n",
        "23 -1.347768  1.033796  1.856137 -0.434432 -0.476912   -0.505515   -0.456516   \n",
        "24 -1.347768  1.033796  1.856137 -0.188742 -0.239522   -0.370411   -0.202808   \n",
        "25 -1.352759  1.038478  0.982119 -0.962939 -0.984926   -0.978820   -0.995318   \n",
        "26 -1.352759  1.038478  1.617768 -0.690206 -0.697684   -0.722741   -0.681453   \n",
        "27 -1.352759  1.038478  1.856137 -0.338173 -0.277504   -0.285640   -0.268197   \n",
        "28 -1.352759  1.033796  1.697224 -0.253832 -0.108957   -0.260032   -0.069416   \n",
        "29 -1.352759  1.033796  1.856137 -0.874014 -0.897092   -0.909944   -0.901158   \n",
        "30 -1.352759  1.033796  1.617768 -0.329922 -0.215783   -0.496685   -0.317892   \n",
        "31 -1.352759  1.033796  1.856137 -0.221287 -0.135070   -0.227360   -0.153113   \n",
        "32 -1.347768  1.033796  1.538312 -0.327172 -0.305991   -0.352751   -0.430361   \n",
        "33 -1.347768  1.029114  1.617768 -0.449558 -0.408069   -0.592935   -0.446054   \n",
        "34 -1.347768  1.029114  1.776680  0.013402  0.085702   -0.147887    0.095364   \n",
        "35 -1.347768  1.029114  1.617768 -0.651244 -0.607476   -0.755413   -0.616064   \n",
        "36 -1.347768  1.029114  1.538312 -0.384010 -0.251391   -0.387189   -0.328354   \n",
        "37 -1.352759  1.029114  1.856137 -0.515564 -0.351095   -0.463130   -0.252504   \n",
        "38 -1.342777  1.029114  1.856137 -0.534358 -0.493529   -0.650332   -0.498365   \n",
        "39 -1.342777  1.029114  1.856137 -0.093400  0.420422   -0.042807    0.513850   \n",
        "40 -1.342777  1.029114  1.856137 -0.444974 -0.282252   -0.423393   -0.273428   \n",
        "41 -1.342777  1.029114  1.776680 -0.779131 -0.538633   -0.802214   -0.655297   \n",
        "42 -1.342777  1.033796  1.617768 -0.881348 -0.797388   -0.850781   -0.812230   \n",
        "43 -1.342777  1.033796  1.856137 -0.772713 -0.797388   -0.846365   -0.788690   \n",
        "44 -1.342777  1.029114  1.856137 -0.546734 -0.538633   -0.675940   -0.542829   \n",
        "45 -1.342777  1.029114  1.856137 -0.449100 -0.279878   -0.624724   -0.307430   \n",
        "46 -1.342777  1.029114  1.697224 -0.692497 -0.512520   -0.714794   -0.511442   \n",
        "47 -1.347768  1.024432  1.141031 -0.746586 -0.536259   -0.766010   -0.644835   \n",
        "48 -1.342777  1.024432  0.902663 -0.922144 -0.814005   -0.885219   -0.888081   \n",
        "49 -1.347768  1.024432  0.902663 -0.774547 -0.386703   -0.640619   -0.385896   \n",
        "50 -1.347768  1.024432 -0.607004 -0.797466 -0.201539   -0.609713   -0.160960   \n",
        "51 -1.347768  1.024432  1.141031 -0.351924 -0.194418   -0.321845   -0.242042   \n",
        "52 -1.347768  1.024432  0.982119  0.268259  0.748020    0.471118    0.576623   \n",
        "53 -1.347768  1.024432  1.856137 -0.461018 -0.194418   -0.232658   -0.260350   \n",
        "54 -1.352759  1.024432  1.856137 -0.671871 -0.719049   -0.639736   -0.697146   \n",
        "55 -1.352759  1.024432  1.856137 -0.775005 -0.700058   -0.750115   -0.731148   \n",
        "56 -1.352759  1.024432  1.856137 -0.640701 -0.593233   -0.709495   -0.629142   \n",
        "57 -1.352759  1.024432  1.856137 -0.525649 -0.481659   -0.615894   -0.668375   \n",
        "58 -1.352759  1.024432  1.856137 -0.572403 -0.467416   -0.310365   -0.511442   \n",
        "59 -1.357750  1.024432 -2.116671 -1.135747 -1.174838   -1.175736   -1.157482   \n",
        "         ...       ...       ...       ...       ...         ...         ...   \n",
        "\n",
        "       rooms     value  \n",
        "0   2.344709  2.129580  \n",
        "1   2.332181  1.314124  \n",
        "2   1.782656  1.258663  \n",
        "3   0.932945  1.165072  \n",
        "4  -0.012881  1.172871  \n",
        "5   0.087445  0.544598  \n",
        "6  -0.111364  0.800240  \n",
        "7  -0.395127  0.299354  \n",
        "8  -0.942336  0.171967  \n",
        "9  -0.094467  0.470071  \n",
        "10 -0.351386  0.646855  \n",
        "11 -0.315909  0.302821  \n",
        "12 -0.418814  0.057577  \n",
        "13 -0.630096 -0.134804  \n",
        "14 -1.028502 -0.412978  \n",
        "15 -0.918860 -0.579362  \n",
        "16 -0.576723 -0.471039  \n",
        "17 -0.921387 -0.445041  \n",
        "18 -0.989341 -0.417311  \n",
        "19 -0.667100 -0.380914  \n",
        "20 -1.322688 -0.514368  \n",
        "21 -1.135460 -0.407778  \n",
        "22 -1.129407 -0.805540  \n",
        "23 -0.889595 -0.928595  \n",
        "24 -0.668837 -0.643489  \n",
        "25 -0.772110 -0.861002  \n",
        "26 -0.742686 -0.979724  \n",
        "27 -1.085718 -0.878333  \n",
        "28 -1.172884 -0.848869  \n",
        "29 -1.149145 -0.648689  \n",
        "30 -1.022870 -0.732747  \n",
        "31 -1.004921 -0.794275  \n",
        "32 -1.091561 -0.835871  \n",
        "33 -1.313634 -0.883533  \n",
        "34 -0.600252 -0.841937  \n",
        "35 -1.255155 -0.950260  \n",
        "36 -1.459859 -0.886999  \n",
        "37 -1.295054 -0.892199  \n",
        "38 -0.205636 -0.133938  \n",
        "39 -0.674206 -0.267392  \n",
        "40 -0.933178 -0.445908  \n",
        "41 -1.360902 -0.492703  \n",
        "42 -1.497862 -0.763078  \n",
        "43  0.049283 -0.156469  \n",
        "44 -0.451711 -0.194599  \n",
        "45 -0.628412 -0.212797  \n",
        "46 -0.970971 -0.557697  \n",
        "47 -1.124248 -0.601026  \n",
        "48 -1.537024 -0.167734  \n",
        "49 -1.103088 -0.817672  \n",
        "50 -1.552183 -0.302922  \n",
        "51 -1.245470 -0.979724  \n",
        "52 -1.452700 -0.947660  \n",
        "53 -1.380746 -0.889599  \n",
        "54 -1.190044 -1.034318  \n",
        "55 -1.294527 -1.072448  \n",
        "56 -1.244838 -1.034318  \n",
        "57 -1.607241 -1.053383  \n",
        "58 -1.396747 -1.096713  \n",
        "59 -0.688576 -1.272629  \n",
        "         ...       ...  \n",
        "\n",
        "[20640 rows x 9 columns]\n"
       ]
      }
     ],
     "prompt_number": 72
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "housing_age_X = h1[['income', 'value']]\n",
      "standardizeX(housing_age_X, ignore_cols=['lon', 'lat'])\n",
      "housing_age_y = h1_norm[['age']]\n",
      "save_test_train_split(housing_age_X, housing_age_y, \"housing_age\", n_train=18000)\n",
      "\n",
      "\n",
      "housing_inc_X = h1[['age', 'value']]\n",
      "standardizeX(housing_inc_X, ignore_cols=['lon', 'lat'])\n",
      "housing_inc_y = h1_norm[['income']]\n",
      "save_test_train_split(housing_inc_X, housing_inc_y, \"housing_inc\", n_train=18000)\n",
      "\n",
      "\n",
      "housing_val_X = h1[['age', 'income']]\n",
      "standardizeX(housing_val_X, ignore_cols=['lon', 'lat'])\n",
      "housing_val_y = h1_norm[['value']]\n",
      "save_test_train_split(housing_val_X, housing_val_y, \"housing_val\", n_train=18000)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "income 39320.0 2.0\n",
        "income 39320.0 2.0 39318.0\n",
        "value 500001.0 14999.0\n",
        "value"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 500001.0 14999.0 485002.0\n",
        "age"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 52.0 1.0\n",
        "age 52.0 1.0 51.0\n",
        "value 500001.0 14999.0\n",
        "value"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 500001.0 14999.0 485002.0\n",
        "age"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 52.0 1.0\n",
        "age 52.0 1.0 51.0\n",
        "income 39320.0 2.0\n",
        "income"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 39320.0 2.0 39318.0\n"
       ]
      }
     ],
     "prompt_number": 86
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bigsta=None\n",
      "\n",
      "import os\n",
      "\n",
      "for stafile in os.listdir(os.path.join(BASEDIR, 'raw_data/california_snow/')):\n",
      "    if not stafile.endswith('.csv'): continue\n",
      "    \n",
      "    sta = pd.read_csv(os.path.join(BASEDIR, 'raw_data/california_snow/' + stafile))\n",
      "    sta['daynum'] = sta.index\n",
      "    sta = sta.dropna()\n",
      "    sta['snow'].astype('float')\n",
      "    if bigsta is None:\n",
      "        bigsta = sta\n",
      "    else:\n",
      "        bigsta = pd.concat([bigsta, sta])\n",
      "#print bigsta"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 87
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "bigsta['snow'] = (bigsta['snow'] - bigsta['snow'].mean()) / bigsta['snow'].std()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 91
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "bigsta_sorted = bigsta.sort(columns=['daynum', 'lon', 'lat', 'elev'])\n",
      "\n",
      "snow_X = bigsta_sorted[['daynum', 'lon', 'lat', 'elev']]\n",
      "standardizeX(snow_X, ignore_cols=['daynum', 'lon', 'lat'])\n",
      "\n",
      "\n",
      "snow_y = bigsta_sorted[['snow']]\n",
      "\n",
      "save_test_train_split(snow_X, snow_y, \"snow\", n_train=15000)\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "elev 11400.0 5100.0\n",
        "elev 11400.0 5100.0 6300.0\n"
       ]
      }
     ],
     "prompt_number": 92
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "print bigsta_sorted.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "         date      lon     lat   elev      snow  daynum\n",
        "0  2011-11-01  118.177 -36.483  10150 -0.804404       0\n",
        "0  2011-11-01  118.268 -36.200   8300 -0.770013       0\n",
        "0  2011-11-01  118.288 -36.367   8900 -1.025080       0\n",
        "0  2011-11-01  118.293 -36.127   7650 -0.820167       0\n",
        "0  2011-11-01  118.345 -36.563  10700 -0.838795       0\n",
        "\n",
        "[5 rows x 6 columns]\n"
       ]
      }
     ],
     "prompt_number": 77
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import scipy.io    \n",
      "sdir = os.path.join(BASEDIR, 'raw_data')\n",
      "sarcos_train = scipy.io.loadmat(os.path.join(sdir, 'sarcos_inv.mat'))['sarcos_inv'].byteswap().newbyteorder()\n",
      "sarcos_test = scipy.io.loadmat(os.path.join(sdir, 'sarcos_inv_test.mat'))['sarcos_inv_test'].byteswap().newbyteorder()\n",
      "\n",
      "print sarcos_train"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[ 0.019478 -0.134218  0.027439 ..., -0.424812 -0.907553  8.090739]\n",
        " [ 0.017279 -0.137077  0.026999 ..., -0.228739 -1.235817  7.762475]\n",
        " [ 0.016336 -0.140878  0.02725  ...,  0.244491 -1.70088   7.289678]\n",
        " ..., \n",
        " [-0.551577 -0.445141  0.07741  ..., -1.274436 -2.375086  1.892345]\n",
        " [-0.541838 -0.436124  0.0743   ..., -1.577874 -2.317157  2.490944]\n",
        " [-0.530622 -0.426197  0.069838 ..., -0.788937 -2.259228  2.857827]]\n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sarcos_train_X = sarcos_train[:, :21]\n",
      "sarcos_train_y = sarcos_train[:, 21]\n",
      "sarcos_test_X = sarcos_test[:, :21]\n",
      "sarcos_test_y = sarcos_test[:, 21]\n",
      "\n",
      "sarcos_train_X = np.array(sarcos_train_X, copy=True, dtype=float, order=\"C\")[:16384,:]\n",
      "sarcos_train_y = np.array(sarcos_train_y, copy=True, dtype=float, order=\"C\")[:16384]\n",
      "sarcos_test_X = np.array(sarcos_test_X, copy=True, dtype=float, order=\"C\")\n",
      "sarcos_test_y = np.array(sarcos_test_y, copy=True, dtype=float, order=\"C\")\n",
      "\n",
      "train_X_mean = np.reshape(np.mean(sarcos_train_X, axis=0), (1, -1))\n",
      "train_X_std = np.reshape(np.std(sarcos_train_X, axis=0), (1, -1))\n",
      "train_y_mean = np.mean(sarcos_train_y)\n",
      "train_y_std = np.std(sarcos_train_y)\n",
      "    \n",
      "sarcos_train_X = (sarcos_train_X - train_X_mean)/(train_X_std / 50.0)\n",
      "sarcos_test_X = (sarcos_test_X - train_X_mean)/(train_X_std / 50.0)\n",
      "sarcos_train_y = (sarcos_train_y - train_y_mean)/train_y_std\n",
      "sarcos_test_y = (sarcos_test_y - train_y_mean)/train_y_std\n",
      "\n",
      "\n",
      "\n",
      "out_dir = os.path.join(BASEDIR, 'datasets/sarcos/')\n",
      "mkdir_p(out_dir)\n",
      "trainXframe = pd.DataFrame(sarcos_train_X)\n",
      "trainXframe.to_csv(os.path.join(out_dir, 'X_train.txt'), index=False)\n",
      "trainyframe = pd.DataFrame(sarcos_train_y)\n",
      "trainyframe.to_csv(os.path.join(out_dir, 'y_train.txt'), index=False)\n",
      "\n",
      "testXframe = pd.DataFrame(sarcos_test_X)\n",
      "testXframe.to_csv(os.path.join(out_dir, 'X_test.txt'), index=False)\n",
      "testyframe = pd.DataFrame(sarcos_test_y)\n",
      "testyframe.to_csv(os.path.join(out_dir, 'y_test.txt'), index=False)\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 79
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# wiggle preprocessing\n",
      "\n",
      "\n",
      "ycols = ['amp_%02d' % freq for freq in range(60)]\n",
      "X = pd.read_csv(os.path.join(BASEDIR, 'raw_data/wiggle_X_good.txt'), names=['lon', 'lat', 'depth'], sep=r'\\s*')\n",
      "Y = pd.read_csv(os.path.join(BASEDIR, 'raw_data/wiggle_Y_good.txt'), names=ycols, sep=r'\\s*')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 80
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "n = Y.shape[0]\n",
      "for i in range(n):\n",
      "    Y.ix[i,:] = Y.ix[i,:] / np.linalg.norm(Y.ix[i,:], 2)\n",
      "    \n",
      "Y = (Y - Y.mean())/Y.std()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 81
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X['lon'] = X['lon'] % 360 - 180"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 82
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "XY = X.merge(Y, left_index=True, right_index=True)\n",
      "XY=XY.dropna()\n",
      "XY = XY.sort(columns=['lon', 'lat', 'depth'])\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 83
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "XX = XY[['lon', 'lat', 'depth']]\n",
      "standardizeX(XX, ignore_cols=['lon', 'lat'])\n",
      "YY = XY[ycols]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "depth 700.0 0.0\n",
        "depth 700.0 0.0 700.0\n"
       ]
      }
     ],
     "prompt_number": 84
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "save_test_train_split(XX, YY[['amp_05']], \"wiggle_5\", n_train=10000)\n",
      "save_test_train_split(XX, YY[['amp_15']], \"wiggle_15\", n_train=10000)\n",
      "save_test_train_split(XX, YY[['amp_20']], \"wiggle_20\", n_train=10000)\n",
      "save_test_train_split(XX, YY[['amp_45']], \"wiggle_45\", n_train=10000)\n",
      "\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 85
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X = pd.read_csv(os.path.join(BASEDIR, 'raw_data/X_FITZ.txt'), names=['lon', 'lat', 'depth', 'dist', 'azi', 'mb'], sep=r'\\s*')\n",
      "y = pd.read_csv(os.path.join(BASEDIR, 'raw_data/y_FITZ.txt'), names=['amp_transfer'], sep=r'\\s*')\n",
      "\n",
      "y = (y - y.mean())/y.std()\n",
      "X = X[['lon', 'lat', 'depth']]\n",
      "save_test_train_split(X, y, \"seismic_fitz\", n_train=3500)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 111
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X = pd.read_csv(os.path.join(BASEDIR, 'raw_data/X_AS12.txt'), names=['lon', 'lat', 'depth', 'dist', 'azi', 'mb'], sep=r'\\s*')\n",
      "y = pd.read_csv(os.path.join(BASEDIR, 'raw_data/y_AS12.txt'), names=['amp_transfer'], sep=r'\\s*')\n",
      "\n",
      "\n",
      "y = (y - y.mean())/y.std()\n",
      "X = X[['lon', 'lat', 'depth']]\n",
      "save_test_train_split(X, y, \"seismic_as12\", n_train=20000)X = pd.read_csv(os.path.join(BASEDIR, 'raw_data/X_AS12.txt'), names=['lon', 'lat', 'depth', 'dist', 'azi', 'mb'], sep=r'\\s*')\n",
      "y = pd.read_csv(os.path.join(BASEDIR, 'raw_data/y_AS12.txt'), names=['amp_transfer'], sep=r'\\s*')\n",
      "\n",
      "\n",
      "y = (y - y.mean())/y.std()\n",
      "X = X[['lon', 'lat', 'depth']]\n",
      "save_test_train_split(X, y, \"seismic_as12\", n_train=20000)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(25877, 3) (25877, 1) 25877 20000\n",
        "(20000, 3)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "        amp_transfer\n",
        "6756      -0.764122\n",
        "8870       0.032346\n",
        "25545      1.684493\n",
        "25141     -0.565302\n",
        "25134      0.214456\n",
        "9608      -0.088577\n",
        "6038      -0.322334\n",
        "9135      -0.839262\n",
        "21437      0.848224\n",
        "21726     -0.737662\n",
        "15384     -0.122362\n",
        "3101       0.448063\n",
        "2606       0.484243\n",
        "5244      -0.013923\n",
        "3833       0.105997\n",
        "18001      1.089783\n",
        "9285       0.879798\n",
        "24452      0.364199\n",
        "1844      -0.393160\n",
        "8330      -2.051111\n",
        "18811     -0.340341\n",
        "1603      -0.721587\n",
        "17636      2.164483\n",
        "25009      0.270061\n",
        "4682       0.533632\n",
        "10944      0.108581\n",
        "17915     -1.086216\n",
        "14535      0.497035\n",
        "9619      -0.482938\n",
        "4436      -0.445580\n",
        "6441       1.133402\n",
        "6895      -0.882243\n",
        "25168      2.202363\n",
        "8953      -1.045123\n",
        "14708      2.913464\n",
        "11734      1.074340\n",
        "21465     -0.164889\n",
        "8771       0.030962\n",
        "13207     -1.768228\n",
        "18522      0.225052\n",
        "6863      -1.196633\n",
        "21416     -0.146108\n",
        "13526     -1.330761\n",
        "20080     -0.171786\n",
        "7647       0.940515\n",
        "2508      -0.707555\n",
        "4889      -1.199541\n",
        "3075       1.107323\n",
        "9995      -1.203096\n",
        "791        1.180000\n",
        "1026      -0.901203\n",
        "15693      0.871947\n",
        "18998     -0.342705\n",
        "2459      -0.719490\n",
        "23189     -1.201105\n",
        "21847      0.827051\n",
        "18237      0.494443\n",
        "21826      0.542124\n",
        "8509       0.350043\n",
        "2201       1.172106\n",
        "                ...\n",
        "\n",
        "[20000 rows x 1 columns]\n",
        "(5877, 3)\n",
        "(5877, 1)\n"
       ]
      }
     ],
     "prompt_number": 108
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X = pd.read_csv(os.path.join(BASEDIR, 'raw_data/tt_ASAR_X.txt'), names=['lon', 'lat', 'depth'], sep=r'\\s*')\n",
      "y = pd.read_csv(os.path.join(BASEDIR, 'raw_data/tt_ASAR_y.txt'), names=['tt_residual'], sep=r'\\s*')\n",
      "\n",
      "X = X[['lon', 'lat', 'depth']].iloc[:20000,:]\n",
      "y = y.iloc[:20000]\n",
      "save_test_train_split(X, y, \"seismic_tt_ASAR\", n_train=16000)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X = pd.read_csv(os.path.join(BASEDIR, 'raw_data/tt_FITZ_X.txt'), names=['lon', 'lat', 'depth'], sep=r'\\s*')\n",
      "y = pd.read_csv(os.path.join(BASEDIR, 'raw_data/tt_FITZ_y.txt'), names=['tt_residual'], sep=r'\\s*')\n",
      "\n",
      "X = X[['lon', 'lat', 'depth']].iloc[:20000,:]\n",
      "y = y.iloc[:20000]\n",
      "save_test_train_split(X, y, \"seismic_tt_FITZ\", n_train=16000)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "X = pd.read_csv(os.path.join(BASEDIR, 'raw_data/tt_ASAR_X.txt'), names=['lon', 'lat', 'depth'], sep=r'\\s*')\n",
      "y = pd.read_csv(os.path.join(BASEDIR, 'raw_data/tt_ASAR_y.txt'), names=['tt_residual'], sep=r'\\s*')\n",
      "\n",
      "save_test_train_split(X, y, \"seismic_tt_ASAR_50\", n_train=45000)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}