exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:26:27.740710Z",
     "start_time": "2020-04-05T13:26:25.979351Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from data_loaders import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Instructions on how to load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:23:58.403314Z",
     "start_time": "2020-04-05T13:23:49.598822Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(399990, 200)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How to read one file: careful, this might take a long time\n",
    "# data is by defult QUANTILE scaled so between [0, 1]\n",
    "a = read_scale(SAMPLE_PATH, no_obs=400000, slice_size=10)\n",
    "a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:48:05.055200Z",
     "start_time": "2020-04-05T13:48:04.283588Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         3172.0\n",
       "1         3172.8\n",
       "2         3170.0\n",
       "3         3174.4\n",
       "4         3174.4\n",
       "5         3172.8\n",
       "6         3172.8\n",
       "7         3172.2\n",
       "8         3173.2\n",
       "9         3172.6\n",
       "10        3174.8\n",
       "11        3175.0\n",
       "12        3175.6\n",
       "13        3176.0\n",
       "14        3174.8\n",
       "15        3175.2\n",
       "16        3174.8\n",
       "17        3175.0\n",
       "18        3174.4\n",
       "19        3174.0\n",
       "20        3174.0\n",
       "21        3174.2\n",
       "22        3174.8\n",
       "23        3174.0\n",
       "24        3173.0\n",
       "25        3174.0\n",
       "26        3173.6\n",
       "27        3174.0\n",
       "28        3174.0\n",
       "29        3173.0\n",
       "           ...  \n",
       "399970    2984.6\n",
       "399971    2984.6\n",
       "399972    2984.6\n",
       "399973    2984.6\n",
       "399974    2984.4\n",
       "399975    2984.4\n",
       "399976    2985.0\n",
       "399977    2985.0\n",
       "399978    2985.0\n",
       "399979    2985.0\n",
       "399980    2985.0\n",
       "399981    2985.0\n",
       "399982    2985.2\n",
       "399983    2985.6\n",
       "399984    2985.6\n",
       "399985    2985.4\n",
       "399986    2985.4\n",
       "399987    2985.4\n",
       "399988    2985.4\n",
       "399989    2985.2\n",
       "399990    2985.4\n",
       "399991    2985.4\n",
       "399992    2985.4\n",
       "399993    2985.2\n",
       "399994    2985.2\n",
       "399995    2985.2\n",
       "399996    2985.4\n",
       "399997    2985.2\n",
       "399998    2984.2\n",
       "399999    2984.0\n",
       "Name: S1, Length: 400000, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How to read best bid and ask prices quickly\n",
    "buy, sell = read_bid_ask(SAMPLE_PATH, no_obs=400000)\n",
    "sell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:26:48.759807Z",
     "start_time": "2020-04-05T13:26:46.309236Z"
    }
   },
   "outputs": [],
   "source": [
    "# How to iterate over all datafiles available\n",
    "#  the generator loops over all csv files in data/order_books\n",
    "for a in generate_data(read_scale, no_obs=2000, slice_size=10):\n",
    "    print(a)\n",
    "    # Do anything you want with a, like train the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sample model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:33:59.586985Z",
     "start_time": "2020-04-05T13:33:50.590582Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Let's fit a basic NN that tries to predict the best bid price\n",
    "#  10 minutes in the future. Train it on first half of the SAMPLE_PATH file.\n",
    "#  Report test performance on a different file.\n",
    "\n",
    "offset = 120 # 10 minutes roughly\n",
    "data_size = 200000 # size of train/test set\n",
    "slice_size = 10\n",
    "\n",
    "X = read_scale(SAMPLE_PATH, no_obs=2*data_size, slice_size=slice_size)\n",
    "y = read_bid_ask(SAMPLE_PATH, no_obs=2*data_size+offset-slice_size)[1][offset:]\n",
    "\n",
    "X_tr, X_ts = X[:data_size], X[data_size:]\n",
    "y_tr, y_ts = y[:data_size], y[data_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:37:16.752829Z",
     "start_time": "2020-04-05T13:36:54.550139Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/fabiopruneri/Documents/anaconda3/lib/python3.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:566: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "             beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "             hidden_layer_sizes=(100, 100, 100), learning_rate='constant',\n",
       "             learning_rate_init=0.001, max_iter=10, momentum=0.9,\n",
       "             n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
       "             random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
       "             validation_fraction=0.1, verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training model. Takes 25s on my laptop. \n",
    "# Ignore the ConvergenceWarning (need mroe data)\n",
    "\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "nn = MLPRegressor(hidden_layer_sizes=(100, 100, 100,), max_iter=10)\n",
    "nn.fit(X_tr, y_tr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:37:17.486158Z",
     "start_time": "2020-04-05T13:37:16.754809Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8800552784956912"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.score(X_tr, y_tr) # This is the R^2\n",
    "# As expected, performs very well on training data (massive overfitting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-05T13:37:37.548846Z",
     "start_time": "2020-04-05T13:37:36.827640Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.8562876471564211"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn.score(X_ts, y_ts)\n",
    "# And terribly on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       S5      S4      S3      S2      S1      B1      B2      B3      B4  \\\n",
      "0  3174.8  3174.6  3173.8  3173.0  3172.0  3171.8  3171.4  3171.0  3170.8   \n",
      "1  3175.8  3175.0  3174.8  3173.8  3172.8  3172.2  3172.0  3171.8  3171.6   \n",
      "2  3174.6  3174.4  3173.6  3171.2  3170.0  3169.8  3168.8  3168.0  3167.8   \n",
      "3  3176.0  3175.8  3175.0  3174.6  3174.4  3173.2  3173.0  3172.8  3172.2   \n",
      "4  3176.0  3175.8  3175.0  3174.6  3174.4  3172.8  3172.6  3172.2  3172.0   \n",
      "5  3175.0  3174.6  3174.4  3173.4  3172.8  3172.0  3171.8  3171.0  3170.8   \n",
      "6  3174.6  3174.4  3174.2  3173.4  3172.8  3172.4  3172.2  3172.0  3171.8   \n",
      "7  3174.2  3173.4  3173.2  3172.8  3172.2  3172.0  3171.8  3171.0  3170.8   \n",
      "8  3174.6  3174.4  3174.2  3173.4  3173.2  3172.6  3172.4  3172.2  3172.0   \n",
      "9  3174.2  3173.8  3173.4  3173.0  3172.6  3172.4  3172.2  3172.0  3171.8   \n",
      "\n",
      "       B5  SV5  SV4  SV3  SV2  SV1  BV1  BV2  BV3  BV4  BV5  \n",
      "0  3170.4    1    2    3    1    5    1    3    2    6    1  \n",
      "1  3171.4    2    2    1    1    1    1   13    2    3    3  \n",
      "2  3167.6    1   17    1    1    3    3    3    1    1   21  \n",
      "3  3171.8    6    2    3    1   15    1    3    1    1    1  \n",
      "4  3171.8    6    2    3    2   11    5    2    1    7    1  \n",
      "5  3170.2    3    2    4    4   13    7    1   19    1   16  \n",
      "6  3171.0    2    4    4    4   13    1    2    1    2   31  \n",
      "7  3170.2    4    5    3    3    3    5    2   31    1    6  \n",
      "8  3171.8    2    4    4    5    3    3    4    6    6    2  \n",
      "9  3171.0    3    1    5    4    5    3    8   11    2   31  \n",
      "0    3171.9\n",
      "1    3172.5\n",
      "2    3169.9\n",
      "3    3173.8\n",
      "4    3173.6\n",
      "5    3172.4\n",
      "6    3172.6\n",
      "7    3172.1\n",
      "8    3172.9\n",
      "9    3172.5\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv('data/order_books/2011/IF1101.csv').iloc[:, 1:].head(10)\n",
    "price = (df['S1'] + df['B1']) / 2\n",
    "print(df)\n",
    "print(price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "0   3171.8  3171.4  3171.0  3170.8  3170.4    1    3    2    6    1  3171.9\n",
      "1   3172.2  3172.0  3171.8  3171.6  3171.4    1   13    2    3    3  3172.5\n",
      "2   3169.8  3168.8  3168.0  3167.8  3167.6    3    3    1    1   21  3169.9\n",
      "3   3173.2  3173.0  3172.8  3172.2  3171.8    1    3    1    1    1  3173.8\n",
      "4   3172.8  3172.6  3172.2  3172.0  3171.8    5    2    1    7    1  3173.6\n",
      "5   3172.0  3171.8  3171.0  3170.8  3170.2    7    1   19    1   16  3172.4\n",
      "6   3172.4  3172.2  3172.0  3171.8  3171.0    1    2    1    2   31  3172.6\n",
      "7   3172.0  3171.8  3171.0  3170.8  3170.2    5    2   31    1    6  3172.1\n",
      "8   3172.6  3172.4  3172.2  3172.0  3171.8    3    4    6    6    2  3172.9\n",
      "9   3172.4  3172.2  3172.0  3171.8  3171.0    3    8   11    2   31  3172.5\n",
      "10  3174.0  3173.2  3172.6  3172.2  3172.0    2    1    3    1   21  3174.4\n",
      "11  3174.4  3172.6  3172.2  3172.0  3171.8    1    4    7   21    5  3174.7\n",
      "12  3175.2  3174.8  3173.2  3172.6  3172.2    2    1    2    5    7  3175.4\n",
      "13  3175.0  3174.8  3173.2  3172.6  3172.2    3    5    3    6    7  3175.5\n",
      "14  3173.2  3173.0  3172.6  3172.2  3172.0    6    1    7    7   21  3174.0\n",
      "15  3174.8  3174.0  3173.6  3173.4  3173.0    3    1    2    1   10  3175.0\n",
      "16  3174.0  3173.6  3173.4  3173.0  3172.8    1    2    5   10    1  3174.4\n",
      "17  3174.8  3173.6  3173.4  3173.0  3172.8   14    1    4   10    5  3174.9\n",
      "18  3173.4  3173.0  3172.8  3172.6  3172.4    4   12    5   14   10  3173.9\n",
      "19  3173.0  3172.8  3172.6  3172.4  3172.2    8    5   13   10   11  3173.5\n",
      "20  3173.0  3172.8  3172.6  3172.4  3172.2    4    4   12   10   11  3173.5\n",
      "21  3173.2  3173.0  3172.8  3172.6  3172.4    1    2    4   12   10  3173.7\n",
      "22  3174.0  3173.0  3172.8  3172.6  3172.4    6    2    4   12   10  3174.4\n",
      "23  3172.8  3172.6  3172.4  3172.2  3172.0    4   12   10    7   16  3173.4\n",
      "24  3172.8  3172.6  3172.4  3172.2  3172.0    1   12   10    7   16  3172.9\n",
      "25  3172.6  3172.4  3172.2  3172.0  3171.8    5   10    7   16    5  3173.3\n",
      "26  3172.6  3172.4  3172.2  3172.0  3171.8    5   10    8   16    5  3173.1\n",
      "27  3173.0  3172.8  3172.6  3172.4  3172.2    2    3    5   10    8  3173.5\n",
      "28  3173.0  3172.8  3172.6  3172.4  3172.2    4    3    5   10    8  3173.5\n",
      "29  3172.6  3172.4  3172.2  3172.0  3171.8    2   10    8   16    6  3172.8\n",
      "..     ...     ...     ...     ...     ...  ...  ...  ...  ...  ...     ...\n",
      "70  3173.4  3173.0  3172.8  3172.6  3172.4    1    5    3    8    2  3173.7\n",
      "71  3174.0  3173.0  3172.8  3172.6  3172.4   10    4    3    8    2  3174.3\n",
      "72  3174.0  3173.0  3172.8  3172.6  3172.4   10    4    3    8    2  3174.4\n",
      "73  3174.4  3174.0  3173.8  3173.6  3173.4    1   13    1    2    1  3174.5\n",
      "74  3174.6  3174.2  3174.0  3173.8  3173.6    1    1   15    1    2  3174.7\n",
      "75  3174.6  3174.2  3174.0  3173.8  3173.6    3    1   22    1    2  3174.7\n",
      "76  3174.6  3174.0  3173.8  3173.6  3173.4    7   23    1    2    2  3174.7\n",
      "77  3174.8  3174.6  3174.2  3174.0  3173.8    9    9    3   23    1  3174.9\n",
      "78  3174.8  3174.6  3174.2  3174.0  3173.8    9   15    3   23    1  3174.9\n",
      "79  3174.8  3174.6  3174.2  3174.0  3173.8    4   15    3   23    1  3174.9\n",
      "80  3174.8  3174.6  3174.2  3174.0  3173.8   24   15    3   23    1  3174.9\n",
      "81  3174.8  3174.6  3174.2  3174.0  3173.8   22   15    3   23    1  3174.9\n",
      "82  3175.4  3175.0  3174.8  3174.6  3174.2    3    1   19   15    7  3175.5\n",
      "83  3175.6  3175.4  3175.0  3174.8  3174.6    3    3    7   20   15  3175.7\n",
      "84  3176.0  3175.8  3175.6  3175.4  3175.0    3    2    6    4    9  3176.1\n",
      "85  3176.0  3175.8  3175.6  3175.4  3175.0    2    4    6    4   10  3176.1\n",
      "86  3176.2  3176.0  3175.8  3175.6  3175.4    8    2   10    6    4  3176.4\n",
      "87  3176.8  3176.2  3176.0  3175.8  3175.6    3   15    4   10    8  3176.9\n",
      "88  3176.2  3176.0  3175.8  3175.6  3175.4   10    4   10    8    4  3176.4\n",
      "89  3176.6  3176.2  3176.0  3175.8  3175.6    3   14    3   10    8  3176.7\n",
      "90  3176.2  3176.0  3175.8  3175.6  3175.4   15    3   11    8    1  3176.3\n",
      "91  3176.2  3176.0  3175.8  3175.6  3175.4    8    3   11    9    1  3176.4\n",
      "92  3175.8  3175.6  3175.4  3175.0  3174.8    8    9    1    8   20  3176.1\n",
      "93  3176.0  3175.8  3175.6  3175.4  3175.0    6    5    8    1    8  3176.1\n",
      "94  3175.8  3175.6  3175.4  3175.0  3174.8    5    8    1    8   20  3175.9\n",
      "95  3175.8  3175.6  3175.4  3175.0  3174.8    3    7    1    9   20  3176.3\n",
      "96  3176.4  3176.2  3176.0  3175.8  3175.6   16    2    2    3    7  3176.6\n",
      "97  3176.6  3176.4  3176.2  3176.0  3175.8   13   15    3    6    3  3176.8\n",
      "98  3176.8  3176.6  3176.4  3176.2  3176.0    3    5   15    3   10  3176.9\n",
      "99  3177.4  3177.2  3177.0  3176.8  3176.4    4   10    9    4   15  3177.6\n",
      "\n",
      "[100 rows x 11 columns]\n"
     ]
    }
   ],
   "source": [
    "print(read(is_buy=True, path='data/order_books/2011/IF1101.csv', no_obs = 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read(path, is_buy=False, no_obs=None, slice_size=50):\n",
    "    df = pd.read_csv(path).iloc[:, 1:].head(no_obs)\n",
    "    price = (df['S1'] + df['B1']) / 2\n",
    "    df['price'] = price\n",
    "    num_slices = int(no_obs / slice_size)\n",
    "    if is_buy:\n",
    "        new_df = pd.DataFrame(df, columns=['B1', 'B2', 'B3', 'B4', 'B5', 'BV1', 'BV2', 'BV3', 'BV4', 'BV5', 'price'])\n",
    "    else:\n",
    "        new_df = pd.DataFrame(df, columns=['S1', 'S2', 'S3', 'S4', 'S5', 'SV1', 'SV2', 'SV3', 'SV4', 'SV5', 'price'])\n",
    "    return [new_df.iloc[n*slice_size:(n+1)*slice_size, :] for n in range(num_slices)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[       B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "0  2282.0  2280.0  2273.0  2268.0  2265.6    1    1    1    2    3  2285.4\n",
      "1  2280.6  2280.0  2278.0  2273.0  2268.0    1    1    3    1    2  2281.3\n",
      "2  2282.8  2282.0  2281.4  2280.6  2280.0    1    1    1    1    1  2283.9,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "3  2281.4  2280.6  2280.0  2274.8  2273.0    1    1    1    3    1  2283.4\n",
      "4  2280.6  2280.0  2279.0  2273.0  2272.2    1    1    1    1    3  2281.3\n",
      "5  2278.8  2278.4  2273.0  2272.2  2269.2    1    1    1    3    1  2279.9,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "6  2279.2  2278.8  2278.4  2273.0  2272.2    1    1    1    1    3  2281.4\n",
      "7  2280.0  2279.2  2278.8  2278.4  2273.0    1    1    1    1    1  2280.5\n",
      "8  2280.8  2280.0  2279.2  2278.8  2278.4    1    1    1    1    1  2282.0]\n",
      "[       B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "0  2761.0  2760.2  2757.2  2756.8  2756.6    1    1    1    1   14  2765.5\n",
      "1  2761.0  2760.2  2757.2  2756.8  2756.6    1    1    1    1   14  2765.5\n",
      "2  2763.6  2761.8  2761.6  2761.4  2761.2    1    4    1    2    1  2766.2,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "3  2765.6  2763.6  2761.8  2761.6  2761.2    1   33    4    1    1  2767.2\n",
      "4  2763.6  2761.8  2761.6  2761.2  2761.0    2    4    1    1    2  2765.9\n",
      "5  2763.6  2761.8  2761.6  2761.2  2761.0  107    4    1    1    2  2765.6,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "6  2766.8  2763.6  2761.8  2761.6  2761.2    1    3    4    1    1  2767.1\n",
      "7  2766.4  2763.6  2761.8  2761.6  2761.2    1    3    4    1    1  2766.7\n",
      "8  2766.8  2766.4  2763.6  2761.8  2761.6    1    1    3    4    1  2766.9]\n",
      "[       B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "0  2521.0  2515.0  2510.2  2506.2  2502.0    1    1    1    1    2  2523.0\n",
      "1  2506.2  2502.0  2500.0  2495.2  2485.2    1    2    2    1    1  2512.1\n",
      "2  2510.4  2506.2  2502.0  2500.0  2495.2    1    3    2    2    1  2511.4,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "3  2509.6  2506.2  2500.0  2495.2  2485.2    1    3    2    1    1  2510.0\n",
      "4  2509.2  2506.2  2500.0  2495.2  2485.2    1    3    2    1    1  2509.8\n",
      "5  2509.8  2509.2  2506.2  2502.2  2501.2    1    1    2    2    2  2510.4,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "6  2510.2  2509.8  2509.2  2506.2  2502.2    1    1    1    2    2  2510.6\n",
      "7  2510.2  2509.8  2509.2  2507.4  2506.2    1    1    1    4    2  2511.3\n",
      "8  2513.0  2512.4  2511.0  2510.2  2509.8    1    1    3    1    5  2513.7]\n",
      "[       B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "0  2471.2  2470.2  2470.0  2466.0  2465.2    1    1    1    1    2  2475.4\n",
      "1  2471.2  2470.2  2470.0  2466.0  2465.2    1    1    1    1    2  2475.4\n",
      "2  2472.6  2471.8  2471.6  2471.4  2471.2    2    1   22    4    1  2475.8,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "3  2476.8  2472.8  2472.6  2471.8  2471.6    1    2    2    1   22  2477.4\n",
      "4  2476.8  2472.6  2471.8  2471.6  2471.4    1    2    1   23    2  2477.1\n",
      "5  2477.2  2476.8  2472.6  2471.8  2471.6    1    1    2    1   24  2477.6,        B1      B2      B3      B4      B5  BV1  BV2  BV3  BV4  BV5   price\n",
      "6  2476.8  2472.6  2471.8  2471.6  2471.4    1    2    1   24    2  2477.4\n",
      "7  2477.2  2476.8  2472.6  2472.0  2471.6    1    1    2    1   24  2477.6\n",
      "8  2477.6  2477.2  2476.8  2472.6  2472.0    1    1    1    2    1  2477.9]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-96-29181ce6a1d9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgenerate_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'data/order_books/2011/IF1101.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mno_obs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-91-348ea4746496>\u001b[0m in \u001b[0;36mgenerate_data\u001b[0;34m(function, *args, **kwargs)\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchdir\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0;34m'../..'\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mPATHS\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0;32myield\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mgenerate_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'data/order_books/2011/IF1101.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mno_obs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-95-7a2e446bfe2c>\u001b[0m in \u001b[0;36mread\u001b[0;34m(path, is_buy, no_obs, slice_size)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_buy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mno_obs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mslice_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mno_obs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mprice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'S1'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'B1'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'price'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprice\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mnum_slices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mno_obs\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mslice_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    700\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    704\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    433\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 435\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    436\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    437\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1137\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m         \u001b[0mnrows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_validate_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'nrows'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1141\u001b[0m         \u001b[0;31m# May alter columns / col_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1993\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1994\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1995\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1996\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1997\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_low_memory\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_column_data\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_tokens\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/pandas/core/dtypes/common.py\u001b[0m in \u001b[0;36mis_categorical_dtype\u001b[0;34m(arr_or_dtype)\u001b[0m\n\u001b[1;32m    570\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    571\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 572\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mis_categorical_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    573\u001b[0m     \"\"\"\n\u001b[1;32m    574\u001b[0m     \u001b[0mCheck\u001b[0m \u001b[0mwhether\u001b[0m \u001b[0man\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlike\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mCategorical\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "for a in generate_data(read, 'data/order_books/2011/IF1101.csv', no_obs=10, slice_size=3):\n",
    "    print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[       S1      S2      S3      S4      S5  SV1  SV2  SV3  SV4  SV5   price\n",
       " 0  3172.0  3173.0  3173.8  3174.6  3174.8    5    1    3    2    1  3171.9\n",
       " 1  3172.8  3173.8  3174.8  3175.0  3175.8    1    1    1    2    2  3172.5\n",
       " 2  3170.0  3171.2  3173.6  3174.4  3174.6    3    1    1   17    1  3169.9,\n",
       "        S1      S2      S3      S4      S5  SV1  SV2  SV3  SV4  SV5   price\n",
       " 3  3174.4  3174.6  3175.0  3175.8  3176.0   15    1    3    2    6  3173.8\n",
       " 4  3174.4  3174.6  3175.0  3175.8  3176.0   11    2    3    2    6  3173.6\n",
       " 5  3172.8  3173.4  3174.4  3174.6  3175.0   13    4    4    2    3  3172.4,\n",
       "        S1      S2      S3      S4      S5  SV1  SV2  SV3  SV4  SV5   price\n",
       " 6  3172.8  3173.4  3174.2  3174.4  3174.6   13    4    4    4    2  3172.6\n",
       " 7  3172.2  3172.8  3173.2  3173.4  3174.2    3    3    3    5    4  3172.1\n",
       " 8  3173.2  3173.4  3174.2  3174.4  3174.6    3    5    4    4    2  3172.9]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "read(path='data/order_books/2011/IF1101.csv', no_obs=10, slice_size=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1.0,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1.0,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}