train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train, validate & Generate Submitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from keras import losses\n",
    "import keras\n",
    "\n",
    "from src.data import get_generators, format_bytes\n",
    "from src.model import get_2losses_model, add_clf_layer, add_clf_layer_sparse, get_model, get_RAEwSC_compiled, get_RAEwSC_and_WS_compiled\n",
    "from src.losses import cross_entropy_with_axis, model_evaluate, write_submission_files\n",
    "\n",
    "from keras.utils.vis_utils import plot_model\n",
    "# plot_model(model, show_shapes=True, show_layer_names=True)\n",
    "# start tensorboard: tensorboard --logdir ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define global names\n",
    "EXTRA_DATA_MODEL, EXTRA_DATA_MODELwIN, RAEwSCwWSwINwCLF = \"RAE_w_SC_WS\", \"RAE_w_SC_WS_wIN\", \"RAEwSCwWSwINwCLF\"\n",
    "\n",
    "# define model and city\n",
    "MODEL_TYPES = {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\", 3: EXTRA_DATA_MODEL, 4: EXTRA_DATA_MODELwIN, \n",
    "               5: RAEwSCwWSwINwCLF}\n",
    "CITIES = {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "DATA_SPLIT = {0: \"non-overlapping\", 1: \"all_possible_slots\", 2: \"like-test\"}\n",
    "\n",
    "############################################# DEFINE HYPERPARAMETERS\n",
    "DATA_SPLIT_IDX = 2 # choose how to split data\n",
    "params = {\n",
    "    # data params\n",
    "    \"batch_size\": 4,\n",
    "    \"length_seq_in\": 3, #3, # Set up when calling a model\n",
    "    \"length_seq_out\": 3,\n",
    "    \"batch_size_validation\": 48,\n",
    "    \"data_split\": DATA_SPLIT[DATA_SPLIT_IDX], # Set up when calling a model\n",
    "    \"batch_size_test\": 5,\n",
    "    # loss params\n",
    "    'optimizer': 'adam',  # adam or sgd\n",
    "    'learning_rate': 0.001,\n",
    "    'decay_rate': 0.9999,  # Learning rate decay per minibatch.\n",
    "    'min_learning_rate': .0000001,  # Minimum learning rate.\n",
    "}\n",
    "\n",
    "############################################# DEFINE MODEL WEIGHTS\n",
    "models_weight = {\n",
    "    \"ConvLSTM\": { # _weights_e04_valoss0.009_relu \n",
    "        \"Moscow\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_model_ep_2.h5\".format(\"Moscow\"), \n",
    "        \"Istanbul\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_weights_e03_valoss0.009_relu3.hdf5\".format(\"Istanbul\"), \n",
    "        \"Berlin\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_model_ep_5.h5\".format(\"Berlin\"),\n",
    "        \"Model_definition_file\": \"{}/projects/nips_traffic/src/models.py\".format(\"/home/pherruzo\"),\n",
    "        \"Model_definition_function\": \"build_model\" \n",
    "    },\n",
    "    \n",
    "    \"ConvLSTM+Clf\": { # _weights_e04_valoss0.025_relu.hdf5, _weights_e04_valoss0.025_relu2.hdf\n",
    "        \"Moscow\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_weights_e04_valoss0.025_relu2.hdf5\".format(\"Moscow\"),\n",
    "        \"Istanbul\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_weights_06-1.04.hdf5\".format(\"Istanbul\"), # weights_Istanbul_06-1.04.hdf5, weights_Istanbul_04-1.05.hdf5, weights_Istanbul_03-1.05.hdf5  \n",
    "        \"Berlin\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_model_ep_1_reg_clf_cont.h5\".format(\"Berlin\")\n",
    "    },\n",
    "    \n",
    "    \"RAE_w_SC\": { # best: _weights_e03_valoss0.012_raewsc_sgd | prev: _weights_e02_valoss0.014_raewsc_sgd_emb\n",
    "        \"Moscow\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_weights_e03_valoss0.012_raewsc_sgd.hdf5\".format(\"Moscow\"),\n",
    "        \"Istanbul\": None, \n",
    "        \"Berlin\": None\n",
    "    },\n",
    "    \n",
    "    EXTRA_DATA_MODEL: { # best: _e02_valoss0.012_exo_vars, best in test-like: _e91_valoss0.012_just_try \n",
    "        \"Moscow\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_e91_valoss0.012_just_try.hdf5\".format(\"Moscow\"), # works good for num_seq_in=3\n",
    "        \"Istanbul\": None,\n",
    "        \"Berlin\": None\n",
    "    },\n",
    "    \n",
    "    EXTRA_DATA_MODELwIN: { # best (all slots): _e04_valoss0.012_RAEwSCwWTwIN_new_all_slots, # like-test: _e18_valoss0.012_RAEwSCwWTwINz_all_slots_like_test\n",
    "        \"Moscow\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_e18_valoss0.012_RAEwSCwWTwINz_all_slots_like_test.hdf5\".format(\"Moscow\"),\n",
    "        \"Istanbul\": None,\n",
    "        \"Berlin\": None\n",
    "    },\n",
    "    \n",
    "    RAEwSCwWSwINwCLF: { # best (all slots): _e19_valoss0.502_RAEwSCwWSwINwCLF #  _e12_valoss1.294_RAEwSCwWSwINwCLF\n",
    "        \"Moscow\": \"/home/pherruzo/projects/nips_traffic/checkpoints/{}_e44_valoss0.026_RAEwSCwWSwINwCLF.hdf5\".format(\"Moscow\"),\n",
    "        \"Istanbul\": None,\n",
    "        \"Berlin\": None,\n",
    "        \"loss_type\": [None, 'Sparse'][1]#'Categorical'\n",
    "    }\n",
    "}\n",
    "\n",
    "############################################# PREPARE POSSIBLE MODELS\n",
    "def get_a_model(model_type, training_params):\n",
    "    \"\"\" model_type in [\"ConvLSTM+Clf\", \"ConvLSTM\", \"RAE_w_SC\"] \"\"\"\n",
    "    \n",
    "    if model_type==\"ConvLSTM+Clf\":\n",
    "        return get_2losses_model(model_path=models_weight[training_params[\"previus_model\"]][\"Model_definition_file\"], \n",
    "                                model_name=models_weight[training_params[\"previus_model\"]][\"Model_definition_function\"], \n",
    "                                weights_initial_model=None, \n",
    "                                weights_current_model=models_weight[training_params[\"current_model\"]][training_params[\"city\"]], \n",
    "                                add_model=add_clf_layer_sparse, \n",
    "                                add_loss=losses.sparse_categorical_crossentropy,\n",
    "                                sample_weight_mode=\"temporal\")\n",
    "    \n",
    "    elif model_type==\"ConvLSTM\":\n",
    "        from keras import optimizers\n",
    "        return get_model(model_path=models_weight[training_params[\"previus_model\"]][\"Model_definition_file\"], \n",
    "                         function_name=models_weight[training_params[\"previus_model\"]][\"Model_definition_function\"], \n",
    "                         weights=models_weight[training_params[\"previus_model\"]][training_params[\"city\"]], \n",
    "                         opt=optimizers.Adam(lr=0.0001), loss=losses.mean_squared_error)\n",
    "    \n",
    "    elif model_type==\"RAE_w_SC\":\n",
    "        return get_RAEwSC_compiled(weights=models_weight[model_type][training_params[\"city\"]], \n",
    "                                   lr=params['learning_rate'])\n",
    "    elif model_type==EXTRA_DATA_MODEL:\n",
    "        dropout_enc, b_norm_enc = 0.04, True\n",
    "        dropout_dec, b_norm_dec = 0.03, True\n",
    "        print(\"dropout_enc:\", dropout_enc, \"dropout_dec:\", dropout_dec)\n",
    "        # get basic model def get_RAEwSC_and_WS_compiled(weights=None, lr = 0.001, grad_clip=1., loss_weights={'predicted_frames':1., 'predicted_emb':1.}):\n",
    "        return get_RAEwSC_and_WS_compiled(weights=models_weight[model_type][training_params[\"city\"]], lr=params['learning_rate'],\n",
    "                                          loss_weights={'predicted_frames':1., 'predicted_emb':1.},\n",
    "                                           dropout_enc=dropout_enc, dropout_dec=dropout_dec, \n",
    "                                           b_norm_enc=b_norm_enc, b_norm_dec=b_norm_dec)\n",
    "    elif model_type==EXTRA_DATA_MODELwIN:\n",
    "        from src.models.RAEwSCwWSwIN import get_RAEwSC_and_WS_compiled_w_input\n",
    "        \n",
    "        loss_w = {'predicted_frames':1., 'predicted_emb':0.9}\n",
    "        \n",
    "        dropout_enc = 0.05\n",
    "        dropout_dec = 0.03\n",
    "        \n",
    "        print(\"loading {} ...\".format(model_type))\n",
    "        print(\"dropout_enc:\", dropout_enc, \"dropout_dec:\", dropout_dec)\n",
    "        print(\"loss weights:\", loss_w)\n",
    "        return get_RAEwSC_and_WS_compiled_w_input(weights=models_weight[model_type][training_params[\"city\"]], length_seq_in=params['length_seq_in'], \n",
    "                                                  lr=params['learning_rate'], loss_weights=loss_w,\n",
    "                                                  dropout_enc=dropout_enc, dropout_dec=dropout_dec, for_new_model=False)\n",
    "    \n",
    "    elif model_type==RAEwSCwWSwINwCLF:\n",
    "        from src.models.RAEwSCwWSwINxCLF import get_RAEwSCwWSwIwCLF_compiled as get_model\n",
    "        \n",
    "        loss_w = {'predicted_frames':0., 'predicted_emb':1.}\n",
    "        \n",
    "        dropout_enc = 0.05\n",
    "        dropout_dec = 0.03\n",
    "        \n",
    "        print(\"loading {} ...\".format(model_type))\n",
    "        print(\"dropout_enc:\", dropout_enc, \"dropout_dec:\", dropout_dec)\n",
    "        print(\"loss weights:\", loss_w)\n",
    "        return get_model(weights=models_weight[model_type][training_params[\"city\"]], length_seq_in=params['length_seq_in'], \n",
    "                         lr=params['learning_rate'], loss_weights=loss_w,\n",
    "                         dropout_enc=dropout_enc, dropout_dec=dropout_dec)\n",
    "    # get_RAEwSCwWSwIwCLF_compiled\n",
    "\n",
    "############################################# DEFINE TRAINING PARAMS\n",
    "def get_training_params(model_type, city):\n",
    "    \"\"\" model_type in [\"ConvLSTM\", \"ConvLSTM+Clf\", \"RAE_w_SC\", \"RAE_w_SC_WS\"] \n",
    "        city in ['Moscow', 'Istanbul', 'Berlin']\n",
    "    \"\"\"\n",
    "    training_params = {\n",
    "        \"city\": city,\n",
    "        \"previus_model\": \"ConvLSTM\",     # used in old methods, NOT TO CHANGE todo: replace\n",
    "        \"current_model\": \"ConvLSTM+Clf\", # used in old methods, NOT TO CHANGE todo: replace\n",
    "        \"tensorboard_path\": \"/home/pherruzo/projects/nips_traffic/tensorboard/\",\n",
    "        \"log_path\": \"/home/pherruzo/projects/nips_traffic/log_files/\",\n",
    "        \"model_type\": model_type,\n",
    "        \"output_dir\": \"/home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt\"\n",
    "    }\n",
    "    return training_params\n",
    "\n",
    "############################################# LOAD DATA GENERATOR\n",
    "def load_model_and_data(model_type, city, params, length_seq_in=None, data_split=None, debug=False):\n",
    "    \n",
    "    if length_seq_in is not None:\n",
    "        params[\"length_seq_in\"] = length_seq_in\n",
    "        \n",
    "    if data_split is not None:\n",
    "        params[\"data_split\"] = DATA_SPLIT[data_split]\n",
    "        \n",
    "    # load training params\n",
    "    training_params = get_training_params(model_type, city)\n",
    "    \n",
    "    # load data\n",
    "    if \"loss_type\" in models_weight[model_type]:\n",
    "        loss_type = models_weight[model_type][\"loss_type\"]\n",
    "        print(\"---> loss type for clf:\", loss_type)\n",
    "    else:\n",
    "        loss_type = None\n",
    "    training_ds, val_ds, test_ds = get_generators(training_params[\"model_type\"],   params[\"batch_size\"], \n",
    "                                                  training_params[\"city\"],         params[\"length_seq_in\"], \n",
    "                                                  params[\"batch_size_validation\"], data_split=params[\"data_split\"], debug=debug, loss_type=loss_type)\n",
    "    # load model\n",
    "    model = get_a_model(model_type, training_params)\n",
    "    \n",
    "    # print info\n",
    "    print(\"\")\n",
    "    print(\"· length_seq_in:\", params[\"length_seq_in\"])\n",
    "    print(\"· data_split:\", params[\"data_split\"])\n",
    "    print(\"· model_type:\", training_params[\"model_type\"])\n",
    "    print(\"· city      :\", training_params[\"city\"])\n",
    "    print(\" 1 epoch training is {} batches, validation is {}, and test is {}.\".format(len(training_ds), len(val_ds), len(test_ds)))\n",
    "    \n",
    "    return training_ds, val_ds, test_ds, model, training_params\n",
    "\n",
    "def plot(keys_list, history):\n",
    "\n",
    "    leg = []\n",
    "    for k in keys_list:\n",
    "        plt.plot(history.history[k], '-o')\n",
    "        leg.append(k)\n",
    "    \n",
    "    plt.title(leg)\n",
    "    plt.xlabel('epoch')\n",
    "    plt.legend(leg, loc='center')#'upper left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TRAINING"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Load model & data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1022 09:13:58.139183 140354142054208 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W1022 09:13:58.147670 140354142054208 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W1022 09:13:58.150118 140354142054208 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4185: The name tf.truncated_normal is deprecated. Please use tf.random.truncated_normal instead.\n",
      "\n",
      "W1022 09:13:58.166394 140354142054208 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W1022 09:13:58.166708 140354142054208 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:190: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading RAE_w_SC_WS_wIN ...\n",
      "dropout_enc: 0.05 dropout_dec: 0.03\n",
      "loss weights: {'predicted_frames': 1.0, 'predicted_emb': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1022 09:13:58.648825 140354142054208 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:1834: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "W1022 09:13:58.723539 140354142054208 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "W1022 09:13:58.726788 140354142054208 deprecation.py:506] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder output shape: (?, 8, 8, 32) reshaped: (?, ?)\n",
      "Decoder input shape: (?, 2048) reshaped: (?, 8, 8, 32)\n",
      "embeddings.shape: (?, 6, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "concatenation of all inputs: (?, 6, 2388)\n",
      "FC before recurrent embeddings.shape: (?, 6, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_e18_valoss0.012_RAEwSCwWTwINz_all_slots_like_test.hdf5\n",
      "Freezing layers...\n",
      "\n",
      "· length_seq_in: 6\n",
      "· data_split: like-test\n",
      "· model_type: RAE_w_SC_WS_wIN\n",
      "· city      : Moscow\n",
      " 1 epoch training is 357 batches, validation is 5, and test is 72.\n"
     ]
    }
   ],
   "source": [
    "#### load data and model --> \n",
    "# choose model          {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\", 3: \"RAE_w_SC_WS\", 4: \"RAE_w_SC_WS_wIN\", 5: \"RAEwSCwWSwINwCLF\"}\n",
    "# choose city           {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "# choose data-split     {0: \"non-overlapping\", 1: \"all_possible_slots\", 2: \"like-test\"}\n",
    "# choose length_seq_in  3, 4, 5, 6, 7, 8, 9, 10, 11, 12\n",
    "\n",
    "params['learning_rate'] = 0.01\n",
    "debug = False\n",
    "\n",
    "m_type, city = 4, 0\n",
    "data_split_idx = 2\n",
    "length_seq_in = 6\n",
    "\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params, \n",
    "                                                                           length_seq_in=length_seq_in, data_split=data_split_idx, debug=debug)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Define callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('/home/pherruzo/projects/nips_traffic/checkpoints/Moscow_e{epoch:02d}_valoss{val_loss:.3f}_tttest.hdf5',\n",
       " '/home/pherruzo/projects/nips_traffic/tensorboard/RAE_w_SC_WS_wIN/tb_Moscow_1/run_tttest_1')"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras import backend as K \n",
    "\n",
    "class LRTensorBoard(keras.callbacks.TensorBoard):\n",
    "    def __init__(self, log_dir, **kwargs):  # add other arguments to __init__ if you need\n",
    "        super().__init__(log_dir=log_dir, **kwargs)\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        logs.update({'lr': K.eval(self.model.optimizer.lr)})\n",
    "        super().on_epoch_end(epoch, logs)\n",
    "        \n",
    "############################################# PREPARE CALLBACKS\n",
    "run_name = 'tttest'#'RAEwSCaWT_w_IN'\n",
    "run_num = '1'\n",
    "## tensorboard\n",
    "tb_folder = \"/tb_{}_{}/run_{}\".format(training_params[\"city\"],\"1\", run_name+'_'+run_num)\n",
    "tb_folder = training_params[\"tensorboard_path\"]+MODEL_TYPES[m_type]+tb_folder\n",
    "# tb_update_freq = 1000 * params[\"batch_size\"]\n",
    "# print(\"Saving training progress in {} each {} samples.\".format(tb_folder, tb_update_freq))\n",
    "\n",
    "# save best model\n",
    "# checkpoint\n",
    "root= \"/home/pherruzo\" \n",
    "model_dir=root+\"/projects/nips_traffic/checkpoints/\"\n",
    "filepath=model_dir+training_params[\"city\"]+\"_e{epoch:02d}_valoss{val_loss:.3f}_\"+run_name+\".hdf5\"\n",
    "checkpoint = keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', save_weights_only=True, save_best_only=True)\n",
    "\n",
    "# ep 1-4 lr: 0.001\n",
    "# ep 5- lr: 0.0001\n",
    "\n",
    "my_callbacks = [keras.callbacks.TensorBoard(log_dir=tb_folder), checkpoint, LRTensorBoard(log_dir=tb_folder)] #, update_freq=tb_update_freq)]\n",
    "# my_callbacks = [keras.callbacks.TensorBoard(log_dir=tb_folder, histogram_freq=1, write_grads=True), checkpoint, LRTensorBoard(log_dir=tb_folder]\n",
    "# my_callbacks = None\n",
    "filepath, tb_folder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TRAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#K.set_value(model.optimizer.lr, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning rate: 0.01\n",
      "Epoch 2/20\n",
      "2279/2280 [============================>.] - ETA: 0s - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.0445e-08-->> Data has been shuffled in: training\n",
      "2280/2280 [==============================] - 1806s 792ms/step - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.0438e-08 - val_loss: 0.0120 - val_mean_squared_error: 0.0120 - val_r_emb_loss: 3.2361e-09\n",
      "Epoch 3/20\n",
      "2279/2280 [============================>.] - ETA: 0s - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.0804e-08-->> Data has been shuffled in: training\n",
      "2280/2280 [==============================] - 1801s 790ms/step - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.0797e-08 - val_loss: 0.0120 - val_mean_squared_error: 0.0120 - val_r_emb_loss: 6.6008e-09\n",
      "Epoch 4/20\n",
      "2279/2280 [============================>.] - ETA: 0s - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.1589e-08-->> Data has been shuffled in: training\n",
      "2280/2280 [==============================] - 1808s 793ms/step - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.1595e-08 - val_loss: 0.0119 - val_mean_squared_error: 0.0119 - val_r_emb_loss: 3.9253e-08\n",
      "Epoch 5/20\n",
      "2279/2280 [============================>.] - ETA: 0s - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.0835e-08-->> Data has been shuffled in: training\n",
      "2280/2280 [==============================] - 1804s 791ms/step - loss: 0.0121 - mean_squared_error: 0.0121 - r_emb_loss: 2.0826e-08 - val_loss: 0.0119 - val_mean_squared_error: 0.0119 - val_r_emb_loss: 3.6410e-10\n",
      "Epoch 6/20\n",
      "1169/2280 [==============>...............] - ETA: 14:36 - loss: 0.0120 - mean_squared_error: 0.0120 - r_emb_loss: 2.0431e-08"
     ]
    }
   ],
   "source": [
    "DEBUG = False\n",
    "\n",
    "if DEBUG:\n",
    "    print('validation data to memory')\n",
    "    valid_data = ([x_val, y_val], y_val)\n",
    "    my_callbacks = [keras.callbacks.TensorBoard(log_dir=tb_folder, histogram_freq=1, write_grads=True), \n",
    "                    checkpoint, \n",
    "                    LRTensorBoard(log_dir=tb_folder)]\n",
    "else:\n",
    "    valid_data = val_ds\n",
    "    my_callbacks = [keras.callbacks.TensorBoard(log_dir=tb_folder), checkpoint, LRTensorBoard(log_dir=tb_folder)]\n",
    "\n",
    "print(\"learning rate:\", K.get_value(model.optimizer.lr))\n",
    "hist = model.fit_generator(training_ds, epochs=20, verbose=1, callbacks=my_callbacks, \n",
    "                           validation_data=valid_data,\n",
    "                           max_queue_size=10, workers=4, shuffle=True, initial_epoch=1)#, steps_per_epoch=2000) # starts with next epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['val_loss', 'val_Concat_predicted_frames_loss', 'val_softmax_clf_loss', 'val_Concat_predicted_frames_mean_squared_error', 'val_softmax_clf_sparse_categorical_crossentropy', 'loss', 'Concat_predicted_frames_loss', 'softmax_clf_loss', 'Concat_predicted_frames_mean_squared_error', 'softmax_clf_sparse_categorical_crossentropy', 'lr'])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hist.history.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VALIDATION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation of the first model ConvLSTM vs. ConvLSTM+Clf\n",
    "\n",
    "### Moscow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_model_ep_2.h5\n",
      "· data_split: non-overlapping\n",
      "· model_type: ConvLSTM\n",
      "· city      : Moscow\n",
      " 1 epoch training is 3420 batches, validation is 14, and test is 15.\n",
      "Evaluating time-bin/time-slot batch 1/14\n",
      "Evaluating time-bin/time-slot batch 2/14\n",
      "Evaluating time-bin/time-slot batch 3/14\n",
      "Evaluating time-bin/time-slot batch 4/14\n",
      "Evaluating time-bin/time-slot batch 5/14\n",
      "Evaluating time-bin/time-slot batch 6/14\n",
      "Evaluating time-bin/time-slot batch 7/14\n",
      "Evaluating time-bin/time-slot batch 8/14\n",
      "Evaluating time-bin/time-slot batch 9/14\n",
      "Evaluating time-bin/time-slot batch 10/14\n",
      "Evaluating time-bin/time-slot batch 11/14\n",
      "Evaluating time-bin/time-slot batch 12/14\n",
      "Evaluating time-bin/time-slot batch 13/14\n",
      "Evaluating time-bin/time-slot batch 14/14\n",
      "city: Moscow shape: (14, 3, 3)\n",
      "mean mse: 0.012643993\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.5058866e-05' '0.0055825664' '0.032222174']\n",
      " ['10 minutes' '9.9132e-05' '0.005572014' '0.03221288']\n",
      " ['15 minutes' '9.873548e-05' '0.0056006834' '0.03231268']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.01358719\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.553936e-05' '0.0055827647' '0.035095695']\n",
      " ['10 minutes' '9.966829e-05' '0.005572655' '0.035012465']\n",
      " ['15 minutes' '9.9293095e-05' '0.0056013656' '0.035125267']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.32272322301947687\n"
     ]
    }
   ],
   "source": [
    "#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
    "#!!!!!!!!!!!!!!!!!!!! enviando cada Heading a un valor    !!!!!!!!!!!!!!!!!!!!\n",
    "#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
    "\n",
    "\n",
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### M1: ConvLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_model_ep_2.h5\n",
      "· data_split: non-overlapping\n",
      "· model_type: ConvLSTM\n",
      "· city      : Moscow\n",
      " 1 epoch training is 3420 batches, validation is 14, and test is 15.\n",
      "Evaluating time-bin/time-slot batch 1/14\n",
      "Evaluating time-bin/time-slot batch 2/14\n",
      "Evaluating time-bin/time-slot batch 3/14\n",
      "Evaluating time-bin/time-slot batch 4/14\n",
      "Evaluating time-bin/time-slot batch 5/14\n",
      "Evaluating time-bin/time-slot batch 6/14\n",
      "Evaluating time-bin/time-slot batch 7/14\n",
      "Evaluating time-bin/time-slot batch 8/14\n",
      "Evaluating time-bin/time-slot batch 9/14\n",
      "Evaluating time-bin/time-slot batch 10/14\n",
      "Evaluating time-bin/time-slot batch 11/14\n",
      "Evaluating time-bin/time-slot batch 12/14\n",
      "Evaluating time-bin/time-slot batch 13/14\n",
      "Evaluating time-bin/time-slot batch 14/14\n",
      "city: Moscow shape: (14, 3, 3)\n",
      "mean mse: 0.012643993\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.5058866e-05' '0.0055825664' '0.032222174']\n",
      " ['10 minutes' '9.9132e-05' '0.005572014' '0.03221288']\n",
      " ['15 minutes' '9.873548e-05' '0.0056006834' '0.03231268']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012643729\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.553936e-05' '0.0055827647' '0.032218546']\n",
      " ['10 minutes' '9.966829e-05' '0.005572655' '0.0322118']\n",
      " ['15 minutes' '9.9293095e-05' '0.0056013656' '0.032311935']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.26518264871667013\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### M2: ConvLSTM+Clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading weitghs new: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_weights_e04_valoss0.025_diff_slots.hdf5\n",
      "temporal\n",
      "· data_split: all_possible_slots\n",
      "· model_type: ConvLSTM+Clf\n",
      "· city      : Moscow\n",
      " 1 epoch training is 3427 batches, validation is 7, and test is 8.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.012121401\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.7021246e-05' '0.005336735' '0.030951554']\n",
      " ['10 minutes' '9.086769e-05' '0.005297211' '0.030895546']\n",
      " ['15 minutes' '9.0647016e-05' '0.005320187' '0.031022849']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012121498\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.7503046e-05' '0.005336918' '0.030949963']\n",
      " ['10 minutes' '9.1368864e-05' '0.00529756' '0.030895585']\n",
      " ['15 minutes' '9.11543e-05' '0.0053205346' '0.031022895']]\n",
      "-----------------------------\n",
      "mean mse with HEADING as clf: 0.024765693\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.7503046e-05' '0.005336918' '0.06633826']\n",
      " ['10 minutes' '9.1368864e-05' '0.00529756' '0.06942075']\n",
      " ['15 minutes' '9.11543e-05' '0.0053205346' '0.0709072']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4262261191351405\n",
      "Acc y_hat_clf in HEADING: 0.7927780931125732\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 1, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0930 09:33:06.677484 140628157896512 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0930 09:33:06.685691 140628157896512 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0930 09:33:06.686591 140628157896512 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0930 09:33:06.747815 140628157896512 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W0930 09:33:06.748385 140628157896512 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0930 09:33:08.156661 140628157896512 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weitghs new: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_weights_e04_valoss0.025_relu.hdf5\n",
      "temporal\n",
      "lr: 0.0001\n",
      "· data_split: all_possible_slots\n",
      "· model_type: ConvLSTM+Clf\n",
      "· city      : Moscow\n",
      " 1 epoch training is 3427 batches, validation is 7, and test is 8.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.012060078\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.504161e-05' '0.005294468' '0.030697852']\n",
      " ['10 minutes' '8.986601e-05' '0.00525921' '0.030810388']\n",
      " ['15 minutes' '8.995142e-05' '0.0052834027' '0.03093051']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012059943\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.553785e-05' '0.005295244' '0.03069506']\n",
      " ['10 minutes' '9.0369445e-05' '0.0052599707' '0.030809311']\n",
      " ['15 minutes' '9.0457776e-05' '0.0052841245' '0.03092942']]\n",
      "-----------------------------\n",
      "mean mse with HEADING as clf: 0.02506407\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.553785e-05' '0.005295244' '0.06797575']\n",
      " ['10 minutes' '9.0369445e-05' '0.0052599707' '0.07049753']\n",
      " ['15 minutes' '9.0457776e-05' '0.0052841245' '0.07099764']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.42688423572406753\n",
      "Acc y_hat_clf in HEADING: 0.7939766687186413\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 1, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1002 08:50:39.688166 140712892053312 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W1002 08:50:39.696547 140712892053312 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W1002 08:50:39.697452 140712892053312 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W1002 08:50:39.764677 140712892053312 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W1002 08:50:39.765223 140712892053312 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1002 08:50:41.210492 140712892053312 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weitghs new: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_weights_e04_valoss0.025_relu2.hdf5\n",
      "temporal\n",
      "lr: 0.0001\n",
      "· data_split: all_possible_slots\n",
      "· model_type: ConvLSTM+Clf\n",
      "· city      : Moscow\n",
      " 1 epoch training is 3427 batches, validation is 7, and test is 72.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.012037375\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.491112e-05' '0.005278308' '0.030639661']\n",
      " ['10 minutes' '8.9811576e-05' '0.0052422746' '0.030765565']\n",
      " ['15 minutes' '8.976435e-05' '0.005265665' '0.030880405']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012037826\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.539097e-05' '0.00527909' '0.03063932']\n",
      " ['10 minutes' '9.031116e-05' '0.0052430057' '0.030765897']\n",
      " ['15 minutes' '9.026724e-05' '0.0052664024' '0.03088076']]\n",
      "-----------------------------\n",
      "mean mse with HEADING as clf: 0.023907198\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.539097e-05' '0.00527909' '0.06461452']\n",
      " ['10 minutes' '9.031116e-05' '0.0052430057' '0.067056745']\n",
      " ['15 minutes' '9.026724e-05' '0.0052664024' '0.06743904']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.45550401716303857\n",
      "Acc y_hat_clf in HEADING: 0.8034265906112237\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 1, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### M3: RAE_w_SC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "embeddings.shape: (?, 3, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_weights_e03_valoss0.012_raewsc_sgd.hdf5\n",
      "lr: 0.0001\n",
      "· data_split: all_possible_slots\n",
      "· model_type: RAE_w_SC\n",
      "· city      : Moscow\n",
      " 1 epoch training is 20164 batches, validation is 7, and test is 72.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.011781985\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.933762e-05' '0.0050940025' '0.029733649']\n",
      " ['10 minutes' '0.00010692001' '0.00514298' '0.030180553']\n",
      " ['15 minutes' '0.00010690651' '0.0051779905' '0.03039552']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.011873306\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.994132e-05' '0.0051191538' '0.029978836']\n",
      " ['10 minutes' '0.00010755615' '0.005168263' '0.030429129']\n",
      " ['15 minutes' '0.000107549175' '0.0052033146' '0.03064601']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4695847822185742\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 2, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])\n",
    "\n",
    "# here 3 ep after 7 previous ep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "embeddings.shape: (?, 3, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_weights_e07_valoss0.012_raewsc.hdf5\n",
      "lr: 0.0001\n",
      "· data_split: all_possible_slots\n",
      "· model_type: RAE_w_SC\n",
      "· city      : Moscow\n",
      " 1 epoch training is 20164 batches, validation is 7, and test is 72.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.011783949\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.911314e-05' '0.005091771' '0.02972464']\n",
      " ['10 minutes' '0.00010690344' '0.005143501' '0.030185902']\n",
      " ['15 minutes' '0.00010749385' '0.005180878' '0.030415341']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.011882596\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.9769524e-05' '0.005118162' '0.029990768']\n",
      " ['10 minutes' '0.00010759694' '0.005170031' '0.030455142']\n",
      " ['15 minutes' '0.000108197' '0.0052075097' '0.030686174']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.44512526421930093\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 2, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best model performance on \"like-test\" split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "embeddings.shape: (?, 3, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_weights_e03_valoss0.012_raewsc_sgd.hdf5\n",
      "lr: 0.01\n",
      "· data_split: like-test\n",
      "· model_type: RAE_w_SC\n",
      "· city      : Moscow\n",
      " 1 epoch training is 357 batches, validation is 1, and test is 72.\n",
      "--> Evaluating model with keras:\n",
      "[0.012460684403777122, 0.012391732074320316, 6.89523876644671e-05]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/1\n",
      "city: Moscow shape: (1, 3, 3)\n",
      "mean mse: 0.012322592\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010845185' '0.0051686806' '0.031293485']\n",
      " ['10 minutes' '0.00011554455' '0.0051963837' '0.03172064']\n",
      " ['15 minutes' '0.00011273609' '0.0052599995' '0.03192741']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012392129\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010900894' '0.0051863724' '0.031483386']\n",
      " ['10 minutes' '0.000116159004' '0.0052141827' '0.031910203']\n",
      " ['15 minutes' '0.00011332478' '0.0052781715' '0.03211835']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4597972737422279\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 2, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "print(\"--> Evaluating model with keras:\")\n",
    "mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> Evaluating model with keras:\n",
      "[0.012465481646358967, 0.01239652931690216, 6.89523876644671e-05]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/1\n",
      "city: Moscow shape: (1, 3, 3)\n",
      "mean mse: 0.012327139\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000108456465' '0.0051201344' '0.03127118']\n",
      " ['10 minutes' '0.00011458512' '0.0051807207' '0.03177587']\n",
      " ['15 minutes' '0.00011655796' '0.0052235113' '0.032033242']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.01239688\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000109011504' '0.0051373923' '0.03146054']\n",
      " ['10 minutes' '0.000115180665' '0.0051983427' '0.031968005']\n",
      " ['15 minutes' '0.00011718102' '0.0052412394' '0.032225024']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4588287417645216\n"
     ]
    }
   ],
   "source": [
    "#######\n",
    "####### Here is tested with the real test bins. Above I was wrong and they were other time bins (close to the real but still diff)\n",
    "#######\n",
    "\n",
    "\n",
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 2, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "print(\"--> Evaluating model with keras:\")\n",
    "mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### M4: RAE_w_SC_WS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.011778687\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.8760844e-05' '0.0050893747' '0.029712154']\n",
      " ['10 minutes' '0.000106546446' '0.0051415437' '0.030173296']\n",
      " ['15 minutes' '0.00010698004' '0.0051786816' '0.030400863']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.011875393\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.93905e-05' '0.005116369' '0.029971529']\n",
      " ['10 minutes' '0.00010721101' '0.005168676' '0.030436058']\n",
      " ['15 minutes' '0.00010765677' '0.0052058576' '0.030665789']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4530881848924662\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\", 3: \"RAE_w_SC_WS\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 3, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "dropout_enc: 0.05 dropout_dec: 0.05\n",
      "embeddings.shape: (?, 3, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "concatenation of all inputs: (?, 3, 2388)\n",
      "FC before recurrent embeddings.shape: (?, 3, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_e02_valoss0.012_exo_vars.hdf5\n",
      "lr: 0.01 optimizer: <keras.optimizers.Adam object at 0x7fa5c83d3940>\n",
      "· data_split: like-test\n",
      "· model_type: RAE_w_SC_WS\n",
      "· city      : Moscow\n",
      " 1 epoch training is 357 batches, validation is 1, and test is 72.\n",
      "Evaluating time-bin/time-slot batch 1/1\n",
      "city: Moscow shape: (1, 3, 3)\n",
      "mean mse: 0.012315642\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.0001076026' '0.0051608565' '0.031261858']\n",
      " ['10 minutes' '0.00011494466' '0.005190074' '0.031705447']\n",
      " ['15 minutes' '0.000112849455' '0.0052564978' '0.031930648']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012389874\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000108190325' '0.0051796883' '0.031464577']\n",
      " ['10 minutes' '0.00011558382' '0.005209085' '0.0319083']\n",
      " ['15 minutes' '0.0001134756' '0.0052759657' '0.032134004']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.44456637144710537\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\", 3: \"RAE_w_SC_WS\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 3, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sistematic comparation of rae_w_sc_ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "dropout_enc: 0.05 dropout_dec: 0.05\n",
      "embeddings.shape: (?, 3, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "concatenation of all inputs: (?, 3, 2388)\n",
      "FC before recurrent embeddings.shape: (?, 3, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_e02_valoss0.012_exo_vars.hdf5\n",
      "lr: 0.01 optimizer: <keras.optimizers.Adam object at 0x7fa61bdf79b0>\n",
      "· data_split: like-test\n",
      "· model_type: RAE_w_SC_WS\n",
      "· city      : Moscow\n",
      " 1 epoch training is 357 batches, validation is 1, and test is 72.\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 3, 0\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validating over Validation Data Split: \"non-overlapping\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=========================================\n",
      "--> loading weitghs: Moscow_e02_valoss0.012_exo_vars.hdf5\n",
      "--> Evaluating model with keras:\n",
      "[0.011876085240926062, 0.011875380761921406, 7.044652323072244e-07]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.011778668\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.876041e-05' '0.0050893542' '0.029712116']\n",
      " ['10 minutes' '0.000106546584' '0.005141485' '0.03017341']\n",
      " ['15 minutes' '0.00010698072' '0.0051786355' '0.030400718']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.011875369\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.938991e-05' '0.0051163635' '0.029971553']\n",
      " ['10 minutes' '0.00010721106' '0.0051686256' '0.030435985']\n",
      " ['15 minutes' '0.000107657' '0.0052058115' '0.030665722']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4531061810400495\n",
      "\n",
      "=========================================\n",
      "--> loading weitghs: Moscow_e01_valoss0.013_wt.hdf5\n",
      "--> Evaluating model with keras:\n",
      "[0.0158633113439594, 0.014911934467298644, 0.0009513766433104008]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.014565215\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010420014' '0.0060389624' '0.03314874']\n",
      " ['10 minutes' '0.00012163934' '0.007274828' '0.03771726']\n",
      " ['15 minutes' '0.0001276423' '0.0075942236' '0.038959432']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.014796163\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000104759885' '0.006075663' '0.03395137']\n",
      " ['10 minutes' '0.00012208962' '0.007306051' '0.038372107']\n",
      " ['15 minutes' '0.00012790207' '0.007623932' '0.039481606']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.1852068587064765\n",
      "\n",
      "=========================================\n",
      "--> loading weitghs: Moscow_e02_valoss0.012_wt.hdf5\n",
      "--> Evaluating model with keras:\n",
      "[0.014693259128502436, 0.014176505750843458, 0.0005167535295608364]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.0140243955\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000102596474' '0.0060473015' '0.03193568']\n",
      " ['10 minutes' '0.00012028006' '0.007232179' '0.035730034']\n",
      " ['15 minutes' '0.00012921744' '0.0076662838' '0.037255995']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.014074869\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010314833' '0.006096057' '0.0322279']\n",
      " ['10 minutes' '0.00012079323' '0.007279237' '0.035894655']\n",
      " ['15 minutes' '0.00012940132' '0.0077011725' '0.037121456']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.34270825059242493\n",
      "\n",
      "=========================================\n",
      "--> loading weitghs: Moscow_e03_valoss0.012_wt.hdf5\n",
      "--> Evaluating model with keras:\n",
      "[0.015349417658788818, 0.014834222782935415, 0.0005151953337190207]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.014689967\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010484983' '0.0061330465' '0.032344792']\n",
      " ['10 minutes' '0.00012197263' '0.007598544' '0.037646495']\n",
      " ['15 minutes' '0.00013501015' '0.008237997' '0.039886992']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.014625305\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000105370455' '0.00616946' '0.032514267']\n",
      " ['10 minutes' '0.00012238178' '0.0076168403' '0.037521686']\n",
      " ['15 minutes' '0.0001348367' '0.008223646' '0.039219264']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.3935318857719469\n",
      "\n",
      "=========================================\n",
      "--> loading weitghs: Moscow_e04_valoss0.012_wt.hdf5\n",
      "--> Evaluating model with keras:\n",
      "[0.014845927778099264, 0.014330374076962471, 0.0005155537378089191]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.014229393\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00011100799' '0.00621958' '0.032552827']\n",
      " ['10 minutes' '0.00013025098' '0.0071999216' '0.03614852']\n",
      " ['15 minutes' '0.00014305148' '0.0076719495' '0.037887413']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.014219073\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00011272013' '0.0062384275' '0.032707322']\n",
      " ['10 minutes' '0.00013199427' '0.0072136065' '0.036174078']\n",
      " ['15 minutes' '0.00014448389' '0.0076606716' '0.037588354']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.545577774247499\n",
      "\n",
      "=========================================\n",
      "--> loading weitghs: Moscow_e05_valoss0.012_wt.hdf5\n",
      "--> Evaluating model with keras:\n",
      "[0.015684340947440693, 0.015169695285814149, 0.000514645536424853]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Moscow shape: (7, 3, 3)\n",
      "mean mse: 0.014901889\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.0001052284' '0.0061613545' '0.032716025']\n",
      " ['10 minutes' '0.000120803736' '0.00749902' '0.038205046']\n",
      " ['15 minutes' '0.00013356957' '0.008275842' '0.040900107']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.014956203\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010578382' '0.0061815516' '0.03325697']\n",
      " ['10 minutes' '0.00012126091' '0.0074989907' '0.038469516']\n",
      " ['15 minutes' '0.00013339201' '0.008223346' '0.040615015']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.21644124825508618\n",
      "Min mse: 0.011875380761921406 | Model: Moscow_e02_valoss0.012_exo_vars.hdf5\n"
     ]
    }
   ],
   "source": [
    "root = '/home/pherruzo/projects/nips_traffic/checkpoints/'\n",
    "weights = ['Moscow_e02_valoss0.012_exo_vars.hdf5', 'Moscow_e01_valoss0.013_wt.hdf5', 'Moscow_e02_valoss0.012_wt.hdf5', \n",
    "           'Moscow_e03_valoss0.012_wt.hdf5', 'Moscow_e04_valoss0.012_wt.hdf5', 'Moscow_e05_valoss0.012_wt.hdf5']\n",
    "best_mse = 1.\n",
    "best_model = ''\n",
    "for w in weights:\n",
    "    print(\"\\n=========================================\")\n",
    "    print('--> loading weitghs:', w)\n",
    "    model.load_weights(root+w, by_name=True)\n",
    "\n",
    "    print(\"--> Evaluating model with keras:\")\n",
    "    mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "    print(mse_image_and_emb)\n",
    "    \n",
    "    if mse_image_and_emb[1] < best_mse:\n",
    "        best_mse = mse_image_and_emb[1]\n",
    "        best_model = w\n",
    "\n",
    "    print(\"--> Evaluating model with my framework:\")\n",
    "    model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])\n",
    "print(\"\")\n",
    "print(\"---> Min mse: {} | Model: {}\".format(best_mse, best_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validating over Validation Data Split: \"test-like\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "root = '/home/pherruzo/projects/nips_traffic/checkpoints/'\n",
    "weights = ['Moscow_e02_valoss0.012_exo_vars.hdf5', 'Moscow_e01_valoss0.013_wt.hdf5', 'Moscow_e02_valoss0.012_wt.hdf5', \n",
    "           'Moscow_e03_valoss0.012_wt.hdf5', 'Moscow_e04_valoss0.012_wt.hdf5', 'Moscow_e05_valoss0.012_wt.hdf5']\n",
    "best_mse = 1.\n",
    "best_model = ''\n",
    "for w in weights:\n",
    "    print(\"\\n=========================================\")\n",
    "    print('--> loading weitghs:', w)\n",
    "    model.load_weights(root+w, by_name=True)\n",
    "\n",
    "    print(\"--> Evaluating model with keras:\")\n",
    "    mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "    print(mse_image_and_emb)\n",
    "    \n",
    "    if mse_image_and_emb[1] < best_mse:\n",
    "        best_mse = mse_image_and_emb[1]\n",
    "        best_model = w\n",
    "\n",
    "    print(\"--> Evaluating model with my framework:\")\n",
    "    model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])\n",
    "\n",
    "print(\"\")\n",
    "print(\"---> Min mse: {} | Model: {}\".format(best_mse, best_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### M5: RAE_w_SC_WS_wIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['learning_rate'] = 0.01\n",
    "debug = False\n",
    "\n",
    "m_type, city = 4, 0\n",
    "data_split_idx = 2\n",
    "length_seq_in = 6\n",
    "\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params, \n",
    "                                                                           length_seq_in=length_seq_in, data_split=data_split_idx, debug=debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=========================================\n",
      "--> Evaluating model with keras:\n",
      "[0.012540066614747047, 0.012540065683424473, 5.502912570065632e-10]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/1\n",
      "city: Moscow shape: (1, 3, 3)\n",
      "mean mse: 0.012464177\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00011239189' '0.0053015905' '0.031504516']\n",
      " ['10 minutes' '0.000117650496' '0.0053512575' '0.03199256']\n",
      " ['15 minutes' '0.000119427255' '0.0054019815' '0.03227622']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012540261\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000113290065' '0.005317206' '0.03171509']\n",
      " ['10 minutes' '0.000118585995' '0.0053668665' '0.03220445']\n",
      " ['15 minutes' '0.00012037671' '0.005417869' '0.032488614']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4602828635856159\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n=========================================\")\n",
    "\n",
    "print(\"--> Evaluating model with keras:\")\n",
    "mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean mse: 0.012464179\n",
    "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
    " [' 5 minutes' '0.00011239186' '0.0053015915' '0.031504527']\n",
    " ['10 minutes' '0.00011765045' '0.0053512566' '0.03199256']\n",
    " ['15 minutes' '0.00011942723' '0.00540198' '0.03227623']]\n",
    "-----------------------------\n",
    "mean mse like submission: 0.01254026\n",
    "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
    " [' 5 minutes' '0.00011329015' '0.0053172046' '0.031715088']\n",
    " ['10 minutes' '0.000118586075' '0.0053668655' '0.032204457']\n",
    " ['15 minutes' '0.00012037677' '0.005417869' '0.032488607']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "##### Trained with all data\n",
    "###### Eval in all data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading RAE_w_SC_WS_wIN ...\n",
      "dropout_enc: 0.05 dropout_dec: 0.03\n",
      "loss weights: {'predicted_frames': 1.0, 'predicted_emb': 0.9}\n",
      "Encoder output shape: (?, 8, 8, 32) reshaped: (?, ?)\n",
      "Decoder input shape: (?, 2048) reshaped: (?, 8, 8, 32)\n",
      "embeddings.shape: (?, 6, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "concatenation of all inputs: (?, 6, 2388)\n",
      "FC before recurrent embeddings.shape: (?, 6, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_e18_valoss0.012_RAEwSCwWTwINz_all_slots_like_test.hdf5\n",
      "lr: 0.01 optimizer: <keras.optimizers.Adam object at 0x7f9d1c4e2e48>\n",
      "\n",
      "· length_seq_in: 6\n",
      "· data_split: non-overlapping\n",
      "· model_type: RAE_w_SC_WS_wIN\n",
      "· city      : Moscow\n",
      " 1 epoch training is 2280 batches, validation is 5, and test is 72.\n"
     ]
    }
   ],
   "source": [
    "#### load data and model --> \n",
    "# choose model          {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\", 3: \"RAE_w_SC_WS\", 4: \"RAE_w_SC_WS_wIN\"}\n",
    "# choose city           {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "# choose data-split     {0: \"non-overlapping\", 1: \"all_possible_slots\", 2: \"like-test\"}\n",
    "# choose length_seq_in  3, 4, 5, 6, 7, 8, 9, 10, 11, 12\n",
    "\n",
    "params['learning_rate'] = 0.01\n",
    "debug = False\n",
    "\n",
    "m_type, city = 4, 0\n",
    "data_split_idx = 0\n",
    "length_seq_in = 6\n",
    "\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params, \n",
    "                                                                           length_seq_in=length_seq_in, data_split=data_split_idx, debug=debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> Evaluating model with keras:\n",
      "[0.011863351466932468, 0.011863351466932468, 1.7734439076152076e-13]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/5\n",
      "Evaluating time-bin/time-slot batch 2/5\n",
      "Evaluating time-bin/time-slot batch 3/5\n",
      "Evaluating time-bin/time-slot batch 4/5\n",
      "Evaluating time-bin/time-slot batch 5/5\n",
      "city: Moscow shape: (5, 3, 3)\n",
      "mean mse: 0.011725507\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.476603e-05' '0.005103347' '0.02954914']\n",
      " ['10 minutes' '0.000101131714' '0.0051477193' '0.02997674']\n",
      " ['15 minutes' '0.00010321515' '0.00518856' '0.03026494']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.011816756\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '9.5909214e-05' '0.005128963' '0.029793534']\n",
      " ['10 minutes' '0.000102340266' '0.0051734922' '0.030223647']\n",
      " ['15 minutes' '0.00010444787' '0.0052143666' '0.03051411']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4375281034092196\n"
     ]
    }
   ],
   "source": [
    "print(\"--> Evaluating model with keras:\")\n",
    "mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Eval in like-test:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading RAE_w_SC_WS_wIN ...\n",
      "dropout_enc: 0.05 dropout_dec: 0.03\n",
      "loss weights: {'predicted_frames': 1.0, 'predicted_emb': 0.9}\n",
      "Encoder output shape: (?, 8, 8, 32) reshaped: (?, ?)\n",
      "Decoder input shape: (?, 2048) reshaped: (?, 8, 8, 32)\n",
      "embeddings.shape: (?, 6, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "concatenation of all inputs: (?, 6, 2388)\n",
      "FC before recurrent embeddings.shape: (?, 6, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_e07_valoss0.012_RAEwSCwWTwIN_new_all_slots.hdf5\n",
      "lr: 0.01 optimizer: <keras.optimizers.Adam object at 0x7f48e7092da0>\n",
      "\n",
      "· length_seq_in: 6\n",
      "· data_split: like-test\n",
      "· model_type: RAE_w_SC_WS_wIN\n",
      "· city      : Moscow\n",
      " 1 epoch training is 357 batches, validation is 1, and test is 72.\n",
      "--> Evaluating model with keras:\n",
      "[0.012378418818116188, 0.012378418818116188, 2.9945559920641074e-11]\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/1\n",
      "city: Moscow shape: (1, 3, 3)\n",
      "mean mse: 0.012310859\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010369894' '0.0051295073' '0.031218655']\n",
      " ['10 minutes' '0.000109874636' '0.0051862816' '0.031724878']\n",
      " ['15 minutes' '0.000112095135' '0.0052288054' '0.031983938']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.012378887\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00010475746' '0.0051481435' '0.03140138']\n",
      " ['10 minutes' '0.000110974135' '0.005205259' '0.031908516']\n",
      " ['15 minutes' '0.000113211754' '0.0052482225' '0.03216952']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.45462122315333325\n"
     ]
    }
   ],
   "source": [
    "params['learning_rate'] = 0.01\n",
    "debug = False\n",
    "\n",
    "m_type, city = 4, 0\n",
    "data_split_idx = 2\n",
    "length_seq_in = 6\n",
    "\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params, \n",
    "                                                                           length_seq_in=length_seq_in, data_split=data_split_idx, debug=debug)\n",
    "\n",
    "print(\"--> Evaluating model with keras:\")\n",
    "mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params['learning_rate'] = 0.01\n",
    "debug = False\n",
    "\n",
    "m_type, city = 4, 0\n",
    "data_split_idx = 2\n",
    "length_seq_in = 6\n",
    "\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params, \n",
    "                                                                           length_seq_in=length_seq_in, data_split=data_split_idx, debug=debug)\n",
    "\n",
    "print(\"--> Evaluating model with keras:\")\n",
    "mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## M6: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> loss type for clf: Sparse\n",
      "-->> Data has been shuffled in: training\n",
      "loading RAEwSCwWSwINwCLF ...\n",
      "dropout_enc: 0.05 dropout_dec: 0.03\n",
      "loss weights: {'predicted_frames': 0.0, 'predicted_emb': 1.0}\n",
      "Encoder output shape: (?, 8, 8, 32) reshaped: (?, ?)\n",
      "Decoder input shape: (?, 2048) reshaped: (?, 8, 8, 32)\n",
      "embeddings.shape: (?, 6, 2048)\n",
      "future_embeddings.shape: (?, 3, 2048)\n",
      "concatenation of all inputs: (?, 6, 2388)\n",
      "FC before recurrent embeddings.shape: (?, 6, 2048)\n",
      "recurrent embeddings.shape: (?, ?, 2048)\n",
      "prediced_frames.shape: (?, 3, 495, 436, 3)\n",
      "clfs.shape: (?, ?, 5)\n",
      "Freezing layer clf_layer_conv1\n",
      "Freezing layer clf_layer_BN1\n",
      "Freezing layer clf_layer_conv2\n",
      "Freezing layer clf_layer_BN2\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Moscow_e44_valoss0.026_RAEwSCwWSwINwCLF.hdf5\n",
      "Freezing layer clf_layer_conv1\n",
      "Freezing layer clf_layer_BN1\n",
      "Freezing layer clf_layer_conv2\n",
      "Freezing layer clf_layer_BN2\n",
      "lr: 0.01 optimizer: <keras.optimizers.Adam object at 0x7f9d4b573ac8>\n",
      "\n",
      "· length_seq_in: 6\n",
      "· data_split: non-overlapping\n",
      "· model_type: RAEwSCwWSwINwCLF\n",
      "· city      : Moscow\n",
      " 1 epoch training is 2280 batches, validation is 28, and test is 72.\n"
     ]
    }
   ],
   "source": [
    "#### load data and model --> \n",
    "# choose model          {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\", 3: \"RAE_w_SC_WS\", 4: \"RAE_w_SC_WS_wIN\", 5: \"RAEwSCwWSwINwCLF\"}\n",
    "# choose city           {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "# choose data-split     {0: \"non-overlapping\", 1: \"all_possible_slots\", 2: \"like-test\"}\n",
    "# choose length_seq_in  3, 4, 5, 6, 7, 8, 9, 10, 11, 12\n",
    "\n",
    "params['learning_rate'] = 0.01\n",
    "debug = False\n",
    "\n",
    "m_type, city = 5, 0\n",
    "data_split_idx = 0\n",
    "length_seq_in = 6\n",
    "\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params, \n",
    "                                                                           length_seq_in=length_seq_in, data_split=data_split_idx, debug=debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> Evaluating model with keras:\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/28\n",
      "Evaluating time-bin/time-slot batch 2/28\n",
      "Evaluating time-bin/time-slot batch 3/28\n",
      "Evaluating time-bin/time-slot batch 4/28\n",
      "Evaluating time-bin/time-slot batch 5/28\n",
      "Evaluating time-bin/time-slot batch 6/28\n",
      "Evaluating time-bin/time-slot batch 7/28\n",
      "Evaluating time-bin/time-slot batch 8/28\n",
      "Evaluating time-bin/time-slot batch 9/28\n",
      "Evaluating time-bin/time-slot batch 10/28\n",
      "Evaluating time-bin/time-slot batch 11/28\n",
      "Evaluating time-bin/time-slot batch 12/28\n",
      "Evaluating time-bin/time-slot batch 13/28\n",
      "Evaluating time-bin/time-slot batch 14/28\n",
      "Evaluating time-bin/time-slot batch 15/28\n",
      "Evaluating time-bin/time-slot batch 16/28\n",
      "Evaluating time-bin/time-slot batch 17/28\n",
      "Evaluating time-bin/time-slot batch 18/28\n",
      "Evaluating time-bin/time-slot batch 19/28\n",
      "Evaluating time-bin/time-slot batch 20/28\n",
      "Evaluating time-bin/time-slot batch 21/28\n",
      "Evaluating time-bin/time-slot batch 22/28\n",
      "Evaluating time-bin/time-slot batch 23/28\n",
      "Evaluating time-bin/time-slot batch 24/28\n",
      "Evaluating time-bin/time-slot batch 25/28\n",
      "Evaluating time-bin/time-slot batch 26/28\n",
      "Evaluating time-bin/time-slot batch 27/28\n",
      "Evaluating time-bin/time-slot batch 28/28\n",
      "city: Moscow shape: (28, 3, 3)\n",
      "mean mse: 0.014440837\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.0002011169' '0.006675954' '0.036074672']\n",
      " ['10 minutes' '0.0002018543' '0.006714995' '0.036466144']\n",
      " ['15 minutes' '0.00019822127' '0.0067612757' '0.03667329']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.014442413\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.0001988346' '0.006673758' '0.03608823']\n",
      " ['10 minutes' '0.00019945111' '0.006712099' '0.036474533']\n",
      " ['15 minutes' '0.00019575385' '0.0067583723' '0.03668069']]\n",
      "-----------------------------\n",
      "mean mse with HEADING as clf: 0.022331094\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.0001988346' '0.006673758' '0.059731927']\n",
      " ['10 minutes' '0.00019945111' '0.006712099' '0.060200613']\n",
      " ['15 minutes' '0.00019575385' '0.0067583723' '0.06030903']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.5074962435627572\n",
      "Acc y_hat_clf in HEADING: 0.8268083990847751\n"
     ]
    }
   ],
   "source": [
    "print(\"--> Evaluating model with keras:\")\n",
    "# mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "# print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> Evaluating model with keras:\n",
      "--> Evaluating model with my framework:\n",
      "Evaluating time-bin/time-slot batch 1/5\n",
      "Evaluating time-bin/time-slot batch 2/5\n",
      "Evaluating time-bin/time-slot batch 3/5\n",
      "Evaluating time-bin/time-slot batch 4/5\n",
      "Evaluating time-bin/time-slot batch 5/5\n",
      "city: Moscow shape: (5, 3, 3)\n",
      "mean mse: 0.0149290245\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00021393457' '0.0067121834' '0.037486415']\n",
      " ['10 minutes' '0.00021237969' '0.0067554126' '0.037928812']\n",
      " ['15 minutes' '0.00020970749' '0.0068048937' '0.03803748']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.014928271\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00021133879' '0.006708829' '0.037493866']\n",
      " ['10 minutes' '0.00020962332' '0.0067517357' '0.037931755']\n",
      " ['15 minutes' '0.00020689824' '0.0068009854' '0.03803941']]\n",
      "-----------------------------\n",
      "mean mse with HEADING as clf: 0.022994768\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00021133879' '0.006708829' '0.06168472']\n",
      " ['10 minutes' '0.00020962332' '0.0067517357' '0.062139552']\n",
      " ['15 minutes' '0.00020689824' '0.0068009854' '0.062239207']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.49935595763959667\n",
      "Acc y_hat_clf in HEADING: 0.8209242141084647\n"
     ]
    }
   ],
   "source": [
    "# like-test\n",
    "\n",
    "print(\"--> Evaluating model with keras:\")\n",
    "# mse_image_and_emb = model.evaluate_generator(val_ds)\n",
    "# print(mse_image_and_emb)\n",
    "\n",
    "print(\"--> Evaluating model with my framework:\")\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### After retraining in like-data \n",
    "##### test in like-data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Istanbul\n",
    "#### I1: ConvLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Istanbul_model_ep_2.h5\n",
      "· data_split: non-overlapping\n",
      "· model_type: ConvLSTM\n",
      "· city      : Istanbul\n",
      " 1 epoch training is 3420 batches, validation is 14, and test is 15.\n",
      "Evaluating time-bin/time-slot batch 1/14\n",
      "Evaluating time-bin/time-slot batch 2/14\n",
      "Evaluating time-bin/time-slot batch 3/14\n",
      "Evaluating time-bin/time-slot batch 4/14\n",
      "Evaluating time-bin/time-slot batch 5/14\n",
      "Evaluating time-bin/time-slot batch 6/14\n",
      "Evaluating time-bin/time-slot batch 7/14\n",
      "Evaluating time-bin/time-slot batch 8/14\n",
      "Evaluating time-bin/time-slot batch 9/14\n",
      "Evaluating time-bin/time-slot batch 10/14\n",
      "Evaluating time-bin/time-slot batch 11/14\n",
      "Evaluating time-bin/time-slot batch 12/14\n",
      "Evaluating time-bin/time-slot batch 13/14\n",
      "Evaluating time-bin/time-slot batch 14/14\n",
      "city: Istanbul shape: (14, 3, 3)\n",
      "mean mse: 0.009468561\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.484252e-05' '0.003217898' '0.025126921']\n",
      " ['10 minutes' '9.0902184e-05' '0.0032406098' '0.025006728']\n",
      " ['15 minutes' '9.5427364e-05' '0.0032801959' '0.025073526']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.009467641\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.505773e-05' '0.0032169637' '0.025122391']\n",
      " ['10 minutes' '9.1175454e-05' '0.0032403965' '0.025004923']\n",
      " ['15 minutes' '9.5723495e-05' '0.0032800415' '0.025072087']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.6237735774815285\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 1\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1003 09:21:30.699272 139962008799040 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W1003 09:21:30.700322 139962008799040 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W1003 09:21:30.701263 139962008799040 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W1003 09:21:30.768834 139962008799040 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W1003 09:21:30.769308 139962008799040 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1003 09:21:32.049101 139962008799040 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Istanbul_weights_e04_valoss0.009_relu.hdf5\n",
      "· data_split: all_possible_slots\n",
      "· model_type: ConvLSTM\n",
      "· city      : Istanbul\n",
      " 1 epoch training is 3427 batches, validation is 7, and test is 72.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Istanbul shape: (7, 3, 3)\n",
      "mean mse: 0.009236385\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.1224265e-05' '0.0031339214' '0.024371097']\n",
      " ['10 minutes' '8.866059e-05' '0.0031522785' '0.024463298']\n",
      " ['15 minutes' '9.32195e-05' '0.003189214' '0.024554538']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.009235346\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '8.153998e-05' '0.0031343412' '0.024368929']\n",
      " ['10 minutes' '8.8971654e-05' '0.0031527367' '0.024458142']\n",
      " ['15 minutes' '9.3531686e-05' '0.0031897088' '0.024550209']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.6645604784557385\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 1\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1004 15:25:18.903795 140116142995264 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W1004 15:25:18.905065 140116142995264 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W1004 15:25:18.905945 140116142995264 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W1004 15:25:18.972841 140116142995264 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W1004 15:25:18.973316 140116142995264 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1004 15:25:20.292831 140116142995264 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Istanbul_weights_e04_valoss0.009_relu2.hdf5\n",
      "· data_split: all_possible_slots\n",
      "· model_type: ConvLSTM\n",
      "· city      : Istanbul\n",
      " 1 epoch training is 3427 batches, validation is 7, and test is 72.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Istanbul shape: (7, 3, 3)\n",
      "mean mse: 0.009111477\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '7.8989e-05' '0.0030745196' '0.024060233']\n",
      " ['10 minutes' '8.542469e-05' '0.0030886128' '0.024156224']\n",
      " ['15 minutes' '8.966214e-05' '0.0031202328' '0.024249407']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.009109962\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '7.9298465e-05' '0.0030749415' '0.024055013']\n",
      " ['10 minutes' '8.573853e-05' '0.0030890822' '0.024150735']\n",
      " ['15 minutes' '8.998233e-05' '0.0031206848' '0.02424417']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.6528350298896936\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 1\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1005 18:33:51.287383 140681566517056 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W1005 18:33:51.288608 140681566517056 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W1005 18:33:51.289451 140681566517056 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W1005 18:33:51.362081 140681566517056 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W1005 18:33:51.362553 140681566517056 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1005 18:33:52.643657 140681566517056 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Istanbul_weights_e03_valoss0.009_relu3.hdf5\n",
      "· data_split: all_possible_slots\n",
      "· model_type: ConvLSTM\n",
      "· city      : Istanbul\n",
      " 1 epoch training is 3427 batches, validation is 7, and test is 72.\n",
      "Evaluating time-bin/time-slot batch 1/7\n",
      "Evaluating time-bin/time-slot batch 2/7\n",
      "Evaluating time-bin/time-slot batch 3/7\n",
      "Evaluating time-bin/time-slot batch 4/7\n",
      "Evaluating time-bin/time-slot batch 5/7\n",
      "Evaluating time-bin/time-slot batch 6/7\n",
      "Evaluating time-bin/time-slot batch 7/7\n",
      "city: Istanbul shape: (7, 3, 3)\n",
      "mean mse: 0.009066772\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '7.8933466e-05' '0.0030429643' '0.023945691']\n",
      " ['10 minutes' '8.49807e-05' '0.0030559893' '0.024062108']\n",
      " ['15 minutes' '8.90371e-05' '0.0030893798' '0.024151865']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.009066001\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '7.9236386e-05' '0.003043327' '0.023942176']\n",
      " ['10 minutes' '8.529065e-05' '0.0030563776' '0.024059221']\n",
      " ['15 minutes' '8.935541e-05' '0.0030897295' '0.024149287']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.6579608981176259\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 1\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I2: ConvLSTM+Clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading weitghs new: /home/pherruzo/projects/nips_traffic/checkpoints/weights_Istanbul_06-1.04.hdf5\n",
      "temporal\n",
      "· data_split: non-overlapping\n",
      "· model_type: ConvLSTM+Clf\n",
      "· city      : Istanbul\n",
      " 1 epoch training is 3420 batches, validation is 14, and test is 15.\n",
      "Evaluating time-bin/time-slot batch 1/14\n",
      "Evaluating time-bin/time-slot batch 2/14\n",
      "Evaluating time-bin/time-slot batch 3/14\n",
      "Evaluating time-bin/time-slot batch 4/14\n",
      "Evaluating time-bin/time-slot batch 5/14\n",
      "Evaluating time-bin/time-slot batch 6/14\n",
      "Evaluating time-bin/time-slot batch 7/14\n",
      "Evaluating time-bin/time-slot batch 8/14\n",
      "Evaluating time-bin/time-slot batch 9/14\n",
      "Evaluating time-bin/time-slot batch 10/14\n",
      "Evaluating time-bin/time-slot batch 11/14\n",
      "Evaluating time-bin/time-slot batch 12/14\n",
      "Evaluating time-bin/time-slot batch 13/14\n",
      "Evaluating time-bin/time-slot batch 14/14\n",
      "city: Istanbul shape: (14, 3, 3)\n",
      "mean mse: 0.0096311355\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00011611988' '0.003463696' '0.025400912']\n",
      " ['10 minutes' '0.00011366458' '0.0034494302' '0.025202984']\n",
      " ['15 minutes' '0.00011529144' '0.0034895982' '0.02532852']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.0096280435\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000116184245' '0.003462654' '0.025392909']\n",
      " ['10 minutes' '0.0001137908' '0.0034486044' '0.02519677']\n",
      " ['15 minutes' '0.00011543539' '0.0034877318' '0.025318313']]\n",
      "-----------------------------\n",
      "mean mse with HEADING as clf: 0.023589974\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000116184245' '0.003462654' '0.06706913']\n",
      " ['10 minutes' '0.0001137908' '0.0034486044' '0.066951744']\n",
      " ['15 minutes' '0.00011543539' '0.0034877318' '0.067544505']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.6868282127743138\n",
      "Acc y_hat_clf in HEADING: 0.8293507330109012\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 1, 1\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Berlin\n",
    "#### B1: ConvLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Berlin_model_ep_5.h5\n",
      "· data_split: non-overlapping\n",
      "· model_type: ConvLSTM\n",
      "· city      : Berlin\n",
      " 1 epoch training is 3420 batches, validation is 14, and test is 15.\n",
      "Evaluating time-bin/time-slot batch 1/14\n",
      "Evaluating time-bin/time-slot batch 2/14\n",
      "Evaluating time-bin/time-slot batch 3/14\n",
      "Evaluating time-bin/time-slot batch 4/14\n",
      "Evaluating time-bin/time-slot batch 5/14\n",
      "Evaluating time-bin/time-slot batch 6/14\n",
      "Evaluating time-bin/time-slot batch 7/14\n",
      "Evaluating time-bin/time-slot batch 8/14\n",
      "Evaluating time-bin/time-slot batch 9/14\n",
      "Evaluating time-bin/time-slot batch 10/14\n",
      "Evaluating time-bin/time-slot batch 11/14\n",
      "Evaluating time-bin/time-slot batch 12/14\n",
      "Evaluating time-bin/time-slot batch 13/14\n",
      "Evaluating time-bin/time-slot batch 14/14\n",
      "city: Berlin shape: (14, 3, 3)\n",
      "mean mse: 0.007144515\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00024783806' '0.0054533286' '0.015612622']\n",
      " ['10 minutes' '0.00028681455' '0.0054070516' '0.015777105']\n",
      " ['15 minutes' '0.0003035501' '0.0054118345' '0.015800487']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.0071446532\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00024803536' '0.005453691' '0.015611866']\n",
      " ['10 minutes' '0.00028704022' '0.0054075904' '0.01577694']\n",
      " ['15 minutes' '0.00030381166' '0.005412434' '0.015800469']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.5365183158952271\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 2\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### B2: ConvLSTM+Clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n",
      "loading weitghs new: /home/pherruzo/projects/nips_traffic/checkpoints/Berlin_model_ep_1_reg_clf_cont.h5\n",
      "temporal\n",
      "· data_split: non-overlapping\n",
      "· model_type: ConvLSTM+Clf\n",
      "· city      : Berlin\n",
      " 1 epoch training is 3420 batches, validation is 14, and test is 15.\n",
      "Evaluating time-bin/time-slot batch 1/14\n",
      "Evaluating time-bin/time-slot batch 2/14\n",
      "Evaluating time-bin/time-slot batch 3/14\n",
      "Evaluating time-bin/time-slot batch 4/14\n",
      "Evaluating time-bin/time-slot batch 5/14\n",
      "Evaluating time-bin/time-slot batch 6/14\n",
      "Evaluating time-bin/time-slot batch 7/14\n",
      "Evaluating time-bin/time-slot batch 8/14\n",
      "Evaluating time-bin/time-slot batch 9/14\n",
      "Evaluating time-bin/time-slot batch 10/14\n",
      "Evaluating time-bin/time-slot batch 11/14\n",
      "Evaluating time-bin/time-slot batch 12/14\n",
      "Evaluating time-bin/time-slot batch 13/14\n",
      "Evaluating time-bin/time-slot batch 14/14\n",
      "city: Berlin shape: (14, 3, 3)\n",
      "mean mse: 0.0071446365\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.000258809' '0.0054499395' '0.015574783']\n",
      " ['10 minutes' '0.00029097605' '0.0054347995' '0.015759025']\n",
      " ['15 minutes' '0.00030667736' '0.005438801' '0.015787916']]\n",
      "-----------------------------\n",
      "mean mse like submission: 0.007143738\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00025894414' '0.0054489453' '0.015572272']\n",
      " ['10 minutes' '0.00029113545' '0.0054344037' '0.015757509']\n",
      " ['15 minutes' '0.00030684605' '0.0054378654' '0.015785726']]\n",
      "-----------------------------\n",
      "mean mse with HEADING as clf: 0.009356684\n",
      "[['        ' '    Speed   ' '   Volume  ' '   Heading  ']\n",
      " [' 5 minutes' '0.00025894414' '0.0054489453' '0.02233241']\n",
      " ['10 minutes' '0.00029113545' '0.0054344037' '0.022348206']\n",
      " ['15 minutes' '0.00030684605' '0.0054378654' '0.022351397']]\n",
      "=========\n",
      "Acc y_hat in HEADING: 0.4189126180620829\n",
      "Acc y_hat_clf in HEADING: 0.9480102466340998\n"
     ]
    }
   ],
   "source": [
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\", 2: \"RAE_w_SC\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 1, 2\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "model_evaluate(val_ds, model, training_params[\"log_path\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/pherruzo/data/nips_traffic_submissions/raewscwwtwi'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_params[\"output_dir\"] = '/home/pherruzo/data/nips_traffic_submissions/raewscwwtwi'\n",
    "training_params[\"output_dir\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate submission files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "City:Moscow, just wrote file 1/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181031_100m_bins.h5\n",
      "City:Moscow, just wrote file 2/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180323_100m_bins.h5\n",
      "City:Moscow, just wrote file 3/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181216_100m_bins.h5\n",
      "City:Moscow, just wrote file 4/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180415_100m_bins.h5\n",
      "City:Moscow, just wrote file 5/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180724_100m_bins.h5\n",
      "City:Moscow, just wrote file 6/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180731_100m_bins.h5\n",
      "City:Moscow, just wrote file 7/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180716_100m_bins.h5\n",
      "City:Moscow, just wrote file 8/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180516_100m_bins.h5\n",
      "City:Moscow, just wrote file 9/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180823_100m_bins.h5\n",
      "City:Moscow, just wrote file 10/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180314_100m_bins.h5\n",
      "City:Moscow, just wrote file 11/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180301_100m_bins.h5\n",
      "City:Moscow, just wrote file 12/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181017_100m_bins.h5\n",
      "City:Moscow, just wrote file 13/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180915_100m_bins.h5\n",
      "City:Moscow, just wrote file 14/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181001_100m_bins.h5\n",
      "City:Moscow, just wrote file 15/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180728_100m_bins.h5\n",
      "City:Moscow, just wrote file 16/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181014_100m_bins.h5\n",
      "City:Moscow, just wrote file 17/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180405_100m_bins.h5\n",
      "City:Moscow, just wrote file 18/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180422_100m_bins.h5\n",
      "City:Moscow, just wrote file 19/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180408_100m_bins.h5\n",
      "City:Moscow, just wrote file 20/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180606_100m_bins.h5\n",
      "City:Moscow, just wrote file 21/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180209_100m_bins.h5\n",
      "City:Moscow, just wrote file 22/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180524_100m_bins.h5\n",
      "City:Moscow, just wrote file 23/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180801_100m_bins.h5\n",
      "City:Moscow, just wrote file 24/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180825_100m_bins.h5\n",
      "City:Moscow, just wrote file 25/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180508_100m_bins.h5\n",
      "City:Moscow, just wrote file 26/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180908_100m_bins.h5\n",
      "City:Moscow, just wrote file 27/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181209_100m_bins.h5\n",
      "City:Moscow, just wrote file 28/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181122_100m_bins.h5\n",
      "City:Moscow, just wrote file 29/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180125_100m_bins.h5\n",
      "City:Moscow, just wrote file 30/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181112_100m_bins.h5\n",
      "City:Moscow, just wrote file 31/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180609_100m_bins.h5\n",
      "City:Moscow, just wrote file 32/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180303_100m_bins.h5\n",
      "City:Moscow, just wrote file 33/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180619_100m_bins.h5\n",
      "City:Moscow, just wrote file 34/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180205_100m_bins.h5\n",
      "City:Moscow, just wrote file 35/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180106_100m_bins.h5\n",
      "City:Moscow, just wrote file 36/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180929_100m_bins.h5\n",
      "City:Moscow, just wrote file 37/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180128_100m_bins.h5\n",
      "City:Moscow, just wrote file 38/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180421_100m_bins.h5\n",
      "City:Moscow, just wrote file 39/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180608_100m_bins.h5\n",
      "City:Moscow, just wrote file 40/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180917_100m_bins.h5\n",
      "City:Moscow, just wrote file 41/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180803_100m_bins.h5\n",
      "City:Moscow, just wrote file 42/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181218_100m_bins.h5\n",
      "City:Moscow, just wrote file 43/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180505_100m_bins.h5\n",
      "City:Moscow, just wrote file 44/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180226_100m_bins.h5\n",
      "City:Moscow, just wrote file 45/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181201_100m_bins.h5\n",
      "City:Moscow, just wrote file 46/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180123_100m_bins.h5\n",
      "City:Moscow, just wrote file 47/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181227_100m_bins.h5\n",
      "City:Moscow, just wrote file 48/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180529_100m_bins.h5\n",
      "City:Moscow, just wrote file 49/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181116_100m_bins.h5\n",
      "City:Moscow, just wrote file 50/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180110_100m_bins.h5\n",
      "City:Moscow, just wrote file 51/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180112_100m_bins.h5\n",
      "City:Moscow, just wrote file 52/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180912_100m_bins.h5\n",
      "City:Moscow, just wrote file 53/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181113_100m_bins.h5\n",
      "City:Moscow, just wrote file 54/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180325_100m_bins.h5\n",
      "City:Moscow, just wrote file 55/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180629_100m_bins.h5\n",
      "City:Moscow, just wrote file 56/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180214_100m_bins.h5\n",
      "City:Moscow, just wrote file 57/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180519_100m_bins.h5\n",
      "City:Moscow, just wrote file 58/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180926_100m_bins.h5\n",
      "City:Moscow, just wrote file 59/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181119_100m_bins.h5\n",
      "City:Moscow, just wrote file 60/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180322_100m_bins.h5\n",
      "City:Moscow, just wrote file 61/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180714_100m_bins.h5\n",
      "City:Moscow, just wrote file 62/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180821_100m_bins.h5\n",
      "City:Moscow, just wrote file 63/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180420_100m_bins.h5\n",
      "City:Moscow, just wrote file 64/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181012_100m_bins.h5\n",
      "City:Moscow, just wrote file 65/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180712_100m_bins.h5\n",
      "City:Moscow, just wrote file 66/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180620_100m_bins.h5\n",
      "City:Moscow, just wrote file 67/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181009_100m_bins.h5\n",
      "City:Moscow, just wrote file 68/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181130_100m_bins.h5\n",
      "City:Moscow, just wrote file 69/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180810_100m_bins.h5\n",
      "City:Moscow, just wrote file 70/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20181210_100m_bins.h5\n",
      "City:Moscow, just wrote file 71/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180212_100m_bins.h5\n",
      "City:Moscow, just wrote file 72/72: /home/pherruzo/data/nips_traffic_submissions/raewscwwtwi/Moscow/Moscow_test/20180221_100m_bins.h5\n"
     ]
    }
   ],
   "source": [
    "\n",
    "write_submission_files(test_ds, model, training_params[\"output_dir\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W1004 15:44:40.881412 139768439117632 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W1004 15:44:40.882701 139768439117632 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W1004 15:44:40.883593 139768439117632 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W1004 15:44:40.952800 139768439117632 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W1004 15:44:40.953341 139768439117632 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-->> Data has been shuffled in: training\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1004 15:44:42.229185 139768439117632 deprecation_wrapper.py:119] From /home/pherruzo/anaconda3/envs/nips/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading weitghs: /home/pherruzo/projects/nips_traffic/checkpoints/Istanbul_weights_e04_valoss0.009_relu2.hdf5\n",
      "· data_split: all_possible_slots\n",
      "· model_type: ConvLSTM\n",
      "· city      : Istanbul\n",
      " 1 epoch training is 3427 batches, validation is 7, and test is 72.\n",
      "City:Istanbul, just wrote file 1/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180919_100m_bins.h5\n",
      "City:Istanbul, just wrote file 2/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180522_100m_bins.h5\n",
      "City:Istanbul, just wrote file 3/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181006_100m_bins.h5\n",
      "City:Istanbul, just wrote file 4/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180417_100m_bins.h5\n",
      "City:Istanbul, just wrote file 5/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180518_100m_bins.h5\n",
      "City:Istanbul, just wrote file 6/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180415_100m_bins.h5\n",
      "City:Istanbul, just wrote file 7/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180413_100m_bins.h5\n",
      "City:Istanbul, just wrote file 8/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180724_100m_bins.h5\n",
      "City:Istanbul, just wrote file 9/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180811_100m_bins.h5\n",
      "City:Istanbul, just wrote file 10/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181102_100m_bins.h5\n",
      "City:Istanbul, just wrote file 11/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181017_100m_bins.h5\n",
      "City:Istanbul, just wrote file 12/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180915_100m_bins.h5\n",
      "City:Istanbul, just wrote file 13/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181010_100m_bins.h5\n",
      "City:Istanbul, just wrote file 14/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181212_100m_bins.h5\n",
      "City:Istanbul, just wrote file 15/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180806_100m_bins.h5\n",
      "City:Istanbul, just wrote file 16/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180204_100m_bins.h5\n",
      "City:Istanbul, just wrote file 17/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180402_100m_bins.h5\n",
      "City:Istanbul, just wrote file 18/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180719_100m_bins.h5\n",
      "City:Istanbul, just wrote file 19/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180209_100m_bins.h5\n",
      "City:Istanbul, just wrote file 20/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181115_100m_bins.h5\n",
      "City:Istanbul, just wrote file 21/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180804_100m_bins.h5\n",
      "City:Istanbul, just wrote file 22/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180319_100m_bins.h5\n",
      "City:Istanbul, just wrote file 23/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180203_100m_bins.h5\n",
      "City:Istanbul, just wrote file 24/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180104_100m_bins.h5\n",
      "City:Istanbul, just wrote file 25/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180326_100m_bins.h5\n",
      "City:Istanbul, just wrote file 26/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180910_100m_bins.h5\n",
      "City:Istanbul, just wrote file 27/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180717_100m_bins.h5\n",
      "City:Istanbul, just wrote file 28/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180328_100m_bins.h5\n",
      "City:Istanbul, just wrote file 29/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180216_100m_bins.h5\n",
      "City:Istanbul, just wrote file 30/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180528_100m_bins.h5\n",
      "City:Istanbul, just wrote file 31/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180107_100m_bins.h5\n",
      "City:Istanbul, just wrote file 32/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180508_100m_bins.h5\n",
      "City:Istanbul, just wrote file 33/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180213_100m_bins.h5\n",
      "City:Istanbul, just wrote file 34/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181127_100m_bins.h5\n",
      "City:Istanbul, just wrote file 35/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180726_100m_bins.h5\n",
      "City:Istanbul, just wrote file 36/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180428_100m_bins.h5\n",
      "City:Istanbul, just wrote file 37/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180125_100m_bins.h5\n",
      "City:Istanbul, just wrote file 38/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181003_100m_bins.h5\n",
      "City:Istanbul, just wrote file 39/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180613_100m_bins.h5\n",
      "City:Istanbul, just wrote file 40/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180324_100m_bins.h5\n",
      "City:Istanbul, just wrote file 41/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180622_100m_bins.h5\n",
      "City:Istanbul, just wrote file 42/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180722_100m_bins.h5\n",
      "City:Istanbul, just wrote file 43/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181024_100m_bins.h5\n",
      "City:Istanbul, just wrote file 44/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180730_100m_bins.h5\n",
      "City:Istanbul, just wrote file 45/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181223_100m_bins.h5\n",
      "City:Istanbul, just wrote file 46/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180611_100m_bins.h5\n",
      "City:Istanbul, just wrote file 47/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180409_100m_bins.h5\n",
      "City:Istanbul, just wrote file 48/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180215_100m_bins.h5\n",
      "City:Istanbul, just wrote file 49/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180503_100m_bins.h5\n",
      "City:Istanbul, just wrote file 50/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181201_100m_bins.h5\n",
      "City:Istanbul, just wrote file 51/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180816_100m_bins.h5\n",
      "City:Istanbul, just wrote file 52/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180113_100m_bins.h5\n",
      "City:Istanbul, just wrote file 53/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180802_100m_bins.h5\n",
      "City:Istanbul, just wrote file 54/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180607_100m_bins.h5\n",
      "City:Istanbul, just wrote file 55/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180123_100m_bins.h5\n",
      "City:Istanbul, just wrote file 56/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180911_100m_bins.h5\n",
      "City:Istanbul, just wrote file 57/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181116_100m_bins.h5\n",
      "City:Istanbul, just wrote file 58/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180110_100m_bins.h5\n",
      "City:Istanbul, just wrote file 59/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180920_100m_bins.h5\n",
      "City:Istanbul, just wrote file 60/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180527_100m_bins.h5\n",
      "City:Istanbul, just wrote file 61/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181203_100m_bins.h5\n",
      "City:Istanbul, just wrote file 62/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180629_100m_bins.h5\n",
      "City:Istanbul, just wrote file 63/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181004_100m_bins.h5\n",
      "City:Istanbul, just wrote file 64/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181222_100m_bins.h5\n",
      "City:Istanbul, just wrote file 65/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181109_100m_bins.h5\n",
      "City:Istanbul, just wrote file 66/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180605_100m_bins.h5\n",
      "City:Istanbul, just wrote file 67/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181119_100m_bins.h5\n",
      "City:Istanbul, just wrote file 68/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180322_100m_bins.h5\n",
      "City:Istanbul, just wrote file 69/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180807_100m_bins.h5\n",
      "City:Istanbul, just wrote file 70/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180905_100m_bins.h5\n",
      "City:Istanbul, just wrote file 71/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20180308_100m_bins.h5\n",
      "City:Istanbul, just wrote file 72/72: /home/pherruzo/data/nips_traffic_submissions/conv_clf_mixt/Istanbul/Istanbul_test/20181220_100m_bins.h5\n"
     ]
    }
   ],
   "source": [
    "# !rm -rf /home/pherruzo/data/nips_traffic_submissions/conv_clf/Moscow/Moscow_test/\n",
    "\n",
    "# load data and model --> {0: \"ConvLSTM\", 1: \"ConvLSTM+Clf\"}, {0: 'Moscow', 1: 'Istanbul', 2: 'Berlin'}\n",
    "m_type, city = 0, 1\n",
    "training_ds, val_ds, test_ds, model, training_params = load_model_and_data(MODEL_TYPES[m_type], CITIES[city], params)\n",
    "\n",
    "# eval\n",
    "write_submission_files(test_ds, model, training_params[\"output_dir\"], training_params[\"city\"], training_params[\"model_type\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}