Machine_Learning_SVR_&_DL_nested_cross_validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.svm import LinearSVR\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import keras \n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense , Dropout , Conv1D , Flatten , LSTM\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_mse_pointing = []\n",
    "train_mean_relative_width_pointing = []\n",
    "cross_val_mse_pointing = []\n",
    "cross_val_mean_relative_width_pointing = []\n",
    "val_mse_pointing = []\n",
    "val_mean_relative_width_pointing = []\n",
    "test_mse_pointing = []\n",
    "test_mean_relative_width_pointing = []\n",
    "\n",
    "train_mse_gaze = []\n",
    "train_mean_relative_width_gaze = []\n",
    "cross_val_mse_gaze = []\n",
    "cross_val_mean_relative_width_gaze = []\n",
    "val_mse_gaze = []\n",
    "val_mean_relative_width_gaze = []\n",
    "test_mse_gaze = []\n",
    "test_mean_relative_width_gaze = []\n",
    "\n",
    "train_mse_fuse = []\n",
    "train_mean_relative_width_fuse = []\n",
    "cross_val_mse_fuse = []\n",
    "cross_val_mean_relative_width_fuse = []\n",
    "val_mse_fuse = []\n",
    "val_mean_relative_width_fuse = []\n",
    "test_mse_fuse = []\n",
    "test_mean_relative_width_fuse = []\n",
    "\n",
    "train_std_relative_width_pointing = []\n",
    "cross_val_std_relative_width_pointing = []\n",
    "val_std_relative_width_pointing = []\n",
    "test_std_relative_width_pointing = []\n",
    "\n",
    "train_std_relative_width_gaze = []\n",
    "cross_val_std_relative_width_gaze = []\n",
    "val_std_relative_width_gaze = []\n",
    "test_std_relative_width_gaze = []\n",
    "\n",
    "train_std_relative_width_fuse = []\n",
    "cross_val_std_relative_width_fuse = []\n",
    "val_std_relative_width_fuse = []\n",
    "test_std_relative_width_fuse = []\n",
    "\n",
    "train_support_vector_shape_pointing = []\n",
    "cross_val_support_vector_shape_pointing = []\n",
    "val_support_vector_shape_pointing = []\n",
    "test_support_vector_shape_pointing = []\n",
    "\n",
    "train_support_vector_shape_gaze = []\n",
    "cross_val_support_vector_shape_gaze = []\n",
    "val_support_vector_shape_gaze = []\n",
    "test_support_vector_shape_gaze = []\n",
    "\n",
    "train_support_vector_shape_fuse = []\n",
    "cross_val_support_vector_shape_fuse = []\n",
    "val_support_vector_shape_fuse = []\n",
    "test_support_vector_shape_fuse = []\n",
    "\n",
    "\n",
    "FCNN_test_mse_pointing = []\n",
    "FCNN_test_mean_relative_width_pointing = []\n",
    "FCNN_test_mse_gaze = []\n",
    "FCNN_test_mean_relative_width_gaze = []\n",
    "FCNN_test_mse_fuse = []\n",
    "FCNN_test_mean_relative_width_fuse = []\n",
    "FCNN_test_std_relative_width_pointing = []\n",
    "FCNN_test_std_relative_width_gaze = []\n",
    "FCNN_test_std_relative_width_fuse = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sync_confirmed_list_train_all = [[23,27,28,29,30,32,33,37,38,39,40,41,42,45,46,50,53,56,57,59,62,63,66,67,68,69,71,72,73],\n",
    "                                [9,10,11,14,15,17,18,24,25,29,31,39,40,41,42,50,53,56,57,59,62,63,66,67,68,69,71,72,73],\n",
    "                                [9,10,11,14,15,17,18,23,24,25,27,28,30,31,32,33,37,38,45,46,56,57,62,67,68,69,71,72,73],\n",
    "                                [9,10,11,14,15,17,18,23,24,25,27,28,29,30,31,32,33,37,38,39,40,41,42,45,46,50,53,59,63,66],\n",
    "                                [11,17,18,24,30,39,40,63,66,67,68,69,72],\n",
    "                                [32,33,38,24,30,39,40,63,66,67,68,69,72],\n",
    "                                [11,17,18,32,33,38,40,63,66,67,68,69,72],\n",
    "                                [11,17,18,32,33,38,24,30,39,67,68,69,72],\n",
    "                                [11,17,18,32,33,38,24,30,39,40,63,66,72],\n",
    "                                [9,10,14,23,41,42,56,57,73],\n",
    "                                [27,28,29,23,41,42,56,57,73],\n",
    "                                [27,28,29,9,10,14,56,57,73],\n",
    "                                [27,28,29,9,10,14,23,41,42],\n",
    "                                [15,25,31,50,53,59,62,71],\n",
    "                                [37,45,46,50,53,59,62,71],\n",
    "                                [37,45,46,15,25,31,62,71],\n",
    "                                [37,45,46,15,25,31,50,53]]\n",
    "sync_confirmed_list_test_all = [[9,10,11,14,15,17,18,24,25,31],\n",
    "                               [23,27,28,30,32,33,37,38,45,46],\n",
    "                               [29,39,40,41,42,50,53,59,63,66],\n",
    "                               [56,57,62,67,68,69,71,72,73],\n",
    "                               [32,33,38],\n",
    "                               [11,17,18],\n",
    "                               [24,30,39],\n",
    "                               [40,63,66],\n",
    "                               [67,68,69],\n",
    "                               [27,28,29],\n",
    "                               [9,10,14],\n",
    "                               [23,41,42],\n",
    "                               [56,57,73],\n",
    "                               [37,45,46],\n",
    "                               [15,25,31],\n",
    "                               [50,53,59],\n",
    "                               [59,62,71]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sync_confirmed_list_train = sync_confirmed_list_train_all[0]\n",
    "sync_confirmed_list_test = sync_confirmed_list_test_all[0]\n",
    "\n",
    "Participant_file_name = 'ML-PersRef_ML_Final_dataset/datapoints_rawdata_middle_point_with_sequence_with_building_width_'\n",
    "\n",
    "dataset = []\n",
    "dataset_test = []\n",
    "\n",
    "for z in sync_confirmed_list_train:\n",
    "    dataset.append(pd.read_csv(Participant_file_name + str(z)+'.csv'))\n",
    "\n",
    "for j in sync_confirmed_list_test:\n",
    "    dataset_test.append(pd.read_csv(Participant_file_name + str(j)+'.csv'))\n",
    "\n",
    "dataset = pd.concat(dataset).reset_index(drop=True)\n",
    "dataset_test = pd.concat(dataset_test).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_generator(sync_confirmed_list_train,sync_confirmed_list_test,\n",
    "                   scaler,Participant_file_name,method,time_seq):\n",
    "\n",
    "    dataset = []\n",
    "    dataset_test = []\n",
    "\n",
    "    for z in sync_confirmed_list_train:\n",
    "        dataset.append(pd.read_csv(Participant_file_name + str(z)+'.csv'))\n",
    "\n",
    "    for j in sync_confirmed_list_test:\n",
    "        dataset_test.append(pd.read_csv(Participant_file_name + str(j)+'.csv'))\n",
    "\n",
    "    dataset = pd.concat(dataset).reset_index(drop=True)\n",
    "    dataset_test = pd.concat(dataset_test).reset_index(drop=True)\n",
    "\n",
    "    if time_seq == \"sequenced_model\":\n",
    "        \n",
    "        triggers_to_remove = (((dataset.groupby(['Participant','trigger_no']).size().unstack(fill_value=0)) % sequence_no)!=0)\n",
    "        triggers_to_remove = (triggers_to_remove.reset_index().melt(id_vars=['Participant']).query('value == True')).reset_index(drop=True)\n",
    "        # print(dataset.values.astype(float).shape[0])\n",
    "\n",
    "        dataset['column_to_check'] = dataset.Participant * 1000 + dataset.trigger_no\n",
    "        triggers_to_remove['column_to_check'] = triggers_to_remove.Participant * 1000 + triggers_to_remove.trigger_no\n",
    "\n",
    "        dataset = dataset[~(dataset.column_to_check.isin(triggers_to_remove.column_to_check))]\n",
    "\n",
    "        groups = [dataset for _, dataset in dataset.groupby('column_to_check')]\n",
    "        random.seed(42)\n",
    "        random.shuffle(groups)\n",
    "\n",
    "        dataset = pd.concat(groups).reset_index(drop=True)\n",
    "        dataset.to_csv(\"dataset.csv\")\n",
    "        dataset.drop(columns=['column_to_check'],inplace=True)\n",
    "        # print(dataset.values.astype(float).shape[0])\n",
    "\n",
    "        triggers_to_remove = (((dataset_test.groupby(['Participant','trigger_no']).size().unstack(fill_value=0)) % sequence_no)!=0)\n",
    "        triggers_to_remove = (triggers_to_remove.reset_index().melt(id_vars=['Participant']).query('value == True')).reset_index(drop=True)\n",
    "        # print(dataset_test.values.astype(float).shape[0])\n",
    "\n",
    "        dataset_test['column_to_check'] = dataset_test.Participant * 1000 + dataset_test.trigger_no\n",
    "        triggers_to_remove['column_to_check'] = triggers_to_remove.Participant * 1000 + triggers_to_remove.trigger_no\n",
    "\n",
    "        \n",
    "        dataset_test = dataset_test[~(dataset_test.column_to_check.isin(triggers_to_remove.column_to_check))]\n",
    "#         print(dataset_test.shape)\n",
    "        \n",
    "        groups = [dataset_test for _, dataset_test in dataset_test.groupby('column_to_check')]\n",
    "        random.seed(42)\n",
    "        random.shuffle(groups)\n",
    "            \n",
    "        dataset_test = pd.concat(groups).reset_index(drop=True)\n",
    "        dataset_test.to_csv(\"dataset_test.csv\")\n",
    "        \n",
    "        dataset_test.drop(columns=['column_to_check'],inplace=True)\n",
    "#         print(dataset_test.shape)\n",
    "        # print(dataset_test.values.astype(float).shape[0])\n",
    "\n",
    "        print(((dataset.shape[0]//4)//sequence_no)*sequence_no)\n",
    "        \n",
    "        dataset_train = dataset.iloc[:-(((dataset.shape[0]//4)//sequence_no)*sequence_no),:]\n",
    "        dataset_val = dataset.iloc[-(((dataset.shape[0]//4)//sequence_no)*sequence_no):,:]\n",
    "        dataset_train.reset_index(inplace=True,drop=True)\n",
    "        dataset_val.reset_index(inplace=True,drop=True)\n",
    "\n",
    "    else:\n",
    "        \n",
    "        print(\"non_time_sequenced\")\n",
    "    \n",
    "        dataset_train, dataset_val = train_test_split(dataset, test_size=0.25, random_state=42)\n",
    "        dataset_train.reset_index(inplace=True,drop=True)\n",
    "        dataset_val.reset_index(inplace=True,drop=True)\n",
    "    \n",
    "    if method == \"No_gaze_removal_nor_GT_scaling\":\n",
    "        \n",
    "        xtrain_cross_val = dataset[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x']].values.astype(float)\n",
    "        ytrain_cross_val = dataset[['angle_center']].values.astype(float)\n",
    "        ytrain_cross_val_width = dataset[['Buidling_half_width']].values.astype(float)\n",
    "        \n",
    "        xtrain = dataset_train[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x']].values.astype(float)\n",
    "        ytrain = dataset_train[['angle_center']].values.astype(float)\n",
    "        ytrain_width = dataset_train[['Buidling_half_width']].values.astype(float)\n",
    "\n",
    "        xval = dataset_val[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x']].values.astype(float)\n",
    "        yval = dataset_val[['angle_center']].values.astype(float)\n",
    "        yval_width = dataset_val[['Buidling_half_width']].values.astype(float)\n",
    "\n",
    "        xtest = dataset_test[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x']].values.astype(float)\n",
    "        ytest = dataset_test[['angle_center']].values.astype(float)\n",
    "        ytest_width = dataset_test[['Buidling_half_width']].values.astype(float)\n",
    "\n",
    "        \n",
    "    elif method == \"gaze_removal_and_GT_scaling\":\n",
    "        \n",
    "        dataset.dropna(inplace = True)\n",
    "        dataset_train.dropna(inplace = True)\n",
    "        dataset_val.dropna(inplace = True)\n",
    "        dataset_test.dropna(inplace = True)\n",
    "        \n",
    "        xtrain_cross_val = dataset[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        ytrain_cross_val = dataset[['angle_center_scaled']].values.astype(float)\n",
    "        ytrain_cross_val_width = dataset[['Buidling_half_width_scaled']].values.astype(float)\n",
    "        \n",
    "        xtrain = dataset_train[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        ytrain = dataset_train[['angle_center_scaled']].values.astype(float)\n",
    "        ytrain_width = dataset_train[['Buidling_half_width_scaled']].values.astype(float)\n",
    "        \n",
    "        xtrain = dataset_train[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        ytrain = dataset_train[['angle_center_scaled']].values.astype(float)\n",
    "        ytrain_width = dataset_train[['Buidling_half_width_scaled']].values.astype(float)\n",
    "\n",
    "        xval = dataset_val[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        yval = dataset_val[['angle_center_scaled']].values.astype(float)\n",
    "        yval_width = dataset_val[['Buidling_half_width_scaled']].values.astype(float)\n",
    "\n",
    "        xtest = dataset_test[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        ytest = dataset_test[['angle_center_scaled']].values.astype(float)\n",
    "        ytest_width = dataset_test[['Buidling_half_width_scaled']].values.astype(float)\n",
    "        \n",
    "\n",
    "        \n",
    "    elif method == \"gaze_removal_but_no_GT_scaling\":\n",
    "        \n",
    "        dataset.dropna(inplace = True)\n",
    "        dataset_train.dropna(inplace = True)\n",
    "        dataset_val.dropna(inplace = True)\n",
    "        dataset_test.dropna(inplace = True)\n",
    "        \n",
    "        xtrain_cross_val = dataset[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        ytrain_cross_val = dataset[['angle_center']].values.astype(float)\n",
    "        ytrain_cross_val_width = dataset[['Buidling_half_width']].values.astype(float)\n",
    "        \n",
    "#         print(dataset)\n",
    "        \n",
    "        xtrain = dataset_train[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        ytrain = dataset_train[['angle_center']].values.astype(float)\n",
    "        ytrain_width = dataset_train[['Buidling_half_width']].values.astype(float)\n",
    "        \n",
    "        xval = dataset_val[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        yval = dataset_val[['angle_center']].values.astype(float)\n",
    "        yval_width = dataset_val[['Buidling_half_width']].values.astype(float)\n",
    "\n",
    "        xtest = dataset_test[['fzd_hands0_fingerTip_coordinates_x',\n",
    "               'fzd_hands0_fingerTip_coordinates_y',\n",
    "               'fzd_hands0_fingerTip_coordinates_z', 'eyegaze_new_coordinates_x_without_tablet_&_road_gazes']].values.astype(float)\n",
    "        ytest = dataset_test[['angle_center']].values.astype(float)\n",
    "        ytest_width = dataset_test[['Buidling_half_width']].values.astype(float)\n",
    "\n",
    "    sc_xtrain_cross_val = scaler\n",
    "    xtrain_cross_val = sc_xtrain_cross_val.fit_transform(xtrain_cross_val)\n",
    "    xtrain_cross_val_eye_gaze = xtrain_cross_val[:,-1].reshape(-1, 1)\n",
    "    xtrain_cross_val_pointing = xtrain_cross_val[:,0:3]  \n",
    "        \n",
    "    sc_xtrain = scaler\n",
    "    xtrain = sc_xtrain.fit_transform(xtrain)\n",
    "    xtrain_eye_gaze = xtrain[:,-1].reshape(-1, 1)\n",
    "    xtrain_pointing = xtrain[:,0:3]   \n",
    "    \n",
    "    sc_xval = scaler\n",
    "    xval = sc_xtrain.transform(xval)\n",
    "    xval_eye_gaze = xval[:,-1].reshape(-1, 1)\n",
    "    xval_pointing = xval[:,0:3]\n",
    "    \n",
    "    sc_xtest = scaler\n",
    "    xtest = sc_xtrain.transform(xtest)\n",
    "    xtest_eye_gaze = xtest[:,-1].reshape(-1, 1)\n",
    "    xtest_pointing = xtest[:,0:3]\n",
    "    \n",
    "    \n",
    "#     print(xtrain_cross_val.shape, xtrain.shape, xval.shape, xtest.shape,\n",
    "#       xtrain_cross_val_eye_gaze.shape, xtrain_eye_gaze.shape, xval_eye_gaze.shape, xtest_eye_gaze.shape,\n",
    "#       xtrain_cross_val_pointing.shape, xtrain_pointing.shape, xval_pointing.shape, xtest_pointing.shape,\n",
    "#       ytrain_cross_val.shape, ytrain.shape, yval.shape, ytest.shape)\n",
    "    \n",
    "    if time_seq == \"sequenced_model\":\n",
    "        print(time_seq)\n",
    "    \n",
    "#         print(xtrain_cross_val[0:25,:])\n",
    "#         print(ytrain_cross_val[0:25,:])\n",
    "#         print(\"------\")\n",
    "        \n",
    "        xtrain_cross_val = xtrain_cross_val.reshape(xtrain_cross_val.shape[0]//sequence_no,sequence_no,xtrain_cross_val.shape[1])\n",
    "        xtrain_cross_val_eye_gaze = xtrain_cross_val_eye_gaze.reshape(xtrain_cross_val_eye_gaze.shape[0]//sequence_no,sequence_no,xtrain_cross_val_eye_gaze.shape[1])\n",
    "        xtrain_cross_val_pointing = xtrain_cross_val_pointing.reshape(xtrain_cross_val_pointing.shape[0]//sequence_no,sequence_no,xtrain_cross_val_pointing.shape[1])\n",
    "\n",
    "        xtrain = xtrain.reshape(xtrain.shape[0]//sequence_no,sequence_no,xtrain.shape[1])\n",
    "        xtrain_eye_gaze = xtrain_eye_gaze.reshape(xtrain_eye_gaze.shape[0]//sequence_no,sequence_no,xtrain_eye_gaze.shape[1])\n",
    "        xtrain_pointing = xtrain_pointing.reshape(xtrain_pointing.shape[0]//sequence_no,sequence_no,xtrain_pointing.shape[1])\n",
    "\n",
    "        xval = xval.reshape(xval.shape[0]//sequence_no,sequence_no,xval.shape[1])\n",
    "        xval_eye_gaze = xval_eye_gaze.reshape(xval_eye_gaze.shape[0]//sequence_no,sequence_no,xval_eye_gaze.shape[1])\n",
    "        xval_pointing = xval_pointing.reshape(xval_pointing.shape[0]//sequence_no,sequence_no,xval_pointing.shape[1])\n",
    "\n",
    "        xtest = xtest.reshape(xtest.shape[0]//sequence_no,sequence_no,xtest.shape[1])\n",
    "        xtest_eye_gaze = xtest_eye_gaze.reshape(xtest_eye_gaze.shape[0]//sequence_no,sequence_no,xtest_eye_gaze.shape[1])\n",
    "        xtest_pointing = xtest_pointing.reshape(xtest_pointing.shape[0]//sequence_no,sequence_no,xtest_pointing.shape[1])\n",
    "        \n",
    "#         print(xtrain_cross_val.shape, xtrain.shape, xval.shape, xtest.shape,\n",
    "#               xtrain_cross_val_eye_gaze.shape, xtrain_eye_gaze.shape, xval_eye_gaze.shape, xtest_eye_gaze.shape,\n",
    "#               xtrain_cross_val_pointing.shape, xtrain_pointing.shape, xval_pointing.shape, xtest_pointing.shape,\n",
    "#               ytrain_cross_val.shape, ytrain.shape, yval.shape, ytest.shape)\n",
    "        \n",
    "        ytrain_cross_val = ytrain_cross_val.reshape(ytrain_cross_val.shape[0]//sequence_no,sequence_no,ytrain_cross_val.shape[1]) \n",
    "        ytrain = ytrain.reshape(ytrain.shape[0]//sequence_no,sequence_no,ytrain.shape[1])\n",
    "        yval = yval.reshape(yval.shape[0]//sequence_no,sequence_no,yval.shape[1]) \n",
    "        ytest = ytest.reshape(ytest.shape[0]//sequence_no,sequence_no,ytest.shape[1]) \n",
    "        \n",
    "        ytrain_cross_val_width = ytrain_cross_val_width.reshape(ytrain_cross_val_width.shape[0]//sequence_no,sequence_no,ytrain_cross_val_width.shape[1]) \n",
    "        ytrain_width = ytrain_width.reshape(ytrain_width.shape[0]//sequence_no,sequence_no,ytrain_width.shape[1])\n",
    "        yval_width = yval_width.reshape(yval_width.shape[0]//sequence_no,sequence_no,yval_width.shape[1])\n",
    "        ytest_width = ytest_width.reshape(ytest_width.shape[0]//sequence_no,sequence_no,ytest_width.shape[1])\n",
    "        \n",
    "#         print(xtrain_cross_val.shape, xtrain.shape, xval.shape, xtest.shape,\n",
    "#               xtrain_cross_val_eye_gaze.shape, xtrain_eye_gaze.shape, xval_eye_gaze.shape, xtest_eye_gaze.shape,\n",
    "#               xtrain_cross_val_pointing.shape, xtrain_pointing.shape, xval_pointing.shape, xtest_pointing.shape,\n",
    "#               ytrain_cross_val.shape, ytrain.shape, yval.shape, ytest.shape)\n",
    "        \n",
    "#         print(\"------\")\n",
    "        \n",
    "#         print(xtrain_cross_val[1,:,:])\n",
    "#         print(ytrain_cross_val[1,:,:])\n",
    "#         print(np.mean(ytrain_cross_val[1,:,:]))\n",
    "#         print(np.median(ytrain_cross_val[1,:,:]))\n",
    "#         print(\"median axis\")\n",
    "#         print(np.median(ytrain_cross_val,axis=1))\n",
    "#         print((np.median(ytrain_cross_val,axis=1)).shape)\n",
    "#         print(\"median axis\")\n",
    "        \n",
    "        ytrain_cross_val = np.median(ytrain_cross_val,axis=1)\n",
    "        ytrain = np.median(ytrain,axis=1)\n",
    "        yval = np.median(yval,axis=1)\n",
    "        ytest = np.median(ytest,axis=1)\n",
    "        \n",
    "        ytrain_cross_val_width = np.median(ytrain_cross_val_width,axis=1)\n",
    "        ytrain_width = np.median(ytrain_width,axis=1)\n",
    "        yval_width = np.median(yval_width,axis=1)\n",
    "        ytest_width = np.median(ytest_width,axis=1)\n",
    "        \n",
    "#         print(xtrain_cross_val[1,:,:])\n",
    "#         print(ytrain_cross_val[1,:,:])\n",
    "#         print(np.mean(ytrain_cross_val[1,:,:]))\n",
    "        \n",
    "\n",
    "\n",
    "    print(xtrain_cross_val.shape, xtrain.shape, xval.shape, xtest.shape,\n",
    "      xtrain_cross_val_eye_gaze.shape, xtrain_eye_gaze.shape, xval_eye_gaze.shape, xtest_eye_gaze.shape,\n",
    "      xtrain_cross_val_pointing.shape, xtrain_pointing.shape, xval_pointing.shape, xtest_pointing.shape,\n",
    "      ytrain_cross_val.shape, ytrain.shape, yval.shape, ytest.shape)\n",
    "    \n",
    "    \n",
    "    \n",
    "    return xtrain_cross_val, xtrain, xval, xtest, xtrain_cross_val_eye_gaze, \\\n",
    "            xtrain_eye_gaze, xval_eye_gaze, xtest_eye_gaze,xtrain_cross_val_pointing, \\\n",
    "            xtrain_pointing, xval_pointing, xtest_pointing,ytrain_cross_val, ytrain, yval, ytest, \\\n",
    "            ytrain_cross_val_width, ytrain_width, yval_width, ytest_width"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LSTM_model(sequence_no,dim_num):\n",
    "    \n",
    "    #create model\n",
    "    model = Sequential()\n",
    "    #add model layers\n",
    "    model.add(LSTM(10, input_shape=(sequence_no,dim_num)))\n",
    "    model.add(Dense(1,activation='linear'))\n",
    "    print(model.summary())\n",
    "    # Compile model\n",
    "    # optimizerr = keras.optimizers.SGD(lr=0.001)\n",
    "    model.compile(loss='mse', optimizer='adam',metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "\n",
    "def CNN_model(sequence_no,dim_num):\n",
    "    \n",
    "    #create model\n",
    "    model = Sequential()\n",
    "    #add model layers\n",
    "    model.add(Conv1D(64, kernel_size=3, activation='relu', input_shape=(sequence_no,dim_num)))\n",
    "#     model.add(Conv1D(8, kernel_size=1, activation='relu'))\n",
    "#     model.add(Conv2D(32, kernel_size=3, activation=’relu’))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(1,activation='linear'))\n",
    "    print(model.summary())\n",
    "    # Compile model\n",
    "    # optimizerr = keras.optimizers.SGD(lr=0.001)\n",
    "    model.compile(loss='mse', optimizer='adam',metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "def baseline_model(dim_num):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(32, input_dim=dim_num,  activation='relu', kernel_initializer='uniform'))\n",
    "#     model.add(Dropout(0.5))\n",
    "    model.add(Dense(16, activation='relu', kernel_initializer='uniform'))\n",
    "#     model.add(Dropout(0.5))\n",
    "    model.add(Dense(8, activation='relu', kernel_initializer='uniform'))\n",
    "#     model.add(Dropout(0.5))\n",
    "#     model.add(Dense(4, activation='relu', kernel_initializer='normal'))\n",
    "#     model.add(Dropout(0.5))\n",
    "    model.add(Dense(1,activation='linear'))\n",
    "#     print(model.summary())\n",
    "    # Compile model\n",
    "    # optimizerr = keras.optimizers.SGD(lr=0.001)\n",
    "    model.compile(loss='mse', optimizer='adam',metrics=['mse'])\n",
    "    return model\n",
    "\n",
    "def Main_Network_caller(xtrain1, ytrain1,xval1, yval1,xtest1):\n",
    "    checkpoint_name = 'Weights-best.hdf5' \n",
    "    checkpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 0, save_best_only = True, mode ='auto')\n",
    "    callbacks_list = [checkpoint]\n",
    "\n",
    "    dim_num = xtrain1.shape[-1]\n",
    "    \n",
    "    if type_ == \"FCNN\":\n",
    "        NN_model = baseline_model(dim_num)\n",
    "    elif type_ == \"CNN\":\n",
    "        NN_model = CNN_model(sequence_no,dim_num)\n",
    "    elif type_ == \"LSTM\":\n",
    "        NN_model = LSTM_model(sequence_no,dim_num)\n",
    "    \n",
    "    NN_model.fit(xtrain1, ytrain1, epochs=1000, batch_size=32,\n",
    "                 validation_data = (xval1, yval1), callbacks=callbacks_list, verbose = 0)\n",
    "\n",
    "    # Load wights file of the best model :\n",
    "    wights_file = 'Weights-best.hdf5' # choose the best checkpoint \n",
    "    NN_model.load_weights(wights_file) # load it\n",
    "    NN_model.compile(loss='mse', optimizer='adam', metrics=['mse'])\n",
    "\n",
    "    pred = NN_model.predict(xtest1)\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def results_calculator(true_value,prediction,relative_width,tolerance_degree):\n",
    "    \n",
    "    mse = mean_squared_error(true_value, prediction)\n",
    "    relative_width = abs(true_value-prediction)/(relative_width + tolerance_degree)\n",
    "    return mse, relative_width\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def regressor(type_,eps):\n",
    "    \n",
    "    xtrain_cross_val, xtrain, xval, xtest, xtrain_cross_val_eye_gaze, \\\n",
    "    xtrain_eye_gaze, xval_eye_gaze, xtest_eye_gaze,xtrain_cross_val_pointing, \\\n",
    "    xtrain_pointing, xval_pointing, xtest_pointing,ytrain_cross_val, \\\n",
    "    ytrain, yval, ytest, ytrain_cross_val_width, ytrain_width, \\\n",
    "    yval_width, ytest_width = data_generator(sync_confirmed_list_train,\n",
    "                                         sync_confirmed_list_test,scaler,Participant_file_name,method,time_seq)\n",
    "    \n",
    "    #print('----------------------')\n",
    "    #print(\"----train_cross_val----\")\n",
    "    #print('----------------------')\n",
    "    \n",
    "    if type_ == \"SVR\":\n",
    "        \n",
    "        print(type_)\n",
    "#         print(eps)\n",
    "        regressor = SVR(kernel='rbf', epsilon=eps)\n",
    "\n",
    "        regressor.fit(xtrain_cross_val,ytrain_cross_val)\n",
    "        cross_val_support_vector_shape_fuse.append(regressor.support_vectors_.shape)\n",
    "        pred = cross_val_predict(regressor, xtrain_cross_val, ytrain_cross_val).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytrain_cross_val,pred,ytrain_cross_val_width,tolerance_degree)\n",
    "\n",
    "        cross_val_mse_fuse.append(mse)\n",
    "        cross_val_mean_relative_width_fuse.append(relative_width.mean())\n",
    "        cross_val_std_relative_width_fuse.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xtrain_cross_val_pointing,ytrain_cross_val)\n",
    "        cross_val_support_vector_shape_pointing.append(regressor.support_vectors_.shape)\n",
    "        pred = cross_val_predict(regressor, xtrain_cross_val_pointing, ytrain_cross_val).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytrain_cross_val,pred,ytrain_cross_val_width,tolerance_degree)\n",
    "\n",
    "        cross_val_mse_pointing.append(mse)\n",
    "        cross_val_mean_relative_width_pointing.append(relative_width.mean())\n",
    "        cross_val_std_relative_width_pointing.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xtrain_cross_val_eye_gaze,ytrain_cross_val)\n",
    "        cross_val_support_vector_shape_gaze.append(regressor.support_vectors_.shape)\n",
    "        pred = cross_val_predict(regressor, xtrain_cross_val_eye_gaze, ytrain_cross_val).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytrain_cross_val,pred,ytrain_cross_val_width,tolerance_degree)\n",
    "\n",
    "        cross_val_mse_gaze.append(mse)\n",
    "        cross_val_mean_relative_width_gaze.append(relative_width.mean())\n",
    "        cross_val_std_relative_width_gaze.append(relative_width.std())\n",
    "\n",
    "        #print('=========================')\n",
    "        #print('------------------------')\n",
    "        #print(\"---------train----------\")\n",
    "        #print('------------------------')\n",
    "\n",
    "        regressor.fit(xtrain,ytrain)\n",
    "        train_support_vector_shape_fuse.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xtrain).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytrain,pred,ytrain_width,tolerance_degree)\n",
    "        xtrain_fuse = regressor.support_vectors_\n",
    "        ytrain_fuse = ytrain[regressor.support_]\n",
    "\n",
    "        train_mse_fuse.append(mse)\n",
    "        train_mean_relative_width_fuse.append(relative_width.mean())\n",
    "        train_std_relative_width_fuse.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xtrain_pointing,ytrain)\n",
    "        train_support_vector_shape_pointing.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xtrain_pointing).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytrain,pred,ytrain_width,tolerance_degree)\n",
    "\n",
    "        xtrain_pointing = regressor.support_vectors_\n",
    "        ytrain_pointing = ytrain[regressor.support_]\n",
    "\n",
    "        train_mse_pointing.append(mse)\n",
    "        train_mean_relative_width_pointing.append(relative_width.mean())\n",
    "        train_std_relative_width_pointing.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xtrain_eye_gaze,ytrain)\n",
    "        train_support_vector_shape_gaze.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xtrain_eye_gaze).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytrain,pred,ytrain_width,tolerance_degree)\n",
    "\n",
    "        xtrain_eye_gaze = regressor.support_vectors_\n",
    "        ytrain_eye_gaze = ytrain[regressor.support_]\n",
    "\n",
    "        train_mse_gaze.append(mse)\n",
    "        train_mean_relative_width_gaze.append(relative_width.mean())\n",
    "        train_std_relative_width_gaze.append(relative_width.std())\n",
    "\n",
    "        #print('----------------------')\n",
    "        #print(\"---------val----------\")\n",
    "        #print('----------------------')\n",
    "\n",
    "        regressor.fit(xval,yval)\n",
    "        val_support_vector_shape_fuse.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xval).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(yval,pred,yval_width,tolerance_degree)\n",
    "\n",
    "        val_mse_fuse.append(mse)\n",
    "        val_mean_relative_width_fuse.append(relative_width.mean())\n",
    "        val_std_relative_width_fuse.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xval_pointing,yval)\n",
    "        val_support_vector_shape_pointing.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xval_pointing).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(yval,pred,yval_width,tolerance_degree)\n",
    "\n",
    "        val_mse_pointing.append(mse)\n",
    "        val_mean_relative_width_pointing.append(relative_width.mean())\n",
    "        val_std_relative_width_pointing.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xval_eye_gaze,yval)\n",
    "        val_support_vector_shape_gaze.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xval_eye_gaze).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(yval,pred,yval_width,tolerance_degree)\n",
    "\n",
    "        val_mse_gaze.append(mse)\n",
    "        val_mean_relative_width_gaze.append(relative_width.mean())\n",
    "        val_std_relative_width_gaze.append(relative_width.std())\n",
    "\n",
    "        #print('------------------------')\n",
    "        #print(\"----------test----------\")\n",
    "        #print('------------------------')\n",
    "\n",
    "        regressor.fit(xtrain_fuse,ytrain_fuse)\n",
    "        test_support_vector_shape_fuse.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xtest).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytest,pred,ytest_width,tolerance_degree)\n",
    "\n",
    "        test_mse_fuse.append(mse)\n",
    "        test_mean_relative_width_fuse.append(relative_width.mean())\n",
    "        test_std_relative_width_fuse.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xtrain_pointing,ytrain_pointing)\n",
    "        test_support_vector_shape_pointing.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xtest_pointing).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytest,pred,ytest_width,tolerance_degree)\n",
    "\n",
    "        test_mse_pointing.append(mse)\n",
    "        test_mean_relative_width_pointing.append(relative_width.mean())\n",
    "        test_std_relative_width_pointing.append(relative_width.std())\n",
    "\n",
    "        regressor.fit(xtrain_eye_gaze,ytrain_eye_gaze)\n",
    "        test_support_vector_shape_gaze.append(regressor.support_vectors_.shape)\n",
    "        pred=regressor.predict(xtest_eye_gaze).reshape(-1, 1)\n",
    "        mse, relative_width = results_calculator(ytest,pred,ytest_width,tolerance_degree)\n",
    "\n",
    "        test_mse_gaze.append(mse)\n",
    "        test_mean_relative_width_gaze.append(relative_width.mean())\n",
    "        test_std_relative_width_gaze.append(relative_width.std())\n",
    "        \n",
    "#     elif type_ == \"FCNN\":\n",
    "    else:\n",
    "        \n",
    "        print(type_)\n",
    "        \n",
    "        pred = Main_Network_caller(xtrain, ytrain,xval, yval,xtest)\n",
    "        mse, relative_width = results_calculator(ytest,pred,ytest_width,tolerance_degree)\n",
    "#         print('===================')\n",
    "#         print('Test results fuse')\n",
    "#         print('-------------------')\n",
    "#         print(mse)\n",
    "#         print(relative_width.mean())\n",
    "#         print(relative_width.std())\n",
    "        \n",
    "        FCNN_test_mse_fuse.append(mse)\n",
    "        FCNN_test_mean_relative_width_fuse.append(relative_width.mean())\n",
    "        FCNN_test_std_relative_width_fuse.append(relative_width.std())\n",
    "        \n",
    "        pred = Main_Network_caller(xtrain_pointing, ytrain,xval_pointing, yval,xtest_pointing)\n",
    "        mse, relative_width = results_calculator(ytest,pred,ytest_width,tolerance_degree)\n",
    "#         print('===================')\n",
    "#         print('Test results pointing')\n",
    "#         print('-------------------')\n",
    "#         print(mse)\n",
    "#         print(relative_width.mean())\n",
    "#         print(relative_width.std())\n",
    "        \n",
    "        FCNN_test_mse_pointing.append(mse)\n",
    "        FCNN_test_mean_relative_width_pointing.append(relative_width.mean())\n",
    "        FCNN_test_std_relative_width_pointing.append(relative_width.std())\n",
    "        \n",
    "        pred = Main_Network_caller(xtrain_eye_gaze, ytrain,xval_eye_gaze, yval,xtest_eye_gaze)\n",
    "        mse, relative_width = results_calculator(ytest,pred,ytest_width,tolerance_degree)\n",
    "#         print('===================')\n",
    "#         print('Test results gaze')\n",
    "#         print('-------------------')\n",
    "#         print(mse)\n",
    "#         print(relative_width.mean())\n",
    "#         print(relative_width.std())\n",
    "        \n",
    "        FCNN_test_mse_gaze.append(mse)\n",
    "        FCNN_test_mean_relative_width_gaze.append(relative_width.mean())\n",
    "        FCNN_test_std_relative_width_gaze.append(relative_width.std())\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(1):\n",
    "for i in range(len(sync_confirmed_list_train_all)):\n",
    "    print(i)\n",
    "    sequence_no = 7\n",
    "    \n",
    "    sync_confirmed_list_train = sync_confirmed_list_train_all[i]\n",
    "    sync_confirmed_list_test = sync_confirmed_list_test_all[i]\n",
    "#     print(sync_confirmed_list_train)\n",
    "#     print(sync_confirmed_list_test)\n",
    "    \n",
    "    # eps = 12\n",
    "    scaler = StandardScaler() #MinMaxScaler()\n",
    "    tolerance_degree = 0\n",
    "\n",
    "#     Participant_file_name = 'ML-PersRef_ML_Final_dataset/datapoints_rawdata_first_middle_points_with_sequence_' + str(sequence_no) + '_with_building_width_'\n",
    "    Participant_file_name = 'ML-PersRef_ML_Final_dataset/datapoints_rawdata_first_middle_points_with_building_width_'\n",
    "    method = \"No_gaze_removal_nor_GT_scaling\"\n",
    "#     method = \"gaze_removal_but_no_GT_scaling\"\n",
    "\n",
    "    \n",
    "#     time_seq = \"sequenced_model\"\n",
    "    time_seq = 0\n",
    "\n",
    "\n",
    "    type_ = \"SVR\"\n",
    "    regressor(type_,2)\n",
    "\n",
    "#     type_ = \"FCNN\"\n",
    "#     regressor(type_,0)\n",
    "    \n",
    "#     type_ = \"CNN\"\n",
    "#     regressor(type_,0)\n",
    "    \n",
    "#     type_ = \"LSTM\"\n",
    "#     regressor(type_,0)\n",
    "\n",
    "    # for eps in np.arange(0, 7, 1):\n",
    "    #     try:\n",
    "    #         regressor(\"SVR\",sync_confirmed_list_train,sync_confirmed_list_test,eps,scaler,tolerance_degree,Participant_file_name,method)\n",
    "    #     except:\n",
    "    #         break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "print(test_support_vector_shape_fuse)\n",
    "print(test_mse_fuse)\n",
    "print(test_mean_relative_width_fuse)\n",
    "print(test_std_relative_width_fuse)\n",
    "print(test_support_vector_shape_pointing)\n",
    "print(test_mse_pointing)\n",
    "print(test_mean_relative_width_pointing)\n",
    "print(test_std_relative_width_pointing)\n",
    "print(test_support_vector_shape_gaze)\n",
    "print(test_mse_gaze)\n",
    "print(test_mean_relative_width_gaze)\n",
    "print(test_std_relative_width_gaze)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}