notebooks/models_postprocessing/jtree/2_encode_molecules.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get encoded atoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "import sys\n",
    "sys.path.append('../../../scripts/SOTA/jtree/')\n",
    "# sys.path.append('../../../../icml18-jtnn/jtnn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "VOCAB_PATH = '../../../data/raw/baselines/jtree/vocab.txt'\n",
    "MODEL_PATH = '../../../data/raw/baselines/jtree'\n",
    "DATASET_PATH = '../../../data/3_final_data/split_data'\n",
    "\n",
    "RAW_PATH = '../../../data/raw/baselines/jtree'\n",
    "\n",
    "SMILES_COLUMN = 'smiles'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import math, random, sys\n",
    "from optparse import OptionParser\n",
    "from collections import deque\n",
    "\n",
    "import rdkit\n",
    "import rdkit.Chem as Chem\n",
    "from mol_tree import MolTree, Vocab\n",
    "import numpy as np\n",
    "from jtnn_enc import JTNNEncoder\n",
    "from mpn_more_atom_feats import MPN,  mol2graph\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = [x.strip(\"\\r\\n \") for x in open(VOCAB_PATH)] \n",
    "vocab = Vocab(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size = int(450)\n",
    "latent_size = int(56)\n",
    "depth = int(3)\n",
    "stereo = True if int(1) == 1 else False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mol/anaconda3/envs/jtree/lib/python2.7/site-packages/torch/nn/_reduction.py:49: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.\n",
      "  warnings.warn(warning.format(ret))\n"
     ]
    }
   ],
   "source": [
    "from jtnn_vae import JTNNVAE\n",
    "model = JTNNVAE(vocab, hidden_size, latent_size, depth, stereo=stereo)\n",
    "model.load_state_dict(torch.load(os.path.join(MODEL_PATH, 'MPNVAE-h450-L56-d3-beta0.001/model.iter-4')))\n",
    "model = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_latent_representation_dataset(df, SMILES_COLUMN, VALUE_COLUMN):\n",
    "    import numpy as np    \n",
    "    vectors = []\n",
    "    broken_smiles = {}\n",
    "    for smiles in tqdm(df[SMILES_COLUMN].values):\n",
    "        try:\n",
    "            latent_representation = model.encode_latent_mean([smiles])\n",
    "        except (KeyError, RuntimeError), e:\n",
    "            broken_smiles[smiles]=e\n",
    "            continue\n",
    "        vectors.append(latent_representation.cpu().detach().numpy())\n",
    "    X = np.array(vectors)\n",
    "    y = df[~df[SMILES_COLUMN].isin(list(broken_smiles.keys()))][VALUE_COLUMN].values\n",
    "    assert len(X)==len(y)\n",
    "    return X, y, broken_smiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_prefix = ['logp_wo_logp_json_wo_averaging', 'logd_Lip_wo_averaging']\n",
    "VALUE_COLS = ['logP', 'logD']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9581/9581 [10:03<00:00, 15.87it/s]\n",
      "100%|██████████| 2054/2054 [02:21<00:00, 14.55it/s]\n",
      "100%|██████████| 2053/2053 [02:09<00:00, 15.90it/s]\n",
      "100%|██████████| 2916/2916 [02:59<00:00, 16.24it/s]\n",
      "100%|██████████| 625/625 [00:41<00:00, 14.95it/s]\n",
      "100%|██████████| 625/625 [00:43<00:00, 14.46it/s]\n"
     ]
    }
   ],
   "source": [
    "X_trains = []\n",
    "y_trains = []\n",
    "trains_errs = []\n",
    "X_vals = []\n",
    "y_vals = []\n",
    "vals_errs = []\n",
    "X_tests = []\n",
    "y_tests = []\n",
    "tests_errs = []\n",
    "\n",
    "for VALUE_COLUMN,dataset_prefix in zip(VALUE_COLS, datasets_prefix):\n",
    "    dataset_train = pd.read_csv(os.path.join(DATASET_PATH, dataset_prefix+'_train.csv'), index_col=0)\n",
    "    dataset_val = pd.read_csv(os.path.join(DATASET_PATH, dataset_prefix+'_validation.csv'), index_col=0)\n",
    "    dataset_test = pd.read_csv(os.path.join(DATASET_PATH, dataset_prefix+'_test.csv'), index_col=0)\n",
    "    X_train, y_train, train_errs = create_latent_representation_dataset(dataset_train, SMILES_COLUMN, VALUE_COLUMN)\n",
    "    X_trains.append(X_train)\n",
    "    y_trains.append(y_train)\n",
    "    trains_errs+=train_errs\n",
    "\n",
    "    X_val, y_val, val_errs = create_latent_representation_dataset(dataset_val, SMILES_COLUMN, VALUE_COLUMN)\n",
    "    X_vals.append(X_val)\n",
    "    y_vals.append(y_val)\n",
    "    vals_errs+=val_errs\n",
    "    \n",
    "    X_test, y_test, test_errs = create_latent_representation_dataset(dataset_test, SMILES_COLUMN, VALUE_COLUMN)\n",
    "    X_tests.append(X_test)\n",
    "    y_tests.append(y_test)\n",
    "    tests_errs+=test_errs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(RAW_PATH,'X_train.npy'), 'wb') as f:\n",
    "    np.save(f,np.concatenate(X_trains, axis = 0))\n",
    "with open(os.path.join(RAW_PATH,'y_train.npy'), 'wb') as f:\n",
    "    np.save(f,np.concatenate(y_trains, axis = 0))  \n",
    "    \n",
    "with open(os.path.join(RAW_PATH,'X_val.npy'), 'wb') as f:\n",
    "    np.save(f,np.concatenate(X_vals, axis = 0))\n",
    "with open(os.path.join(RAW_PATH,'y_val.npy'), 'wb') as f:\n",
    "    np.save(f,np.concatenate(y_vals, axis = 0))\n",
    "\n",
    "with open(os.path.join(RAW_PATH,'X_test.npy'), 'wb') as f:\n",
    "    np.save(f,np.concatenate(X_tests, axis = 0))\n",
    "with open(os.path.join(RAW_PATH,'y_test.npy'), 'wb') as f:\n",
    "    np.save(f,np.concatenate(y_tests, axis = 0))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(RAW_PATH,'train_errs.txt'), 'w') as f:\n",
    "    f.write('\\n'.join(trains_errs))\n",
    "with open(os.path.join(RAW_PATH,'val_errs.txt'), 'w') as f:\n",
    "    f.write('\\n'.join(vals_errs))\n",
    "with open(os.path.join(RAW_PATH,'test_errs.json'), 'w') as f:\n",
    "    f.write('\\n'.join(tests_errs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train model based on pretrained vectors "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pytorch NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from sklearn.metrics import roc_auc_score, r2_score, mean_squared_error\n",
    "\n",
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "RAW_PATH = '../../../data/raw/baselines/jtree'\n",
    "\n",
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "fname = os.path.join(RAW_PATH, 'logs', 'exp_0')\n",
    "os.makedirs(fname)\n",
    "writer = SummaryWriter(fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorboardX import SummaryWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopping:\n",
    "    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n",
    "    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt'):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            patience (int): How long to wait after last time validation loss improved.\n",
    "                            Default: 7\n",
    "            verbose (bool): If True, prints a message for each validation loss improvement.\n",
    "                            Default: False\n",
    "            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n",
    "                            Default: 0\n",
    "            path (str): Path for the checkpoint to be saved to.\n",
    "                            Default: 'checkpoint.pt'\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.verbose = verbose\n",
    "        self.counter = 0\n",
    "        self.best_score = None\n",
    "        self.early_stop = False\n",
    "        self.val_loss_min = np.Inf\n",
    "        self.delta = delta\n",
    "        self.path = path\n",
    "\n",
    "    def __call__(self, val_loss, model):\n",
    "\n",
    "        score = -val_loss\n",
    "\n",
    "        if self.best_score is None:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(val_loss, model)\n",
    "        elif score < self.best_score + self.delta:\n",
    "            self.counter += 1\n",
    "            print('EarlyStopping counter:',self.counter,' out of ',self.patience)\n",
    "            if self.counter >= self.patience:\n",
    "                self.early_stop = True\n",
    "        else:\n",
    "            self.best_score = score\n",
    "            self.save_checkpoint(val_loss, model)\n",
    "            self.counter = 0\n",
    "\n",
    "    def save_checkpoint(self, val_loss, model):\n",
    "        '''Saves model when validation loss decrease.'''\n",
    "        if self.verbose:\n",
    "            print('Validation loss decreased (', round(self.val_loss_min, 6), '-->', round(val_loss, 6),').  Saving model ...')\n",
    "        torch.save(model.state_dict(), self.path)\n",
    "        self.val_loss_min = val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        In the constructor we instantiate two nn.Linear modules and assign them as\n",
    "        member variables.\n",
    "        \"\"\"\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(D_in, H)\n",
    "        self.linear2 = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        In the forward function we accept a Tensor of input data and we must return\n",
    "        a Tensor of output data. We can use Modules defined in the constructor as\n",
    "        well as arbitrary operators on Tensors.\n",
    "        \"\"\"\n",
    "        h_relu = self.linear1(x).clamp(min=0)\n",
    "        y_pred = self.linear2(h_relu)\n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, loader, optimizer, device):\n",
    "    model.train()\n",
    "\n",
    "    for step, batch in enumerate(tqdm(loader, desc=\"Iteration\")):\n",
    "        x, y = batch\n",
    "        x = x.to(device)\n",
    "        y = y.to(device)\n",
    "        y_pred = model(x)\n",
    "\n",
    "        #Whether y is non-null or not.\n",
    "        is_valid = y**2 > 0\n",
    "        #Loss matrix\n",
    "        loss = criterion(y_pred, y)\n",
    "        #loss matrix after removing null target\n",
    "        # loss_mat = torch.where(is_valid, loss_mat, torch.zeros(loss_mat.shape).to(loss_mat.device).to(loss_mat.dtype))\n",
    "        #\n",
    "        optimizer.zero_grad()\n",
    "        # loss = torch.sum(loss_mat)/torch.sum(is_valid)\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "\n",
    "def eval(model, loader, scaler, device,  train = False):\n",
    "    model.eval()\n",
    "    y_true = []\n",
    "    y_scores = []\n",
    "\n",
    "    for step, batch in enumerate(tqdm(loader, desc=\"Iteration\")):\n",
    "        x, y = batch\n",
    "        x = x.to(device)\n",
    "        y = y.to(device)\n",
    "        \n",
    "\n",
    "        with torch.no_grad():\n",
    "            pred = model(x)\n",
    "\n",
    "        y_true.append(y.squeeze())\n",
    "        if train:\n",
    "            y_scores.append(pred.squeeze())\n",
    "        else:\n",
    "            y_scores.append(torch.Tensor(scaler.inverse_transform(pred.squeeze().cpu())))\n",
    "\n",
    "    y_true = torch.cat(y_true, dim = 0).cpu().numpy()\n",
    "    y_scores = torch.cat(y_scores, dim = 0).cpu().numpy()\n",
    "\n",
    "    r2 = r2_score(y_true, y_scores)\n",
    "    rmse = mean_squared_error(y_true, y_scores)**0.5\n",
    "\n",
    "    return r2, rmse #y_true.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "D_in, H, D_out = 56, 100, 1\n",
    "\n",
    "device = torch.device(\"cuda:0\")\n",
    "\n",
    "# Construct our model by instantiating the class defined above\n",
    "model = TwoLayerNet(D_in, H, D_out)\n",
    "model = model.to(device)\n",
    "\n",
    "\n",
    "# Construct our loss function and an Optimizer. The call to model.parameters()\n",
    "# in the SGD constructor will contain the learnable parameters of the two\n",
    "# nn.Linear modules which are members of the model.\n",
    "criterion = torch.nn.MSELoss(reduction='mean')\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(RAW_PATH,'X_train.npy'), 'rb') as f:\n",
    "    X_train = np.load(f)\n",
    "with open(os.path.join(RAW_PATH,'y_train.npy'), 'rb') as f:\n",
    "    y_train = np.load(f)\n",
    "    \n",
    "X_train = torch.Tensor(X_train) # transform to torch tensor\n",
    "# y_train = torch.Tensor(y_train)\n",
    "\n",
    "y_train_transformed = torch.Tensor(scaler.fit_transform(y_train.reshape(-1, 1)).reshape(-1))\n",
    "\n",
    "train_loader = DataLoader(TensorDataset(X_train,y_train_transformed), batch_size = 32) # create your dataloader\n",
    "    \n",
    "with open(os.path.join(RAW_PATH,'X_val.npy'), 'rb') as f:\n",
    "    X_val = np.load(f)\n",
    "with open(os.path.join(RAW_PATH,'y_val.npy'), 'rb') as f:\n",
    "    y_val = np.load(f)\n",
    "    \n",
    "X_val = torch.Tensor(X_val) # transform to torch tensor\n",
    "y_val = torch.Tensor(y_val)\n",
    "\n",
    "val_loader = DataLoader(TensorDataset(X_val, y_val), batch_size = 32) # create your dataloader\n",
    "\n",
    "with open(os.path.join(RAW_PATH,'X_test.npy'), 'rb') as f:\n",
    "    X_test = np.load(f)\n",
    "with open(os.path.join(RAW_PATH,'y_test.npy'), 'rb') as f:\n",
    "    y_test = np.load(f)\n",
    "    \n",
    "X_test = torch.Tensor(X_test) # transform to torch tensor\n",
    "y_test = torch.Tensor(y_test)\n",
    "\n",
    "test_loader = DataLoader(TensorDataset(X_test, y_test), batch_size = 32) # create your dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "early_stopping = EarlyStopping(patience=50, verbose=True, path=os.path.join(fname, 'chkpnt' + '.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration:  13%|█▎        | 38/295 [00:00<00:00, 361.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.40it/s]\n",
      "Iteration:  33%|███▎      | 98/295 [00:00<00:00, 973.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 932.10it/s] \n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 566.57it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 565.53it/s]\n",
      "Iteration:  12%|█▏        | 34/295 [00:00<00:00, 336.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.048476\n",
      "train rmse: 0.975461\n",
      " val r2: 0.037040\n",
      " val rmse: 1.789631\n",
      "test r2: 0.056633\n",
      "test rmse: 1.773488\n",
      "('Validation loss decreased (', inf, '-->', 1.789631, ').  Saving model ...')\n",
      "\n",
      "====epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 340.71it/s]\n",
      "Iteration:  35%|███▌      | 104/295 [00:00<00:00, 1039.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1107.84it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 978.64it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 987.18it/s]\n",
      "Iteration:  13%|█▎        | 38/295 [00:00<00:00, 372.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.053232\n",
      "train rmse: 0.973020\n",
      " val r2: 0.041596\n",
      " val rmse: 1.785392\n",
      "test r2: 0.059655\n",
      "test rmse: 1.770644\n",
      "('Validation loss decreased (', 1.789631, '-->', 1.785392, ').  Saving model ...')\n",
      "\n",
      "====epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.26it/s]\n",
      "Iteration:  42%|████▏     | 124/295 [00:00<00:00, 1238.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1117.00it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1018.49it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 561.71it/s]\n",
      "Iteration:  13%|█▎        | 37/295 [00:00<00:00, 366.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054364\n",
      "train rmse: 0.972438\n",
      " val r2: 0.042513\n",
      " val rmse: 1.784537\n",
      "test r2: 0.059099\n",
      "test rmse: 1.771168\n",
      "('Validation loss decreased (', 1.785392, '-->', 1.784537, ').  Saving model ...')\n",
      "\n",
      "====epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.64it/s]\n",
      "Iteration:  43%|████▎     | 127/295 [00:00<00:00, 1251.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1113.30it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 565.94it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 563.98it/s]\n",
      "Iteration:  13%|█▎        | 37/295 [00:00<00:00, 362.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.053494\n",
      "train rmse: 0.972886\n",
      " val r2: 0.040913\n",
      " val rmse: 1.786027\n",
      "test r2: 0.056788\n",
      "test rmse: 1.773342\n",
      "('EarlyStopping counter:', 1, ' out of ', 50)\n",
      "\n",
      "====epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.19it/s]\n",
      "Iteration:  44%|████▍     | 131/295 [00:00<00:00, 1309.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1115.77it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 558.35it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 564.55it/s]\n",
      "Iteration:  12%|█▏        | 36/295 [00:00<00:00, 359.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.053537\n",
      "train rmse: 0.972863\n",
      " val r2: 0.040399\n",
      " val rmse: 1.786506\n",
      "test r2: 0.055906\n",
      "test rmse: 1.774171\n",
      "('EarlyStopping counter:', 2, ' out of ', 50)\n",
      "\n",
      "====epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 360.47it/s]\n",
      "Iteration:  43%|████▎     | 126/295 [00:00<00:00, 1251.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1111.95it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 561.21it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 569.85it/s]\n",
      "Iteration:  13%|█▎        | 39/295 [00:00<00:00, 383.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054282\n",
      "train rmse: 0.972480\n",
      " val r2: 0.040908\n",
      " val rmse: 1.786033\n",
      "test r2: 0.055922\n",
      "test rmse: 1.774156\n",
      "('EarlyStopping counter:', 3, ' out of ', 50)\n",
      "\n",
      "====epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 362.27it/s]\n",
      "Iteration:  38%|███▊      | 112/295 [00:00<00:00, 1115.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 936.40it/s] \n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 560.35it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 566.74it/s]\n",
      "Iteration:  14%|█▍        | 41/295 [00:00<00:00, 406.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054010\n",
      "train rmse: 0.972620\n",
      " val r2: 0.040521\n",
      " val rmse: 1.786393\n",
      "test r2: 0.055221\n",
      "test rmse: 1.774814\n",
      "('EarlyStopping counter:', 4, ' out of ', 50)\n",
      "\n",
      "====epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.88it/s]\n",
      "Iteration:  40%|████      | 119/295 [00:00<00:00, 1173.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1115.61it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1017.86it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 568.12it/s]\n",
      "Iteration:  14%|█▎        | 40/295 [00:00<00:00, 399.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054677\n",
      "train rmse: 0.972277\n",
      " val r2: 0.041272\n",
      " val rmse: 1.785693\n",
      "test r2: 0.055894\n",
      "test rmse: 1.774182\n",
      "('EarlyStopping counter:', 5, ' out of ', 50)\n",
      "\n",
      "====epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 362.02it/s]\n",
      "Iteration:  42%|████▏     | 124/295 [00:00<00:00, 1235.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1118.71it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 996.26it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 549.86it/s]\n",
      "Iteration:  12%|█▏        | 36/295 [00:00<00:00, 350.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054519\n",
      "train rmse: 0.972358\n",
      " val r2: 0.040989\n",
      " val rmse: 1.785957\n",
      "test r2: 0.055353\n",
      "test rmse: 1.774691\n",
      "('EarlyStopping counter:', 6, ' out of ', 50)\n",
      "\n",
      "====epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 362.66it/s]\n",
      "Iteration:  44%|████▍     | 131/295 [00:00<00:00, 1304.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1118.93it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 973.12it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1016.07it/s]\n",
      "Iteration:  14%|█▍        | 42/295 [00:00<00:00, 418.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054721\n",
      "train rmse: 0.972255\n",
      " val r2: 0.041359\n",
      " val rmse: 1.785613\n",
      "test r2: 0.055256\n",
      "test rmse: 1.774781\n",
      "('EarlyStopping counter:', 7, ' out of ', 50)\n",
      "\n",
      "====epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 362.89it/s]\n",
      "Iteration:  38%|███▊      | 112/295 [00:00<00:00, 1114.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1117.44it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 550.86it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1014.92it/s]\n",
      "Iteration:  13%|█▎        | 38/295 [00:00<00:00, 372.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054269\n",
      "train rmse: 0.972487\n",
      " val r2: 0.040955\n",
      " val rmse: 1.785989\n",
      "test r2: 0.054563\n",
      "test rmse: 1.775432\n",
      "('EarlyStopping counter:', 8, ' out of ', 50)\n",
      "\n",
      "====epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.78it/s]\n",
      "Iteration:  40%|████      | 118/295 [00:00<00:00, 1168.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 937.69it/s] \n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 560.96it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1005.91it/s]\n",
      "Iteration:  13%|█▎        | 39/295 [00:00<00:00, 389.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.053570\n",
      "train rmse: 0.972846\n",
      " val r2: 0.040230\n",
      " val rmse: 1.786664\n",
      "test r2: 0.053744\n",
      "test rmse: 1.776201\n",
      "('EarlyStopping counter:', 9, ' out of ', 50)\n",
      "\n",
      "====epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.73it/s]\n",
      "Iteration:  37%|███▋      | 110/295 [00:00<00:00, 1083.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 938.26it/s] \n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 559.59it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 565.12it/s]\n",
      "Iteration:  13%|█▎        | 39/295 [00:00<00:00, 383.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.053809\n",
      "train rmse: 0.972723\n",
      " val r2: 0.040348\n",
      " val rmse: 1.786554\n",
      "test r2: 0.053835\n",
      "test rmse: 1.776116\n",
      "('EarlyStopping counter:', 10, ' out of ', 50)\n",
      "\n",
      "====epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.94it/s]\n",
      "Iteration:  39%|███▉      | 115/295 [00:00<00:00, 1148.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 935.59it/s] \n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 566.36it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 557.71it/s]\n",
      "Iteration:  13%|█▎        | 38/295 [00:00<00:00, 369.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054004\n",
      "train rmse: 0.972623\n",
      " val r2: 0.040386\n",
      " val rmse: 1.786519\n",
      "test r2: 0.054264\n",
      "test rmse: 1.775713\n",
      "('EarlyStopping counter:', 11, ' out of ', 50)\n",
      "\n",
      "====epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 479.81it/s]\n",
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1796.17it/s]\n",
      "Iteration:   0%|          | 0/64 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1025.74it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1010.77it/s]\n",
      "Iteration:  16%|█▋        | 48/295 [00:00<00:00, 472.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.055108\n",
      "train rmse: 0.972056\n",
      " val r2: 0.041317\n",
      " val rmse: 1.785652\n",
      "test r2: 0.055277\n",
      "test rmse: 1.774762\n",
      "('EarlyStopping counter:', 12, ' out of ', 50)\n",
      "\n",
      "====epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 413.15it/s]\n",
      "Iteration:  41%|████      | 120/295 [00:00<00:00, 1174.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1116.25it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 977.87it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1019.89it/s]\n",
      "Iteration:  14%|█▎        | 40/295 [00:00<00:00, 391.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054054\n",
      "train rmse: 0.972598\n",
      " val r2: 0.040070\n",
      " val rmse: 1.786813\n",
      "test r2: 0.054168\n",
      "test rmse: 1.775803\n",
      "('EarlyStopping counter:', 13, ' out of ', 50)\n",
      "\n",
      "====epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 362.17it/s]\n",
      "Iteration:  44%|████▍     | 131/295 [00:00<00:00, 1306.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1121.47it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1004.16it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1008.78it/s]\n",
      "Iteration:  12%|█▏        | 36/295 [00:00<00:00, 358.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054293\n",
      "train rmse: 0.972475\n",
      " val r2: 0.040103\n",
      " val rmse: 1.786782\n",
      "test r2: 0.054466\n",
      "test rmse: 1.775523\n",
      "('EarlyStopping counter:', 14, ' out of ', 50)\n",
      "\n",
      "====epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 362.04it/s]\n",
      "Iteration:  46%|████▌     | 136/295 [00:00<00:00, 1341.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1117.35it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 565.68it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 564.46it/s]\n",
      "Iteration:  14%|█▍        | 41/295 [00:00<00:00, 408.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.054830\n",
      "train rmse: 0.972198\n",
      " val r2: 0.040217\n",
      " val rmse: 1.786676\n",
      "test r2: 0.054911\n",
      "test rmse: 1.775105\n",
      "('EarlyStopping counter:', 15, ' out of ', 50)\n",
      "\n",
      "====epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.36it/s]\n",
      "Iteration:  43%|████▎     | 126/295 [00:00<00:00, 1243.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1117.99it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 563.29it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1016.59it/s]\n",
      "Iteration:  13%|█▎        | 38/295 [00:00<00:00, 372.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.055903\n",
      "train rmse: 0.971646\n",
      " val r2: 0.041000\n",
      " val rmse: 1.785947\n",
      "test r2: 0.056139\n",
      "test rmse: 1.773952\n",
      "('EarlyStopping counter:', 16, ' out of ', 50)\n",
      "\n",
      "====epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.84it/s]\n",
      "Iteration:  38%|███▊      | 113/295 [00:00<00:00, 1125.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1115.67it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 566.01it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 567.53it/s]\n",
      "Iteration:  13%|█▎        | 39/295 [00:00<00:00, 374.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.055210\n",
      "train rmse: 0.972003\n",
      " val r2: 0.040161\n",
      " val rmse: 1.786728\n",
      "test r2: 0.055315\n",
      "test rmse: 1.774726\n",
      "('EarlyStopping counter:', 17, ' out of ', 50)\n",
      "\n",
      "====epoch 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 385.96it/s]\n",
      "Iteration:  47%|████▋     | 138/295 [00:00<00:00, 1378.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1117.04it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1013.59it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 566.44it/s]\n",
      "Iteration:  13%|█▎        | 38/295 [00:00<00:00, 366.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.055070\n",
      "train rmse: 0.972075\n",
      " val r2: 0.040191\n",
      " val rmse: 1.786700\n",
      "test r2: 0.055014\n",
      "test rmse: 1.775008\n",
      "('EarlyStopping counter:', 18, ' out of ', 50)\n",
      "\n",
      "====epoch 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.31it/s]\n",
      "Iteration:  42%|████▏     | 125/295 [00:00<00:00, 1242.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 1117.33it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 564.99it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 1023.18it/s]\n",
      "Iteration:  13%|█▎        | 37/295 [00:00<00:00, 365.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.055823\n",
      "train rmse: 0.971688\n",
      " val r2: 0.041456\n",
      " val rmse: 1.785522\n",
      "test r2: 0.056057\n",
      "test rmse: 1.774029\n",
      "('EarlyStopping counter:', 19, ' out of ', 50)\n",
      "\n",
      "====epoch 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 361.44it/s]\n",
      "Iteration:  39%|███▉      | 115/295 [00:00<00:00, 1147.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Evaluation\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration: 100%|██████████| 295/295 [00:00<00:00, 941.92it/s] \n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 561.44it/s]\n",
      "Iteration: 100%|██████████| 64/64 [00:00<00:00, 561.24it/s]\n",
      "Iteration:  12%|█▏        | 34/295 [00:00<00:00, 337.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train r2: 0.055955\n",
      "train rmse: 0.971620\n",
      " val r2: 0.041619\n",
      " val rmse: 1.785370\n",
      "test r2: 0.056788\n",
      "test rmse: 1.773341\n",
      "('EarlyStopping counter:', 20, ' out of ', 50)\n",
      "\n",
      "====epoch 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration:  37%|███▋      | 109/295 [00:00<00:00, 347.12it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-74-a7388332ea38>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m         \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"====epoch \"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"====Evaluation\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-70-343ec3813921>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, loader, optimizer, device)\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/mol/anaconda3/envs/jtree/lib/python2.7/site-packages/torch/optim/adam.pyc\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m                 \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m                     \u001b[0;31m# Maintains the maximum of all 2nd moment running avg. till now\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(1, 100):\n",
    "        print(\"====epoch \" + str(epoch))\n",
    "        \n",
    "        train(model, train_loader, optimizer, device)\n",
    "\n",
    "        print(\"====Evaluation\")\n",
    "\n",
    "        train_r2, train_rmse = eval(model, train_loader, scaler,device,  train = True)\n",
    "        \n",
    "        val_r2, val_rmse = eval(model, val_loader, scaler, device)\n",
    "        test_r2, test_rmse = eval( model, test_loader, scaler, device)\n",
    "\n",
    "        print(\"train r2: %f\\ntrain rmse: %f\\n val r2: %f\\n val rmse: %f\\ntest r2: %f\\ntest rmse: %f\"\\\n",
    "              %(train_r2, train_rmse, val_r2, val_rmse, test_r2, test_rmse))\n",
    "\n",
    "        # val_acc_list.append(val_acc)\n",
    "        # test_acc_list.append(test_acc)\n",
    "        # train_acc_list.append(train_acc)\n",
    "\n",
    "#         if not args.filename == \"\":\n",
    "        writer.add_scalar('data/train r2', train_r2, epoch)\n",
    "        writer.add_scalar('data/train rmse', train_rmse, epoch)\n",
    "\n",
    "        writer.add_scalar('data/val r2', val_r2, epoch)\n",
    "        writer.add_scalar('data/val rmse', val_rmse, epoch)\n",
    "        writer.add_scalar('data/test r2', test_r2, epoch)\n",
    "        writer.add_scalar('data/test rmse', test_rmse, epoch)\n",
    "\n",
    "        early_stopping(val_rmse, model)\n",
    "\n",
    "        if early_stopping.early_stop:\n",
    "            print(\"Early stopping\")\n",
    "            break\n",
    "\n",
    "        print(\"\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBRegressor\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(RAW_PATH,'X_train.npy'), 'rb') as f:\n",
    "    X_train = np.load(f).squeeze()\n",
    "with open(os.path.join(RAW_PATH,'y_train.npy'), 'rb') as f:\n",
    "    y_train = np.load(f)\n",
    "    \n",
    "y_train_transformed = scaler.fit_transform(y_train.reshape(-1, 1)).reshape(-1)\n",
    "    \n",
    "with open(os.path.join(RAW_PATH,'X_val.npy'), 'rb') as f:\n",
    "    X_val = np.load(f).squeeze()\n",
    "with open(os.path.join(RAW_PATH,'y_val.npy'), 'rb') as f:\n",
    "    y_val = np.load(f)\n",
    "\n",
    "with open(os.path.join(RAW_PATH,'X_test.npy'), 'rb') as f:\n",
    "    X_test = np.load(f).squeeze()\n",
    "with open(os.path.join(RAW_PATH,'y_test.npy'), 'rb') as f:\n",
    "    y_test = np.load(f) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train with whole vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBRegressor(n_estimators = 100, max_depth = 6, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_transformed = scaler.fit_transform(y_train.reshape(-1, 1)).reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, importance_type='gain',\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=6,\n",
       "       min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "       nthread=None, objective='reg:linear', random_state=42, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = scaler.inverse_transform(model.predict(X_test))\n",
    "val_preds = scaler.inverse_transform(model.predict(X_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Valid RMSE =', 0.905357986166681)\n",
      "Valid R2-score is 0.753553741911\n",
      "('Test RMSE =', 0.9247779104382)\n",
      "Test R2-score is 0.743492953339\n"
     ]
    }
   ],
   "source": [
    "print(\"Valid RMSE =\", mean_squared_error(y_val, val_preds)**0.5)\n",
    "print(\"Valid R2-score is {0}\".format(r2_score(y_val, val_preds)))\n",
    "\n",
    "print(\"Test RMSE =\", mean_squared_error(y_test, test_preds)**0.5)\n",
    "print(\"Test R2-score is {0}\".format(r2_score(y_test, test_preds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance = model.feature_importances_\n",
    "features = list(range(X_train.shape[1]))\n",
    "df = pd.DataFrame(columns = ['feature importance', 'feature name'])\n",
    "df['feature importance'] = feature_importance\n",
    "df['feature name'] = [str(i)+ '_tree_feats' for i in range(X_train.shape[1]//2)] +\\\n",
    "                    [str(i)+ '_graph_feats' for i in range(X_train.shape[1]//2, X_train.shape[1])]\n",
    "df_20_most_important = df.sort_values(by='feature importance', ascending = False)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_20_most_important = pd.read_csv(os.path.join(RAW_PATH,'experiments_result.csv'), index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| 2_tree_feats | 0.13753037 |\n",
      "| 17_tree_feats | 0.11225141 |\n",
      "| 22_tree_feats | 0.061198685 |\n",
      "| 27_tree_feats | 0.040304124 |\n",
      "| 21_tree_feats | 0.03701255 |\n",
      "| 10_tree_feats | 0.029927588999999997 |\n",
      "| 16_tree_feats | 0.026088234 |\n",
      "| 32_graph_feats | 0.02460154 |\n",
      "| 3_tree_feats | 0.024430856 |\n",
      "| 24_tree_feats | 0.02399634 |\n",
      "| 42_graph_feats | 0.021596342 |\n",
      "| 26_tree_feats | 0.021326277 |\n",
      "| 1_tree_feats | 0.021274897999999997 |\n",
      "| 19_tree_feats | 0.018548844 |\n",
      "| 15_tree_feats | 0.017468281000000002 |\n",
      "| 29_graph_feats | 0.01733831 |\n",
      "| 36_graph_feats | 0.015398215 |\n",
      "| 8_tree_feats | 0.014655213 |\n",
      "| 12_tree_feats | 0.014508405 |\n",
      "| 13_tree_feats | 0.013611487 |\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(df_20_most_important)):\n",
    "    print '|', df_20_most_important.iloc[i]['feature name'], '|', df_20_most_important.iloc[i]['feature importance'], '|'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train with tree features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBRegressor(n_estimators = 100, max_depth = 6, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_transformed = scaler.fit_transform(y_train.reshape(-1, 1)).reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, importance_type='gain',\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=6,\n",
       "       min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "       nthread=None, objective='reg:linear', random_state=42, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train[:, :X_train.shape[1]//2], y_train_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = scaler.inverse_transform(model.predict(X_test[:, :X_train.shape[1]//2]))\n",
    "val_preds = scaler.inverse_transform(model.predict(X_val[:, :X_train.shape[1]//2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Valid RMSE =', 0.9402256664953701)\n",
      "Valid R2-score is 0.734205640941\n",
      "('Test RMSE =', 0.9550508773345758)\n",
      "Test R2-score is 0.726424361726\n"
     ]
    }
   ],
   "source": [
    "print(\"Valid RMSE =\", mean_squared_error(y_val, val_preds)**0.5)\n",
    "print(\"Valid R2-score is {0}\".format(r2_score(y_val, val_preds)))\n",
    "\n",
    "print(\"Test RMSE =\", mean_squared_error(y_test, test_preds)**0.5)\n",
    "print(\"Test R2-score is {0}\".format(r2_score(y_test, test_preds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance = model.feature_importances_\n",
    "df = pd.DataFrame(columns = ['feature importance', 'feature name'])\n",
    "df['feature importance'] = feature_importance\n",
    "df['feature name'] = [str(i)+ '_tree_feats' for i in range(X_train.shape[1]//2)]\n",
    "df_20_most_important = df.sort_values(by='feature importance', ascending = False)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature importance</th>\n",
       "      <th>feature name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.176946</td>\n",
       "      <td>2_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.152939</td>\n",
       "      <td>17_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.080504</td>\n",
       "      <td>22_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.051779</td>\n",
       "      <td>27_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.047035</td>\n",
       "      <td>21_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.037569</td>\n",
       "      <td>3_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.037453</td>\n",
       "      <td>10_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.037348</td>\n",
       "      <td>16_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.031914</td>\n",
       "      <td>24_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.030068</td>\n",
       "      <td>26_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.028452</td>\n",
       "      <td>1_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.025844</td>\n",
       "      <td>15_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.024838</td>\n",
       "      <td>19_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.024040</td>\n",
       "      <td>12_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.020233</td>\n",
       "      <td>14_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.019516</td>\n",
       "      <td>23_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.019088</td>\n",
       "      <td>13_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.019017</td>\n",
       "      <td>8_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.016931</td>\n",
       "      <td>9_tree_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.016481</td>\n",
       "      <td>5_tree_feats</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    feature importance   feature name\n",
       "2             0.176946   2_tree_feats\n",
       "17            0.152939  17_tree_feats\n",
       "22            0.080504  22_tree_feats\n",
       "27            0.051779  27_tree_feats\n",
       "21            0.047035  21_tree_feats\n",
       "3             0.037569   3_tree_feats\n",
       "10            0.037453  10_tree_feats\n",
       "16            0.037348  16_tree_feats\n",
       "24            0.031914  24_tree_feats\n",
       "26            0.030068  26_tree_feats\n",
       "1             0.028452   1_tree_feats\n",
       "15            0.025844  15_tree_feats\n",
       "19            0.024838  19_tree_feats\n",
       "12            0.024040  12_tree_feats\n",
       "14            0.020233  14_tree_feats\n",
       "23            0.019516  23_tree_feats\n",
       "13            0.019088  13_tree_feats\n",
       "8             0.019017   8_tree_feats\n",
       "9             0.016931   9_tree_feats\n",
       "5             0.016481   5_tree_feats"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_20_most_important"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train with graph features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBRegressor(n_estimators = 100, max_depth = 6, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, importance_type='gain',\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=6,\n",
       "       min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
       "       nthread=None, objective='reg:linear', random_state=42, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train[:, X_train.shape[1]//2:], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = model.predict(X_test[:, X_train.shape[1]//2:])\n",
    "val_preds = model.predict(X_val[:, X_train.shape[1]//2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Valid RMSE =', 1.3417327115036215)\n",
      "Valid R2-score is 0.458730481237\n",
      "('Test RMSE =', 1.3324128192277744)\n",
      "Test R2-score is 0.467521581709\n"
     ]
    }
   ],
   "source": [
    "print(\"Valid RMSE =\", mean_squared_error(y_val, val_preds)**0.5)\n",
    "print(\"Valid R2-score is {0}\".format(r2_score(y_val, val_preds)))\n",
    "\n",
    "print(\"Test RMSE =\", mean_squared_error(y_test, test_preds)**0.5)\n",
    "print(\"Test R2-score is {0}\".format(r2_score(y_test, test_preds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance = model.feature_importances_\n",
    "df = pd.DataFrame(columns = ['feature importance', 'feature name'])\n",
    "df['feature importance'] = feature_importance\n",
    "df['feature name'] = [str(i)+ '_graph_feats' for i in range(X_train.shape[1]//2, X_train.shape[1])]\n",
    "df_20_most_important = df.sort_values(by='feature importance', ascending = False)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature importance</th>\n",
       "      <th>feature name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.055859</td>\n",
       "      <td>55_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.051214</td>\n",
       "      <td>50_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.050143</td>\n",
       "      <td>32_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.049575</td>\n",
       "      <td>36_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.048136</td>\n",
       "      <td>39_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.046766</td>\n",
       "      <td>42_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.044520</td>\n",
       "      <td>37_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.042289</td>\n",
       "      <td>46_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.039642</td>\n",
       "      <td>41_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.038473</td>\n",
       "      <td>49_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.038248</td>\n",
       "      <td>52_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.036629</td>\n",
       "      <td>51_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.036443</td>\n",
       "      <td>53_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.034955</td>\n",
       "      <td>45_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.033760</td>\n",
       "      <td>48_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.033757</td>\n",
       "      <td>54_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.032955</td>\n",
       "      <td>29_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.032867</td>\n",
       "      <td>43_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.032256</td>\n",
       "      <td>38_graph_feats</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.030901</td>\n",
       "      <td>31_graph_feats</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    feature importance    feature name\n",
       "27            0.055859  55_graph_feats\n",
       "22            0.051214  50_graph_feats\n",
       "4             0.050143  32_graph_feats\n",
       "8             0.049575  36_graph_feats\n",
       "11            0.048136  39_graph_feats\n",
       "14            0.046766  42_graph_feats\n",
       "9             0.044520  37_graph_feats\n",
       "18            0.042289  46_graph_feats\n",
       "13            0.039642  41_graph_feats\n",
       "21            0.038473  49_graph_feats\n",
       "24            0.038248  52_graph_feats\n",
       "23            0.036629  51_graph_feats\n",
       "25            0.036443  53_graph_feats\n",
       "17            0.034955  45_graph_feats\n",
       "20            0.033760  48_graph_feats\n",
       "26            0.033757  54_graph_feats\n",
       "1             0.032955  29_graph_feats\n",
       "15            0.032867  43_graph_feats\n",
       "10            0.032256  38_graph_feats\n",
       "3             0.030901  31_graph_feats"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_20_most_important"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 48 candidates, totalling 96 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  40 tasks      | elapsed: 14.8min\n",
      "[Parallel(n_jobs=5)]: Done  96 out of  96 | elapsed: 48.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7722961415729136\n",
      "{'colsample_bytree': 0.7, 'silent': 1, 'learning_rate': 0.05, 'nthread': 4, 'min_child_weight': 4, 'n_estimators': 1000, 'subsample': 0.7, 'max_depth': 5}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "xgb1 = XGBRegressor()\n",
    "parameters = {'nthread':[4], #when use hyperthread, xgboost may become slower\n",
    "              'learning_rate': [.03, 0.05, .07], #so called `eta` value\n",
    "              'max_depth': [5, 10, 20, 50],\n",
    "              'min_child_weight': [4],\n",
    "              'silent': [1],\n",
    "              'subsample': [0.7],\n",
    "              'colsample_bytree': [0.7],\n",
    "              'n_estimators': [100, 200, 500, 1000]}\n",
    "\n",
    "xgb_grid = GridSearchCV(xgb1,\n",
    "                        parameters,\n",
    "                        cv = 2,\n",
    "                        n_jobs = 5,\n",
    "                        verbose=True)\n",
    "\n",
    "xgb_grid.fit(np.concatenate((X_train, X_val), axis=0),\n",
    "         np.concatenate((y_train, y_val), axis=0))\n",
    "\n",
    "print(xgb_grid.best_score_)\n",
    "print(xgb_grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XGBRegressor(colsample_bytree=0.7, silent = 1, learning_rate = 0.05, nthread = 4, min_child_weight = 4, n_estimators = 1000, subsample = 0.7, max_depth = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=0.7, gamma=0, importance_type='gain',\n",
       "       learning_rate=0.05, max_delta_step=0, max_depth=5,\n",
       "       min_child_weight=4, missing=None, n_estimators=1000, n_jobs=1,\n",
       "       nthread=4, objective='reg:linear', random_state=0, reg_alpha=0,\n",
       "       reg_lambda=1, scale_pos_weight=1, seed=None, silent=1,\n",
       "       subsample=0.7)"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = model.predict(X_test)\n",
    "val_preds = model.predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Valid RMSE =', 0.8063512258633068)\n",
      "Valid R2-score is 0.804507520131\n",
      "('Test RMSE =', 0.8211428563420121)\n",
      "Test R2-score is 0.797762427624\n"
     ]
    }
   ],
   "source": [
    "print(\"Valid RMSE =\", mean_squared_error(y_val, val_preds)**0.5)\n",
    "print(\"Valid R2-score is {0}\".format(r2_score(y_val, val_preds)))\n",
    "\n",
    "print(\"Test RMSE =\", mean_squared_error(y_test, test_preds)**0.5)\n",
    "print(\"Test R2-score is {0}\".format(r2_score(y_test, test_preds)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_RMSE = [0.5342044580494232, 0.5405131737041414, 0.5144339590477948]\n",
    "test_R2 = [0.9212916061654113, 0.9134389008189245, 0.9156177478399877]\n",
    "\n",
    "val_RMSE = [0.5330971133864346, 0.5393970943801238, 0.5320911200134589]\n",
    "val_R2 = [0.9167385707444774, 0.9193261655969616, 0.9188716678036257]\n",
    "\n",
    "train_RMSE = [0.18383073729520522, 0.18567786361473315, 0.17486524864784958]\n",
    "train_R2 = [0.9662330071097933, 0.9655380319633392, 0.969380081805695]\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE = 0.18145794985259597 +- 0.004722340577254158\n",
      "Train R2-score is 0.9670503736262758+-0.0016716065654506672\n",
      "Valid RMSE = 0.5348617759266725 +- 0.0032331450913189994\n",
      "Valid R2-score is 0.9183121347150216+-0.0011280424749930953\n",
      "Test RMSE = 0.5297171969337865 +- 0.011109545250605615\n",
      "Test R2-score is 0.9167827516081078+-0.003310002079504476\n"
     ]
    }
   ],
   "source": [
    "print(\"Train RMSE =\", np.mean(train_RMSE),'+-',np.std(train_RMSE))\n",
    "print(\"Train R2-score is {0}+-{1}\".format(np.mean(train_R2),np.std(train_R2)))\n",
    "\n",
    "print(\"Valid RMSE =\", np.mean(val_RMSE),'+-',np.std(val_RMSE))\n",
    "print(\"Valid R2-score is {0}+-{1}\".format(np.mean(val_R2),np.std(val_R2)))\n",
    "\n",
    "print(\"Test RMSE =\", np.mean(test_RMSE),'+-',np.std(test_RMSE))\n",
    "print(\"Test R2-score is {0}+-{1}\".format(np.mean(test_R2),np.std(test_R2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "jtree",
   "language": "python",
   "name": "jtree"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}