RNN_CV19Net.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    },
    "colab": {
      "name": "RNN-CV19Net.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ammarisme/covid-19/blob/master/RNN_CV19Net.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C83XgaWprQnn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%matplotlib inline\n",
        "!pip install torch-geometric \\\n",
        "  torch-sparse==latest+cu101 \\\n",
        "  torch-scatter==latest+cu101 \\\n",
        "  torch-cluster==latest+cu101 \\\n",
        "  -f https://pytorch-geometric.com/whl/torch-1.5.0.html"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gRUdaEpW2GsF",
        "colab_type": "text"
      },
      "source": [
        "!pip install torch-geometric \\\n",
        "  torch-sparse==latest+cu101 \\\n",
        "  torch-scatter==latest+cu101 \\\n",
        "  torch-cluster==latest+cu101 \\\n",
        "  -f https://pytorch-geometric.com/whl/torch-1.5.0.html\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "anKz91FjrQn3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import unicode_literals, print_function, division\n",
        "from io import open\n",
        "import unicodedata\n",
        "import string\n",
        "import re\n",
        "import random\n",
        "import os\n",
        "\n",
        "import time\n",
        "import math\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch import optim\n",
        "import torch.nn.functional as F\n",
        "from torch_geometric.data import Data, DataLoader, InMemoryDataset\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from datetime import datetime\n",
        "\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print('running on '+ str(device))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bFsXKb1CzwSY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "import os.path\n",
        "from os import path\n",
        "import sys\n",
        "from os import listdir\n",
        "from os.path import isfile, join\n",
        "\n",
        "drive.mount('/content/drive')\n",
        "PATH = '/content/drive/My Drive/covid'\n",
        "sys.path.append(PATH)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HidtRwtG9CFu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CovidDataSet(InMemoryDataset):\n",
        "    def __init__(self, root, input_sequence, output_sequence, transform=None, pre_transform=None):\n",
        "        super(CovidDataSet, self).__init__(root, transform, pre_transform)\n",
        "        self.data, self.slices = torch.load(self.processed_paths[0])\n",
        "\n",
        "    @property\n",
        "    def raw_dir(self):\n",
        "      if os.path.exists(self.root+PROCESSED_DIR):\n",
        "        return self.root+'/cleaned'\n",
        "      else:\n",
        "        os.mkdir(self.root+PROCESSED_DIR)\n",
        "        return self.root+'/cleaned'\n",
        "        \n",
        "    @property\n",
        "    def processed_dir(self):\n",
        "      if os.path.exists(self.root+PROCESSED_DIR):\n",
        "        return self.root+PROCESSED_DIR\n",
        "      else:\n",
        "        os.mkdir(self.root+PROCESSED_DIR)\n",
        "        return self.root+PROCESSED_DIR\n",
        "\n",
        "    @property\n",
        "    def raw_file_names(self):\n",
        "      mypath = self.raw_dir\n",
        "      filenames = [f for f in listdir(mypath) if isfile(join(mypath, f))]\n",
        "      return filenames\n",
        "\n",
        "    @property\n",
        "    def processed_file_names(self):\n",
        "        return ['processed.dt']\n",
        "\n",
        "    def download(self):\n",
        "        pass\n",
        "    \n",
        "    def process(self):\n",
        "        \n",
        "        data_list = []\n",
        "        output_sequence_len = configuration['output_sequence_len']\n",
        "        input_sequence_len = configuration['input_sequence_len']\n",
        "        feature_length = configuration['feature_len']\n",
        "        yhat_size = configuration['yhat_size']\n",
        "\n",
        "        for raw_path in self.raw_paths:\n",
        "          df = pd.read_csv(raw_path)\n",
        "          for synthetic_seq in df['synthesis_seq'].unique():\n",
        "            synthetic_data = df[df['synthesis_seq']==synthetic_seq]\n",
        "\n",
        "            for country in synthetic_data['countryterritoryCode'].unique():\n",
        "              country_data = synthetic_data[synthetic_data['countryterritoryCode'] == country]\n",
        "              _country_code = country_data['_country_code'].values[0]\n",
        "              \n",
        "              country_data_i = country_data[:-output_sequence_len]\n",
        "              country_data_o = country_data[input_sequence_len:]\n",
        "              \n",
        "              country_data_array = np.array([country_data_i['cases'].to_numpy(),\n",
        "                                             country_data_i['deaths'].to_numpy()\n",
        "                                             ])\n",
        "              feature_length = len(country_data_array)\n",
        "              country_data_array = country_data_array.reshape(feature_length,len(country_data_i))\n",
        "\n",
        "              country_data_array_y = np.array([country_data_o['cases'].to_numpy(), country_data_o['deaths'].to_numpy()])\n",
        "              country_data_array_y = country_data_array_y.reshape(2,len(country_data_o))\n",
        "\n",
        "              x = country_data_array[:feature_length].T\n",
        "              y = country_data_array_y[:yhat_size].T\n",
        "\n",
        "              sets =0\n",
        "              x_list = []\n",
        "              dict_x = dict()\n",
        "              for i in range(input_sequence_len):\n",
        "                array_len = ((len(x) -i) - ((len(x)-i)%input_sequence_len))+i\n",
        "                if array_len <= 0:\n",
        "                  continue\n",
        "                sets = int( array_len/ input_sequence_len)\n",
        "                if sets <= 0:\n",
        "                  continue\n",
        "                #print('input seq : ', i , ' ', array_len , ' ',array_len-i , ' number of sets : ', sets)\n",
        "                x_temp = x[i:array_len].T.reshape(sets,feature_length,input_sequence_len)\n",
        "                x_temp = x_temp.reshape(feature_length,sets,input_sequence_len)\n",
        "                uniq_keys = np.array([i+(input_sequence_len*k) for k in range(input_sequence_len)])\n",
        "                \n",
        "                arrays_split = np.hsplit(x_temp,sets)\n",
        "                dict_x.update(dict(zip(uniq_keys, arrays_split)))\n",
        "              \n",
        "              dict_y = dict()\n",
        "              y_list = []\n",
        "              for i in range(output_sequence_len):\n",
        "                array_len_y = (len(y)-i) - ((len(y)  - i)%output_sequence_len)+i\n",
        "                if array_len_y <= 0:\n",
        "                  continue\n",
        "                sets = int(array_len_y / output_sequence_len)\n",
        "                if sets <= 0:\n",
        "                  continue\n",
        "                \n",
        "                #print('output seq : ', i , ' ', array_len_y , ' ',array_len_y-(i) , ' number of sets : ', sets)\n",
        "                y_temp = y[i:array_len_y].T.reshape(sets, 2, output_sequence_len)\n",
        "                uniq_keys = np.array([i+(output_sequence_len*k) for k in range(output_sequence_len)])\n",
        "                y_temp = y_temp.reshape(2,sets,output_sequence_len)\n",
        "                arrays_split = np.hsplit(y_temp,sets)\n",
        "                dict_y.update(dict(zip(uniq_keys, arrays_split)))\n",
        "              \n",
        "\n",
        "              temp_x_list  = [dict_x[i].T for i in sorted(dict_x.keys())]\n",
        "              temp_y_list  = [dict_y[i].T for i in sorted(dict_y.keys())]\n",
        "\n",
        "              xy_list = [Data(x = torch.from_numpy(features).type(torch.FloatTensor).squeeze()) for features in temp_x_list]\n",
        "\n",
        "              for i in sorted(dict_y.keys()):\n",
        "                xy_list[i].y = torch.from_numpy(temp_y_list[i]).squeeze()\n",
        "\n",
        "              data_list += xy_list\n",
        "          print('processed : '+ raw_path)\n",
        "        data, slices = self.collate(data_list)\n",
        "        torch.save((data, slices), self.processed_paths[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Rv5DjHyrQor",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class RNNModel(nn.Module):\n",
        "    def __init__(self, input_size, parameter_sizes, repeats ,output_size):\n",
        "        super(RNNModel, self).__init__()\n",
        "        self.input_size = input_size\n",
        "        self.repeater_input_size = parameter_sizes[0]\n",
        "        self.hidden_size = parameter_sizes[1]\n",
        "        self.repeats = repeats\n",
        "        self.output_size = output_size\n",
        "\n",
        "        self.fc1 = nn.Linear(input_size, self.repeater_input_size)\n",
        "        self.relu_activation = nn.ReLU()\n",
        "\n",
        "        self.layers = dict()\n",
        "\n",
        "        k = 0\n",
        "        for i in range(repeats):\n",
        "          i = i+k\n",
        "          self.layers['fc_'+str(i)] = nn.Linear(self.repeater_input_size, self.hidden_size )\n",
        "          self.layers['gru_'+str(i+1)] = nn.GRU(self.hidden_size, self.hidden_size )\n",
        "          self.layers['fc_'+str(i+2)] = nn.Linear(self.hidden_size, self.repeater_input_size)\n",
        "          k+=2\n",
        "\n",
        "        self.module_list = nn.ModuleDict(self.layers)\n",
        "\n",
        "        self.fc2 = nn.Linear(self.repeater_input_size, output_size)\n",
        "        \n",
        "    def forward(self, input, hidden):\n",
        "      output = self.fc1(input)\n",
        "      output = self.relu_activation(output)\n",
        "\n",
        "      k = 0\n",
        "      for i in range(self.repeats):\n",
        "        i = i+k\n",
        "        output = self.layers['fc_'+str(i)](output)\n",
        "        output = self.relu_activation(output)\n",
        "\n",
        "        output, hidden[i-k] = self.layers['gru_'+str(i+1)](output, hidden[i-k])#should be different. check the nlp page\n",
        "        output = self.relu_activation(output)\n",
        "        hidden[i-k] = self.relu_activation(hidden[i-k])\n",
        "\n",
        "        output = self.layers['fc_'+str(i+2)](output)\n",
        "        output = self.relu_activation(output)\n",
        "        k +=2\n",
        "\n",
        "      output = self.fc2(output)\n",
        "      output = self.relu_activation(output)\n",
        "\n",
        "      return output, hidden\n",
        "\n",
        "    def initHidden(self, batch_size):\n",
        "        return torch.zeros(1, batch_size, self.hidden_size, device=device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wZBmfYEtiWem",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_rnn(input_tensor, target_tensor, rnn_model, rnn_model_optimizer, criterion,batch_size):\n",
        "  input_tensor = input_tensor.type(torch.FloatTensor)\n",
        "  target_tensor= target_tensor.type(torch.FloatTensor)\n",
        "\n",
        "  rnn_model_hidden = []\n",
        "  for i in range(rnn_model.repeats):\n",
        "    rnn_model_hidden.append(rnn_model.initHidden(batch_size))\n",
        "\n",
        "  rnn_model_optimizer.zero_grad()\n",
        "  \n",
        "  loss = 0  \n",
        "  for ei in range(configuration['input_sequence_len']):\n",
        "    target_tensor_seq = target_tensor.view(configuration['output_sequence_len'], batch_size, configuration['yhat_size'])\n",
        "    input_tensor_seq = input_tensor.view(configuration['input_sequence_len'],batch_size, configuration['feature_len'])[ei]\n",
        "    input_tensor_seq = input_tensor_seq.view(1, batch_size, configuration['feature_len']).to(device)\n",
        "\n",
        "    rnn_model_output, rnn_model_hidden = rnn_model(\n",
        "        input_tensor_seq, rnn_model_hidden)\n",
        "    #print(rnn_model_output.size(), target_tensor_seq[ei].size())\n",
        "    loss += criterion(rnn_model_output.squeeze(), target_tensor_seq[ei].squeeze().to(device))\n",
        "\n",
        "  loss.backward()\n",
        "  rnn_model_optimizer.step()\n",
        "\n",
        "  return loss.item() / configuration['input_sequence_len']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Bb9yefOKZrz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate_rnn(input_tensor, target_tensor, rnn_model, criterion,batch_size):\n",
        "  with torch.no_grad():\n",
        "    input_tensor = input_tensor.type(torch.FloatTensor)\n",
        "    target_tensor= target_tensor.type(torch.FloatTensor)\n",
        "\n",
        "    rnn_model_hidden = []\n",
        "    for i in range(rnn_model.repeats):\n",
        "      rnn_model_hidden.append(rnn_model.initHidden(batch_size))\n",
        "\n",
        "    validation_loss = 0\n",
        "    \n",
        "    for ei in range(configuration['input_sequence_len']):\n",
        "      target_tensor_seq = target_tensor.view(configuration['output_sequence_len'], batch_size, configuration['yhat_size'])\n",
        "      input_tensor_seq = input_tensor.view(configuration['input_sequence_len'],batch_size, configuration['feature_len'])[ei]\n",
        "      input_tensor_seq = input_tensor_seq.view(1, batch_size, configuration['feature_len']).to(device)\n",
        "\n",
        "      rnn_model_output, rnn_model_hidden = rnn_model(\n",
        "          input_tensor_seq, rnn_model_hidden)\n",
        "      validation_loss += criterion(rnn_model_output.squeeze(), target_tensor_seq[ei].squeeze().to(device))\n",
        "\n",
        "  return validation_loss.item() / configuration['input_sequence_len']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h3sGikLKufvN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "configuration = {\n",
        "    'input_sequence_len' : 10,\n",
        "    'output_sequence_len' : 10,\n",
        "    'training_batch_size' : 1024,\n",
        "    'training_dataset_length' :32768,\n",
        "    'validation_batch_size' : 1024,\n",
        "    'yhat_size' : 2,\n",
        "    'feature_len' : 2,\n",
        "    'output_size' : 2,\n",
        "}\n",
        "\n",
        "\n",
        "INPUT_ROOT = PATH+'/input_mix'\n",
        "DATA_TAG = \"seq2seq_\"+str(configuration['input_sequence_len'])+'_'+str(configuration['output_sequence_len'])\n",
        "PROCESSED_DIR = '/processed_'+DATA_TAG\n",
        "\n",
        "validation_dataset = CovidDataSet(INPUT_ROOT+'/validation', configuration['input_sequence_len'], configuration['output_sequence_len'])\n",
        "validation_dataset = validation_dataset.shuffle()\n",
        "validation_dataset = validation_dataset[3000:]\n",
        "validation_dataloader = DataLoader(validation_dataset,batch_size=configuration['validation_batch_size'])\n",
        "\n",
        "training_dataset = CovidDataSet(INPUT_ROOT+'/training', configuration['input_sequence_len'], configuration['input_sequence_len'])\n",
        "training_dataset = training_dataset.shuffle()\n",
        "training_dataset = training_dataset[:configuration['training_dataset_length']]\n",
        "training_dataloader = DataLoader(training_dataset,batch_size=configuration['training_batch_size'])\n",
        "\n",
        "#training_dataset = validation_dataset[:3000]\n",
        "#training_dataset = training_dataset[:configuration['training_dataset_length']]\n",
        "#training_dataloader = DataLoader(training_dataset,batch_size=configuration['training_batch_size'])\n",
        "\n",
        "print('batches validation :', len(validation_dataloader), 'training : ', len(training_dataloader), 'sample ', validation_dataset[100].x)\n",
        "print('dataset length validation :', len(validation_dataset), 'training : ', len(training_dataset))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHLM8BnLR9sv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_repeat_counts(alphas , training_dataset_length, feature_len,yhat_size,\\\n",
        "                           enc_repeated_nn_parameters = 132096):\n",
        "  repeats = []\n",
        "  model1 = RNNModel(**{\n",
        "         'input_size': configuration['feature_len'],\n",
        "         'parameter_sizes': [128, 256] ,\n",
        "         'repeats' : 1 ,\n",
        "         'output_size' : yhat_size\n",
        "        })\n",
        "  model2 = RNNModel(**{\n",
        "         'input_size': configuration['feature_len'],\n",
        "         'parameter_sizes': [128, 256] ,\n",
        "         'repeats' : 2 ,\n",
        "         'output_size' : yhat_size\n",
        "        }\n",
        "  )\n",
        "\n",
        "  enc_repeated_nn_parameters = get_n_params(model2) - get_n_params(model1)\n",
        "\n",
        "  for alpha in alphas:\n",
        "    preferred_parameters  = (1/alpha)* (training_dataset_length / (configuration['feature_len']+configuration['yhat_size']))\n",
        "    enc_repeats = int(\n",
        "        (preferred_parameters - \n",
        "        (preferred_parameters%enc_repeated_nn_parameters))\n",
        "        / enc_repeated_nn_parameters)\n",
        "    for j in range(1, enc_repeats+1):\n",
        "      if j not in repeats:\n",
        "        repeats.append(j)\n",
        "  return repeats\n",
        "\n",
        "def generate_model_parameters(repeat_counts, feature_len, output_size):\n",
        "  model_dict = dict()\n",
        "  model_id = 0\n",
        "  for enc_repeat in repeat_counts:\n",
        "    model_dict[model_id] = [{\n",
        "         'input_size': feature_len,\n",
        "         'parameter_sizes': [256, 128] ,\n",
        "         'repeats' : enc_repeat ,\n",
        "         'output_size' : output_size\n",
        "        }]\n",
        "    model_id+=1\n",
        "  return model_dict\n",
        "\n",
        "def save_model_parameters(model_parameters_dict, models = None, fileName=None):\n",
        "  if fileName == None:\n",
        "    fileName = 'model_parameters_'+str(np.round(np.random.randn(1000,200)[0][0]*1000))\n",
        "\n",
        "  model_summary = pd.DataFrame(columns =[])\n",
        "  for i in sorted(model_parameters_dict.keys()):\n",
        "    param_size = 0\n",
        "    if models != None:\n",
        "      param_size = get_n_params(models[i][0])\n",
        "      param_size += get_n_params(models[i][1])\n",
        "    row = pd.Series({\n",
        "                        'model_id':i,\n",
        "                        'repeats' : model_parameters_dict[i][0]['repeats'],\n",
        "                        'parameters' :param_size\n",
        "    })\n",
        "    row_df = pd.DataFrame([row], index = [i])\n",
        "    model_summary = pd.concat([model_summary, row_df])\n",
        "  model_summary.to_csv(fileName)\n",
        "\n",
        "#Create a new directory to store model results\n",
        "MODEL_PREFIX = '_rnn'\n",
        "MODEL_DIR =  PATH + '/models_'+MODEL_PREFIX\n",
        "dirs = []\n",
        "for f in listdir(MODEL_DIR):\n",
        "  if \"run\" in f:\n",
        "    dirs.append(int(f[3]))\n",
        "if os.path.exists(MODEL_DIR+\"/run\"+str(max(dirs)+1)) == False:\n",
        "  os.mkdir(MODEL_DIR+\"/run\"+str(max(dirs)+1))\n",
        "  MODEL_DIR = MODEL_DIR+\"/run\"+str(max(dirs)+1)\n",
        "\n",
        "\n",
        "if os.path.exists(MODEL_DIR)==False:\n",
        "  os.mkdir(MODEL_DIR)\n",
        "session_id =str(1001)+ '_'+str(datetime.now(tz=None).microsecond)\n",
        "\n",
        "repeat_counts = generate_repeat_counts([1,2,3], 500*len(training_dataset), feature_len = 8, yhat_size=2)\n",
        "\n",
        "model_configurations = generate_model_parameters(repeat_counts, configuration['feature_len'],configuration['output_size'])\n",
        "save_model_parameters(model_configurations, fileName= MODEL_DIR + '/rnn_model_configurations_'+session_id+'.csv')\n",
        "models = dict()\n",
        "print(len(model_configurations))\n",
        "print(len(repeat_counts))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W34bK4bcqCMi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Run all the candidate models\n",
        "session_id =str(1001)+ '_'+str(datetime.now(tz=None).microsecond)\n",
        "learning_rate = 0.1\n",
        "for i in model_configurations:\n",
        "  #instantiate the models\n",
        "  if i != 4:\n",
        "    print('skipping', i)\n",
        "    continue\n",
        "  current_model_dir =  MODEL_DIR +'/'+str(i)\n",
        "  if os.path.exists(current_model_dir) == False:\n",
        "    os.mkdir(current_model_dir)\n",
        "  \n",
        "  rnn_model = current_model\n",
        "  rnn_model_optimizer= current_optimizer\n",
        "  criterion = nn.MSELoss()\n",
        "\n",
        "  losses = trainValidationIters(training_dataloader,validation_dataloader,\n",
        "                    rnn_model,\n",
        "                    rnn_model_optimizer,\n",
        "                    criterion, n_iters = 10000,output_sequence_len=configuration['output_sequence_len'],\n",
        "                    input_sequence_len=configuration['input_sequence_len'],\n",
        "                    current_model_dir=current_model_dir,  model_id = i,session_id = session_id,\n",
        "                     print_every=10, save_every = 50)\n",
        "  save_dict({\n",
        "      'model' : rnn_model.state_dict(),\n",
        "      'rnn_model_optimizer' : rnn_model_optimizer.state_dict(),\n",
        "      'losses' : losses\n",
        "  }, current_model_dir+'/'+str(i)+'models_'+session_id+'.pt')\n",
        "\n",
        "  print('Model ',str(i), ': trained and evaluated')\n",
        "  current_model = rnn_model\n",
        "  current_optimizer = rnn_model_optimizer\n",
        "  models[i] = {\n",
        "      'rnn_model' : rnn_model.state_dict(),\n",
        "      'rnn_model_optimizer' : rnn_model_optimizer.state_dict(),\n",
        "      'losses' : losses\n",
        "  }\n",
        "\n",
        "save_dict(models, MODEL_DIR+'/models_'+session_id+'.pt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Nn2IqgCTe47",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#torch.save(current_model.state_dict(), current_model_dir+'/model_manual_save_0007_7.pt')\n",
        "#torch.save(current_optimizer.state_dict(), current_model_dir+'/optimizer_manual_save_0007_7.pt')\n",
        "torch.save(rnn_model.state_dict(), current_model_dir+'/model_manual_save_0008_7.pt')\n",
        "torch.save(rnn_model_optimizer.state_dict(), current_model_dir+'/optimizer_manual_save_0008_7.pt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zk5WWsBe43Jz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_batch(dataloader, rnn_model, rnn_model_optimizer, criterion):\n",
        "  loss = 0\n",
        "  for batch in dataloader:\n",
        "    batch_size = int(len(batch.x)/configuration['input_sequence_len'])\n",
        "    loss += (train_rnn(batch.x.view(batch_size, configuration['input_sequence_len'] , configuration['feature_len']), batch.y, rnn_model,\n",
        "                      rnn_model_optimizer, criterion, batch_size))/ batch_size\n",
        "  return loss / len(dataloader)\n",
        "\n",
        "def batch_evaluate(dataloader, rnn_model, criterion, output_sequence_len):\n",
        "  loss = 0\n",
        "  for batch in dataloader:\n",
        "    batch_size = int(len(batch.x)/configuration['input_sequence_len'])\n",
        "    loss += (evaluate_rnn(batch.x.view(batch_size, configuration['input_sequence_len'] , configuration['feature_len']), batch.y, rnn_model,\n",
        "                     criterion, batch_size))/ batch_size\n",
        "  return loss / len(dataloader)\n",
        "\n",
        "def trainValidationIters(train_dataloader,validation_dataloader, rnn_model, rnn_model_optimizer, \n",
        "                        criterion ,n_iters, output_sequence_len,input_sequence_len,\n",
        "                        current_model_dir, model_id,session_id,print_every=1000, plot_every=100,save_every = 100):\n",
        "\n",
        "  start = time.time()\n",
        "  plot_losses = []\n",
        "  \n",
        "  # Reset every print_every # Reset every plot_every\n",
        "  print_training_loss_total = 0\n",
        "  plot_training_loss_total = 0\n",
        "  print_validation_loss_total = 0\n",
        "  plot_validation_loss_total = 0\n",
        "  save_validation_loss_total = 0\n",
        "  save_training_loss_total = 0\n",
        "      \n",
        "  losses = []\n",
        "\n",
        "  for iter in range(1, n_iters + 1):\n",
        "      rnn_model.train()\n",
        "      training_loss = train_batch(train_dataloader, rnn_model,\n",
        "                                  rnn_model_optimizer, criterion )\n",
        "      \n",
        "      rnn_model.eval()\n",
        "      validation_loss = batch_evaluate(validation_dataloader, rnn_model,\n",
        "                                       criterion,configuration['output_sequence_len'] )\n",
        "      \n",
        "      print_training_loss_total += training_loss\n",
        "      plot_training_loss_total += training_loss\n",
        "      print_validation_loss_total += validation_loss\n",
        "      plot_validation_loss_total += validation_loss\n",
        "      \n",
        "      save_training_loss_total += training_loss\n",
        "      save_validation_loss_total += validation_loss\n",
        "      \n",
        "      if iter % print_every == 0:\n",
        "          print_training_loss_avg = print_training_loss_total / print_every\n",
        "          print_training_loss_total = 0\n",
        "          print_validation_loss_avg = print_validation_loss_total / print_every\n",
        "          print_validation_loss_total = 0\n",
        "          print('%s (%d %d%%) training:%.4f validation:%.4f' % (timeSince(start, iter / n_iters),\n",
        "                                        iter, iter / n_iters * 100, print_training_loss_avg,print_validation_loss_avg, ))\n",
        "          print('train/validation difference : ',print_validation_loss_avg-print_training_loss_avg)\n",
        "\n",
        "      #if iter % plot_every == 0:\n",
        "      #    plot_training_loss_avg = plot_training_loss_total / plot_every\n",
        "      #    plot_validation_loss_avg = plot_validation_loss_total / plot_every\n",
        "      #    plot_training_loss.append(plot_training_loss_avg)\n",
        "      #    plot_validation_loss.append(plot_validation_loss_avg)\n",
        "      #    showPlot(plot_training_losses,plot_validation_loss)\n",
        "      #    plot_training_loss_total = 0\n",
        "      #    plot_validation_loss_total\n",
        "        \n",
        "      if iter % save_every == 0:\n",
        "        save_training_loss_avg = save_training_loss_total / save_every\n",
        "        save_validation_loss_avg = plot_validation_loss_total / save_every\n",
        "        \n",
        "        #'arch': args.arch,\n",
        "        save_checkpoint({\n",
        "            'epoch': iter + 1,\n",
        "            'rnn_model_state_dict': rnn_model.state_dict(),\n",
        "            'training_loss': save_training_loss_avg,\n",
        "            'validation_loss': save_validation_loss_avg,\n",
        "            'rnn_model_optimizer' : rnn_model_optimizer.state_dict(),\n",
        "        }, False, filename=current_model_dir+'/'+str(iter)+'_'+str(session_id)+'.pt')\n",
        "      losses.append([training_loss, validation_loss])\n",
        "\n",
        "  save_dict({\n",
        "      'losses' : losses\n",
        "  }, filename=current_model_dir+'/losses_'+session_id)\n",
        "\n",
        "  return losses"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JbZSFpPGX215",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"\n",
        "Common function\n",
        "\"\"\"\n",
        "def get_n_params(model):\n",
        "    pp=0\n",
        "    for p in list(model.parameters()):\n",
        "        nn=1\n",
        "        for s in list(p.size()):\n",
        "            nn = nn*s\n",
        "        pp += nn\n",
        "    return pp\n",
        "  \n",
        "\n",
        "def save_model_parameters(model_parameters_dict, models = None, fileName=None):\n",
        "  if fileName == None:\n",
        "    fileName = 'model_parameters_'+str(np.round(np.random.randn(1000,200)[0][0]*1000))\n",
        "\n",
        "  model_summary = pd.DataFrame(columns =[])\n",
        "  for i in sorted(model_parameters_dict.keys()):\n",
        "    param_size = 0\n",
        "    if models != None:\n",
        "      param_size = get_n_params(models[i][0])\n",
        "      param_size += get_n_params(models[i][1])\n",
        "    row = pd.Series({\n",
        "                        'model_id':i,\n",
        "                        'encoder repeats' : model_parameters_dict[i][0]['repeats'],\n",
        "                        'decoder repeats' : model_parameters_dict[i][1]['repeats'],\n",
        "                        'parameters' :param_size\n",
        "    })\n",
        "    row_df = pd.DataFrame([row], index = [i])\n",
        "    model_summary = pd.concat([model_summary, row_df])\n",
        "  model_summary.to_csv(fileName)\n",
        "\n",
        "#Generate the list of repeat count parameters list\n",
        "\n",
        "#generate_model_parameters(generate_repeat_counts())\n",
        "\n",
        "def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):\n",
        "    torch.save(state, filename)\n",
        "    if is_best:\n",
        "        shutil.copyfile(filename, 'model_best.pth.tar')\n",
        "\n",
        "def save_dict(dictionary, filename):\n",
        "  np.save(filename, dictionary)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "plt.switch_backend('agg')\n",
        "import matplotlib.ticker as ticker\n",
        "import numpy as np\n",
        "\n",
        "def showPlot(points, points2):\n",
        "    plt.figure()\n",
        "    fig, ax = plt.subplots()\n",
        "    # this locator puts ticks at regular intervals\n",
        "    loc = ticker.MultipleLocator(base=0.2)\n",
        "    ax.yaxis.set_major_locator(loc)\n",
        "    plt.xlabel('Iteration')\n",
        "    plt.ylabel('Loss')\n",
        "    plt.plot(points)\n",
        "    plt.plot(points2)\n",
        "\n",
        "def asMinutes(s):\n",
        "    m = math.floor(s / 60)\n",
        "    s -= m * 60\n",
        "    return '%dm %ds' % (m, s)\n",
        "\n",
        "def timeSince(since, percent):\n",
        "    now = time.time()\n",
        "    s = now - since\n",
        "    es = s / (percent)\n",
        "    rs = es - s\n",
        "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RmGEoDTZX3f5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}