fed-esn.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KmntjQoJ2obO",
        "outputId": "208a58d3-b70a-42a6-fcbc-2ca33bb5103d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import pickle\n",
        "import pathlib\n",
        "import datetime\n",
        "\n",
        "from typing import Optional, Callable, Union\n",
        "\n",
        "import torch\n",
        "from torch import Tensor, Size\n",
        "import torch.nn.functional as F\n",
        "from torch.nn import Module, Parameter\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "import numpy as np\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "BASE_DIR = \"/content/drive/MyDrive/torch-esn-stress\"\n",
        "WESAD_USERS = {\n",
        "    'train': {\n",
        "        25: [0, 3, 5],\n",
        "        50: [0, 3, 5, 6, 9],\n",
        "        75: [0, 3, 5, 6, 9, 10, 12],\n",
        "        100: [0, 3, 5, 6, 9, 10, 12, 13, 14]\n",
        "    }, \n",
        "    'valid': [1, 8, 11],\n",
        "    'test': [2, 4, 7]\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uUQe3eDq4wOk"
      },
      "outputs": [],
      "source": [
        "#!wget https://uni-siegen.sciebo.de/s/HGdUkoNlW1Ub0Gx/download -O WESAD.zip\n",
        "#!unzip \"/content/WESAD.zip\" -d \"/content/drive/MyDrive/torch-esn-stress/data/raw\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wdNUm5UiQMgT"
      },
      "source": [
        "# **Reservoir**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5TD-XDnG0ldD"
      },
      "outputs": [],
      "source": [
        "def uniform(size: Size, rho: Optional[float] = None, sigma: Optional[float] = None,\n",
        "            scale: Optional[float] = None) -> Tensor:\n",
        "    \"\"\"\n",
        "    Uniform random tensor\n",
        "    Can either be rescaled according to spectral radius `rho`, spectral norm `sigma`, or `scale`.\n",
        "    :param size: Size of tensor\n",
        "    :param rho: Spectral radius\n",
        "    :param sigma: Spectral norm\n",
        "    :param scale: Simple rescaling of the standard random matrix\n",
        "    :return: A random tensor\n",
        "    \"\"\"\n",
        "    W = torch.empty(size).uniform_(-1, 1)\n",
        "    rescale_(W, rho, sigma, scale)\n",
        "    return W.data\n",
        "\n",
        "\n",
        "def normal(size: Size, rho: Optional[float] = None, sigma: Optional[float] = None,\n",
        "           scale: Optional[float] = None) -> Tensor:\n",
        "    \"\"\"\n",
        "    Normal random tensor\n",
        "    Can either be rescaled according to spectral radius `rho`, spectral norm `sigma`, or `scale`.\n",
        "    :param size: Size of tensor\n",
        "    :param rho: Spectral radius\n",
        "    :param sigma: Spectral norm\n",
        "    :param scale: Simple rescaling of the standard random matrix\n",
        "    :return: A random tensor\n",
        "    \"\"\"\n",
        "    W = torch.empty(size).normal_(mean=0, std=1)\n",
        "    rescale_(W, rho, sigma, scale)\n",
        "    return W.data\n",
        "\n",
        "\n",
        "def ring(size: Size, rho: Optional[float] = None, sigma: Optional[float] = None,\n",
        "         scale: Optional[float] = None) -> Tensor:\n",
        "    \"\"\"\n",
        "    Ring matrix\n",
        "    See:\n",
        "    C. Gallicchio & A. Micheli (2020). Ring Reservoir Neural Networks for Graphs.\n",
        "    In 2020 International Joint Conference on Neural Networks (IJCNN), IEEE.\n",
        "    https://doi.org/10.1109/IJCNN48605.2020.9206723\n",
        "    :param size: Size of tensor (must be square)\n",
        "    :param rho: Spectral radius (equivalent to others)\n",
        "    :param sigma: Spectral norm (equivalent to others)\n",
        "    :param scale: Simple rescaling of the matrix (equivalent to others)\n",
        "    :return: A re-scaled ring matrix\n",
        "    \"\"\"\n",
        "    assert (len(size) == 2) and (size[0] == size[1])\n",
        "    assert any(arg is not None for arg in [rho, sigma, scale])\n",
        "    if scale is None:\n",
        "        scale = rho if sigma is None else sigma\n",
        "    W = torch.eye(size[0]).roll(1, 0) * scale\n",
        "    return W.data\n",
        "\n",
        "\n",
        "def orthogonal(size: Size, rho: Optional[float] = None, sigma: Optional[float] = None,\n",
        "               scale: Optional[float] = None) -> Tensor:\n",
        "    \"\"\"\n",
        "    Orthogonal matrix\n",
        "    See:\n",
        "    F. Mezzadri (2007). How to Generate Random Matrices from the Classical Compact Groups.\n",
        "    Notices of the American Mathematical Society, 54(5), pp. 592-604.\n",
        "    https://www.ams.org/notices/200705/fea-mezzadri-web.pdf\n",
        "    :param size: Size of tensor (if not square, generates a semi-orthogonal matrix)\n",
        "    :param rho: Spectral radius (equivalent to others)\n",
        "    :param sigma: Spectral norm (equivalent to others)\n",
        "    :param scale: Simple rescaling of the matrix (equivalent to others)\n",
        "    :return: A re-scaled orthogonal matrix\n",
        "    \"\"\"\n",
        "    assert any(arg is not None for arg in [rho, sigma, scale])\n",
        "    if scale is None:\n",
        "        scale = rho if sigma is None else sigma\n",
        "    W = torch.empty(size)\n",
        "    torch.nn.init.orthogonal_(W, scale)\n",
        "    return W.data\n",
        "\n",
        "\n",
        "def ones(size: Size, rho: Optional[float] = None, sigma: Optional[float] = None,\n",
        "         scale: Optional[float] = None) -> Tensor:\n",
        "    \"\"\"\n",
        "    Ones tensor\n",
        "    Can either be rescaled according to spectral radius `rho`, spectral norm `sigma`, or `scale`.\n",
        "    :param size: Size of tensor\n",
        "    :param rho: Spectral radius\n",
        "    :param sigma: Spectral norm\n",
        "    :param scale: Simple rescaling of the standard random matrix\n",
        "    :return: A random tensor\n",
        "    \"\"\"\n",
        "    W = torch.ones(size)\n",
        "    rescale_(W, rho, sigma, scale)\n",
        "    return W.data\n",
        "\n",
        "\n",
        "def zeros(size: Size, rho: Optional[float] = None, sigma: Optional[float] = None,\n",
        "          scale: Optional[float] = None) -> Tensor:\n",
        "    \"\"\"\n",
        "    Zeros tensor\n",
        "    Rescaling is meaningless in this case.\n",
        "    :param size: Size of tensor\n",
        "    :param rho: Spectral radius\n",
        "    :param sigma: Spectral norm\n",
        "    :param scale: Simple rescaling of the standard random matrix\n",
        "    :return: A random tensor\n",
        "    \"\"\"\n",
        "    W = torch.zeros(size)\n",
        "    return W.data\n",
        "\n",
        "\n",
        "def rescale_(W: Tensor, rho: Optional[float] = None, sigma: Optional[float] = None,\n",
        "             scale: Optional[float] = None) -> Tensor:\n",
        "    \"\"\"\n",
        "    Rescale a matrix in-place\n",
        "    Can either be rescaled according to spectral radius `rho`, spectral norm `sigma`, or `scale`.\n",
        "    :param W: Matrix to rescale\n",
        "    :param rho: Spectral radius\n",
        "    :param sigma: Spectral norm\n",
        "    :param scale: Simple rescaling of the standard random matrix\n",
        "    :return: Rescaled matrix\n",
        "    \"\"\"\n",
        "    if rho is not None:\n",
        "        return W.div_(torch.linalg.eigvals(W).abs().max()).mul_(rho).float()\n",
        "    elif sigma is not None:\n",
        "        return W.div_(torch.linalg.matrix_norm(W, ord=2)).mul_(sigma).float()\n",
        "    elif scale is not None:\n",
        "        return W.mul_(scale).float()\n",
        "\n",
        "\n",
        "\n",
        "class Reservoir(Module):\n",
        "    \"\"\"\n",
        "    A Reservoir for Echo State Networks\n",
        "    \n",
        "    Args:\n",
        "        input_size: the number of expected features in the input `x`\n",
        "        hidden_size: the number of features in the hidden state `h`\n",
        "        activation: name of the activation function from `torch` (e.g. `torch.tanh`)\n",
        "        leakage: the value of the leaking parameter `alpha`\n",
        "        input_scaling: the value for the desired scaling of the input (must be `<= 1`)\n",
        "        rho: the desired spectral radius of the recurrent matrix (must be `< 1`)\n",
        "        bias: if ``False``, the layer does not use bias weights `b`\n",
        "        mode: execution mode of the reservoir (vanilla or intrinsic plasticity)\n",
        "        kernel_initializer: the kind of initialization of the input transformation. Default: `'uniform'`\n",
        "        recurrent_initializer: the kind of initialization of the recurrent matrix. Default: `'normal'`\n",
        "        net_gain_and_bias: if ``True``, the network uses additional ``g`` (gain) and ``b`` (bias) parameters. Default: ``False`` \n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self,\n",
        "                 input_size: int, \n",
        "                 hidden_size: int,\n",
        "                 activation: str = 'tanh',\n",
        "                 leakage: float = 1.,\n",
        "                 input_scaling: float = 0.9,\n",
        "                 rho: float = 0.99,\n",
        "                 bias: bool = False,\n",
        "                 kernel_initializer: Union[str, Callable[[Size], Tensor]] = 'uniform',\n",
        "                 recurrent_initializer: Union[str, Callable[[Size], Tensor]] = 'normal',\n",
        "                 net_gain_and_bias: bool = False) -> None:\n",
        "        \n",
        "        super().__init__()\n",
        "        assert rho < 1 and input_scaling <= 1\n",
        "\n",
        "        self.input_scaling = Parameter(torch.tensor(input_scaling), requires_grad=False)\n",
        "        self.rho = Parameter(torch.tensor(rho), requires_grad=False)\n",
        "\n",
        "        self.W_in = Parameter(\n",
        "            init_params(kernel_initializer, scale=input_scaling)([hidden_size, input_size]), \n",
        "            requires_grad=False\n",
        "        ) \n",
        "        self.W_hat = Parameter(\n",
        "            init_params(recurrent_initializer, rho=rho)([hidden_size, hidden_size]),\n",
        "            requires_grad=False\n",
        "        )\n",
        "        self.b = Parameter(\n",
        "            init_params('uniform', scale=input_scaling)(hidden_size), \n",
        "            requires_grad=False\n",
        "        ) if bias else None\n",
        "        self.f = getattr(torch, activation)\n",
        "\n",
        "        self.alpha = Parameter(torch.tensor(leakage), requires_grad=False)\n",
        "\n",
        "        self.net_gain_and_bias = net_gain_and_bias\n",
        "        if net_gain_and_bias:\n",
        "            self.net_a = Parameter(\n",
        "                init_params('ones')(hidden_size),\n",
        "                requires_grad=True\n",
        "            )\n",
        "            self.net_b = Parameter(\n",
        "                init_params('zeros')(hidden_size),\n",
        "                requires_grad=True\n",
        "            )\n",
        "    \n",
        "    @torch.no_grad()\n",
        "    def forward(self, input: Tensor, initial_state: Optional[Tensor] = None, mask: Optional[Tensor] = None) -> Tensor:\n",
        "        if initial_state is None:\n",
        "            initial_state = torch.zeros(self.hidden_size).to(self.W_hat)\n",
        "        \n",
        "        embeddings = torch.stack([state for state in self._state_comp(input.to(self.W_hat), initial_state, mask)], dim=0)\n",
        "        \n",
        "        return embeddings\n",
        "\n",
        "    def _state_comp(self, input: Tensor, initial_state: Tensor, mask: Optional[Tensor] = None):\n",
        "        timesteps = input.shape[0]\n",
        "        state = initial_state\n",
        "        for t in range(timesteps):\n",
        "            in_signal_t = F.linear(input[t].to(self.W_in), self.W_in, self.b) + F.linear(state, self.W_hat)\n",
        "            if self.net_gain_and_bias:\n",
        "                in_signal_t = in_signal_t * self.net_a + self.net_b\n",
        "            h_t = torch.tanh(in_signal_t)\n",
        "            state = (1 - self.alpha) * state + self.alpha * h_t\n",
        "            yield state if mask is None else mask * state\n",
        "\n",
        "    @property\n",
        "    def input_size(self) -> int:\n",
        "        \"\"\"Input dimension\"\"\"\n",
        "        return self.W_in.shape[1]\n",
        "\n",
        "    @property\n",
        "    def hidden_size(self) -> int:\n",
        "        \"\"\"Reservoir state dimension\"\"\"\n",
        "        return self.W_hat.shape[1]\n",
        "\n",
        "\n",
        "def init_params(name: str, **options) -> Callable[[Size], Tensor]:\n",
        "    \"\"\"\n",
        "    Gets a random weight initializer\n",
        "    :param name: Name of the random matrix generator in `esn.initializers`\n",
        "    :param options: Random matrix generator options\n",
        "    :return: A random weight generator function\n",
        "    \"\"\"\n",
        "    #init = getattr(initializers, name)\n",
        "    #return lambda size: init(size, **options)\n",
        "    if name == 'uniform':\n",
        "      return lambda size: uniform(size, **options)\n",
        "    elif name == 'normal':\n",
        "      return lambda size: normal(size, **options)\n",
        "    elif name == 'ring':\n",
        "      return lambda size: ring(size, **options)\n",
        "    elif name == 'orthogonal':\n",
        "      return lambda size: orthogonal(size, **options)\n",
        "    elif name == 'ones':\n",
        "      return lambda size: ones(size, **options)\n",
        "    elif name == 'zeros':\n",
        "      return lambda size: zeros(size, **options)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2faq6AAYQGZY"
      },
      "source": [
        "# **WESAD_Dataset**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BKxjXbCdEiAn"
      },
      "outputs": [],
      "source": [
        "RAW_WESAD_PATH = '/content/drive/MyDrive/torch-esn-stress/data/raw/WESAD'\n",
        "WESAD_PATH = '/content/drive/MyDrive/torch-esn-stress/data/processed/WESAD'\n",
        "USERS = [\"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\", \"11\", \"13\", \"14\", \"15\", \"16\", \"17\"]\n",
        "\n",
        "\n",
        "class WESAD_Dataset(torch.utils.data.Dataset):\n",
        "\n",
        "    def __init__(self, idx: int) -> None:\n",
        "        super().__init__()\n",
        "        self.user = USERS[idx]\n",
        "\n",
        "        u_path = os.path.join(WESAD_PATH, f'{self.user}.pkl')\n",
        "        if not os.path.exists(u_path):\n",
        "            self.user_data = self.preprocess()\n",
        "        else:\n",
        "            self.user_data = pickle.load(open(u_path, 'rb'))\n",
        "\n",
        "        self._seq_length = None\n",
        "        self.X, self.Y = None, None\n",
        "\n",
        "    @property\n",
        "    def seq_length(self):\n",
        "        return self._seq_length\n",
        "\n",
        "    @seq_length.setter\n",
        "    def seq_length(self, new_length: int):\n",
        "        if self._seq_length is None or new_length != self._seq_length:\n",
        "            print(f\"Setting the length of the chunks in WESAD user {self.user} from {self._seq_length} to {new_length}\")\n",
        "            self.X, self.Y = self._to_sequence_chunks(new_length)\n",
        "            self._seq_length = new_length\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.X.shape[1]\n",
        "\n",
        "    def __getitem__(self, i: int):\n",
        "        return self.X[:, i], self.Y[:, i]\n",
        "    \n",
        "    def _to_sequence_chunks(self, length: int):\n",
        "        X, Y = torch.split(self.user_data['X'], length, dim=0), torch.split(self.user_data['Y'], length, dim=0)\n",
        "        if X[-1].shape[0] != length:\n",
        "            X, Y = X[:-1], Y[:-1]\n",
        "\n",
        "        return torch.stack(X, dim=1), torch.stack(Y, dim=1)\n",
        "\n",
        "    def preprocess(self):\n",
        "        print(f\"Preprocessing user {self.user}...\")\n",
        "        with open(os.path.join(RAW_WESAD_PATH, f'S{self.user}', f'S{self.user}.pkl'), 'rb') as f:\n",
        "            data = pickle.load(f, encoding='latin1')\n",
        "        X = np.concatenate([\n",
        "            data['signal']['chest']['ACC'],\n",
        "            data['signal']['chest']['Resp'],\n",
        "            data['signal']['chest']['EDA'],\n",
        "            data['signal']['chest']['ECG'],\n",
        "            data['signal']['chest']['EMG'],\n",
        "            data['signal']['chest']['Temp'],\n",
        "        ], axis=1)\n",
        "        Y = data['label']\n",
        "        X = X[(Y>0) & (Y<5)]\n",
        "        X = torch.tensor((X - np.mean(X, axis=0)) / np.std(X, axis=0))\n",
        "        Y = Y[(Y>0) & (Y<5)]\n",
        "        Y = F.one_hot(torch.tensor(Y-1, dtype=torch.int64), num_classes=4)\n",
        "        u_dict = {'X': X, 'Y': Y}\n",
        "        print(f\"Preprocessing of user {self.user} complete!\")\n",
        "\n",
        "        print(f\"Saving preprocessed data of user {self.user}...\")\n",
        "        pickle.dump(u_dict, open(os.path.join(WESAD_PATH, f'{self.user}.pkl'), 'wb+'))\n",
        "        print(f\"Preprocessed data of user {self.user} saved!\")\n",
        "\n",
        "        return u_dict\n",
        "\n",
        "\n",
        "def seq_collate_fn(batch):\n",
        "  x, y = [], []\n",
        "  for x_i, y_i in batch:\n",
        "      #x.append(x_i[:,2].view(-1,1))\n",
        "      x.append(x_i)\n",
        "      y.append(y_i)\n",
        "  return torch.cat(x).float(), torch.cat(y).float()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "76lxAmXoDwzr"
      },
      "source": [
        "# **Federated Learning**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PTjJOYinDxta"
      },
      "outputs": [],
      "source": [
        "def train(Nu, Nr, Ny, seq_len, perc, lamb, same_res):\n",
        "  SERVER = {\n",
        "    'SEQ_LENGTH': seq_len, 'PERCENTAGE': perc,\n",
        "    'Nr': Nr, 'LAMBDA': lamb, 'SAME_RESERVOIR': same_res,\n",
        "\n",
        "    'reservoir': Reservoir(input_size=Nu, hidden_size=Nr),\n",
        "    'S_train': None, 'Y_train': None,\n",
        "    'S_valid': None, 'Y_valid': None, 'x_valid': None,\n",
        "    'S_test': None, 'Y_test': None, 'x_test': None,\n",
        "\n",
        "    'clients': []\n",
        "  }\n",
        "\n",
        "  CLIENTS = []\n",
        "  for u in WESAD_USERS['train'][perc]:\n",
        "    data = WESAD_Dataset(u)\n",
        "    data.seq_length = seq_len\n",
        "    data_loader = DataLoader(data, batch_size=1, collate_fn=seq_collate_fn)\n",
        "    data_iter = iter(data_loader)\n",
        "    if same_res:\n",
        "      reservoir = SERVER['reservoir']\n",
        "    else:\n",
        "      reservoir = Reservoir(input_size=Nu, hidden_size=Nr)\n",
        "\n",
        "    CLIENTS.append({\n",
        "        'id': u, 'data': data_iter, 'reservoir': reservoir,\n",
        "        'x': None, 'A': torch.zeros(Ny,Nr), 'B': torch.zeros(Nr,Nr),\n",
        "        'W': None, 'W_best': None, 'W_rand': None,\n",
        "        'S': None, 'Y': None, 'finished': False\n",
        "    })\n",
        "\n",
        "  device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "  print(device)\n",
        "  i = 0\n",
        "  while not all([c['finished'] for c in CLIENTS]):\n",
        "    for client in CLIENTS:\n",
        "      if not client['finished']:\n",
        "        id = client['id']\n",
        "        data = client['data']\n",
        "        reservoir = client['reservoir']\n",
        "\n",
        "        batch = next(data, None)\n",
        "        if batch is not None:\n",
        "          u, t = batch\n",
        "          embeddings = reservoir(u, initial_state=client['x']).float()\n",
        "          client['x'] = embeddings[-1]\n",
        "          x = embeddings[-1].view(-1,1) #(Nr, 1)\n",
        "          y = t[-1].view(-1,1)          #(Ny, 1)\n",
        "          client['S'] = x if client['S'] is None else torch.hstack((client['S'],x))\n",
        "          client['Y'] = y if client['Y'] is None else torch.hstack((client['Y'],y))\n",
        "\n",
        "          A_old, B_old = client['A'], client['B']\n",
        "          x, y = x.to(device), y.to(device)\n",
        "          A_new, B_new = y @ x.T, x @ x.T\n",
        "          A = A_old.to(A_new) + A_new\n",
        "          B = B_old.to(B_new) + B_new\n",
        "          client['A'] = A.cpu()\n",
        "          client['B'] = B.cpu()\n",
        "\n",
        "        else:\n",
        "          client['finished'] = True\n",
        "          print(f\"Client {id} finished!\")\n",
        "    \n",
        "    i += 1\n",
        "    print(\"Round\",i)\n",
        "\n",
        "  for client in CLIENTS: client['data']=None\n",
        "  SERVER['clients'] = CLIENTS\n",
        "  return SERVER\n",
        "\n",
        "################################################################################\n",
        "\n",
        "Nu, Ny = 8, 4\n",
        "for perc in [100, 75, 50, 25]:\n",
        "  for seq_len in [350, 700]:\n",
        "    for Nr in [100, 250, 500]:\n",
        "      for lamb in [1e-6]:\n",
        "        print(f'----seq_len={seq_len}, perc={perc}, Nr={Nr}, lamb={lamb}, same_res=True')\n",
        "        file_path = f'{BASE_DIR}/exp/server_{seq_len}_{perc}_{Nr}_{lamb}_same.pkl'\n",
        "        if not os.path.exists(file_path):\n",
        "          s = train(Nu, Nr, Ny, seq_len, perc, lamb, same_res=True)\n",
        "          pickle.dump(s, open(file_path, \"wb\"))\n",
        "        else:\n",
        "          print(f'File {file_path} exists!')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Who242tBU6Ee"
      },
      "source": [
        "# **Results**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dqun_QxTsP8N",
        "outputId": "30c0cc75-4f72-4e78-87ee-bf37b87dd467"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best IncFed params: {'alpha': 0.3, 'seq_len': 350, 'perc': 100, 'Nr': 500, 'lamb': 1e-06, 'same_res': True, 'n_clients': 9}\n",
            "Best FedImp params: {'alpha': 0.3, 'seq_len': 350, 'perc': 100, 'Nr': 500, 'lamb': 1e-06, 'same_res': True, 'n_clients': 9}\n",
            "Best Random params: {'alpha': 1, 'seq_len': 350, 'perc': 100, 'Nr': 500, 'lamb': 1e-06, 'same_res': True, 'n_clients': 9}\n",
            "Best FedAvg params: {'alpha': 0.3, 'seq_len': 350, 'perc': 100, 'Nr': 500, 'lamb': 1e-06, 'same_res': True, 'n_clients': 9}\n",
            "\n",
            "     \t IncFed \t\t\t\t FedImp \t\t\t\t Random \t\t\t\t FedAvg\n",
            "     \t TR         | VL        | TS \t\t TR         | VL        | TS \t\t TR         | VL        | TS \t\t TR         | VL        | TS\n",
            "100% \t 0.8988±0.57|0.7546±0.82|0.7801±0.77 \t 0.7370±1.01|0.7361±0.97|0.8128±0.70 \t 0.7280±1.04|0.7516±0.96|0.8084±0.74 \t 0.7388±0.95|0.7532±0.99|0.8170±0.66\n",
            "75%  \t 0.9170±0.54|0.7469±0.90|0.7504±0.81 \t 0.7177±1.07|0.7008±1.07|0.8108±0.76 \t 0.6962±1.10|0.6647±1.19|0.7887±0.86 \t 0.7024±1.00|0.7627±0.90|0.8052±0.72\n",
            "50%  \t 0.9369±0.49|0.7450±0.88|0.7223±0.80 \t 0.6978±1.11|0.5853±1.26|0.7642±0.93 \t 0.6869±1.13|0.5457±1.34|0.7600±0.95 \t 0.7189±0.99|0.6872±0.97|0.7929±0.75\n",
            "25%  \t 0.9675±0.40|0.6719±1.13|0.7093±0.89 \t 0.6882±0.98|0.5449±1.36|0.7565±0.96 \t 0.6809±0.99|0.5145±1.39|0.7480±1.01 \t 0.7437±0.94|0.6467±1.18|0.7476±0.90\n",
            "\n"
          ]
        }
      ],
      "source": [
        "def exec_experiments(alpha):\n",
        "  results = []\n",
        "  for file in os.listdir(f'{BASE_DIR}/exp'):\n",
        "    if file.endswith(\".pkl\"):\n",
        "      result = {}\n",
        "      server = pickle.load(open(f'{BASE_DIR}/exp/{file}',\"rb\"))\n",
        "      clients = server['clients']\n",
        "      params = file.split('_')\n",
        "      seq_len, perc, Nr, lamb, same_res = int(params[1]), int(params[2]), int(params[3]), float(params[4]), params[5]==\"same.pkl\"\n",
        "      result['params'] = {\n",
        "          'alpha': alpha,\n",
        "          'seq_len': seq_len, 'perc': perc, 'Nr': Nr, \n",
        "          'lamb': lamb, 'same_res': same_res, 'n_clients': len(clients)\n",
        "      }\n",
        "\n",
        "      if server['S_valid'] is None:\n",
        "        valid_data = [WESAD_Dataset(u) for u in WESAD_USERS['valid']]\n",
        "        for d in valid_data: d.seq_length=seq_len\n",
        "        valid_loaders = [DataLoader(d, batch_size=1, collate_fn=seq_collate_fn) for d in valid_data]\n",
        "        for loader in valid_loaders:\n",
        "          for u, t in loader:\n",
        "            embeddings = server['reservoir'](u, initial_state=server['x_valid']).float()\n",
        "            server['x_valid'] = embeddings[-1]\n",
        "            x, y = embeddings[-1].view(-1,1), t[-1].view(-1,1)\n",
        "            server['S_valid'] = x if server['S_valid'] is None else torch.hstack((server['S_valid'],x))\n",
        "            server['Y_valid'] = y if server['Y_valid'] is None else torch.hstack((server['Y_valid'],y))\n",
        "        pickle.dump(server, open(f'{BASE_DIR}/exp/{file}', \"wb\"))\n",
        "      \n",
        "      if server['S_test'] is None:\n",
        "        test_data = [WESAD_Dataset(u) for u in WESAD_USERS['test']]\n",
        "        for d in test_data: d.seq_length=seq_len\n",
        "        test_loaders = [DataLoader(d, batch_size=1, collate_fn=seq_collate_fn) for d in test_data]\n",
        "        for loader in test_loaders:\n",
        "          for u, t in loader:\n",
        "            embeddings = server['reservoir'](u, initial_state=server['x_test']).float()\n",
        "            server['x_test'] = embeddings[-1]\n",
        "            x, y = embeddings[-1].view(-1,1), t[-1].view(-1,1)\n",
        "            server['S_test'] = x if server['S_test'] is None else torch.hstack((server['S_test'],x))\n",
        "            server['Y_test'] = y if server['Y_test'] is None else torch.hstack((server['Y_test'],y))\n",
        "        pickle.dump(server, open(f'{BASE_DIR}/exp/{file}', \"wb\"))\n",
        "\n",
        "      S_train = torch.cat([c['S'] for c in clients], dim=1)\n",
        "      Y_train = torch.cat([c['Y'] for c in clients], dim=1)\n",
        "      S_valid, Y_valid = server['S_valid'], server['Y_valid']\n",
        "      S_test, Y_test = server['S_test'], server['Y_test']\n",
        "\n",
        "      # IncFed\n",
        "      A = torch.stack([c['A'] for c in clients], dim=0).sum(dim=0)\n",
        "      B = torch.stack([c['B'] for c in clients], dim=0).sum(dim=0)\n",
        "      B += lamb*torch.eye(B.size(0))\n",
        "      W_incfed = A @ B.pinverse()\n",
        "\n",
        "      # FedAvg\n",
        "      W_fedavg = torch.zeros(W_incfed.shape)\n",
        "      for client in clients:\n",
        "        W_fedavg += (client['A'] @ client['B'].pinverse())/client['S'].size(1)\n",
        "\n",
        "      #FedImp\n",
        "      p = (150/len(clients))/100\n",
        "      n = int(p*Nr)\n",
        "      k, k_rand = int(alpha*n), int((1-alpha)*n)\n",
        "      A_best, B_best = torch.zeros(A.shape), torch.zeros(B.shape)\n",
        "      for i, client in enumerate(clients):\n",
        "        A_client, B_client = client['A'], client['B']\n",
        "        A_client_best, B_client_best = torch.zeros(A_client.shape), torch.zeros(B_client.shape)\n",
        "\n",
        "        imp = torch.sum(B_client**2, axis=1)\n",
        "        idxs = list(range(imp.size(0)))\n",
        "        _, topk_idxs = torch.topk(imp, k)\n",
        "        topk_idxs = topk_idxs.tolist()\n",
        "\n",
        "        rand_idxs = list(set(idxs)-set(topk_idxs))\n",
        "        np.random.shuffle(rand_idxs)\n",
        "        rand_idxs = rand_idxs[:k_rand]\n",
        "        tot_idxs = topk_idxs + rand_idxs\n",
        "\n",
        "        A_client_best[:,tot_idxs] = A_client[:,tot_idxs]\n",
        "        B_client_best[tot_idxs,tot_idxs] = B_client[tot_idxs,tot_idxs]\n",
        "        A_best += A_client_best\n",
        "        B_best += B_client_best\n",
        "\n",
        "      B_best += lamb*torch.eye(B_best.size(0))\n",
        "      W_fedimp = A_best @ B_best.pinverse()\n",
        "\n",
        "      # Random\n",
        "      A_rand, B_rand = torch.zeros(A.shape), torch.zeros(B.shape)\n",
        "      for client in clients:\n",
        "        idxs = list(range(B_rand.size(0)))\n",
        "        np.random.shuffle(idxs)\n",
        "        idxs = idxs[:n]\n",
        "\n",
        "        A_client, B_client = client['A'], client['B']\n",
        "        A_client_rand, B_client_rand = torch.zeros(A_client.shape), torch.zeros(B_client.shape)\n",
        "        \n",
        "\n",
        "        A_client_rand[:,idxs] = A_client[:,idxs]\n",
        "        B_client_rand[idxs,idxs] = B_client[idxs,idxs]\n",
        "        A_rand += A_client_rand\n",
        "        B_rand += B_client_rand\n",
        "\n",
        "      B_rand += lamb*torch.eye(B_rand.size(0))\n",
        "      W_rand = A_rand @ B_rand.pinverse()\n",
        "\n",
        "      Y_pred_incfed = W_incfed @ S_train\n",
        "      err_incfed = (torch.argmax(Y_train, axis=0) - torch.argmax(Y_pred_incfed, axis=0)).float()\n",
        "      std_incfed = torch.std(err_incfed)\n",
        "      score_incfed = accuracy_score(torch.argmax(Y_train, axis=0),torch.argmax(Y_pred_incfed, axis=0))\n",
        "      Y_pred_fedavg = W_fedavg @ S_train\n",
        "      err_fedavg = (torch.argmax(Y_train, axis=0) - torch.argmax(Y_pred_fedavg, axis=0)).float()\n",
        "      std_fedavg = torch.std(err_fedavg)\n",
        "      score_fedavg = accuracy_score(torch.argmax(Y_train, axis=0),torch.argmax(Y_pred_fedavg, axis=0))\n",
        "      Y_pred_fedimp = W_fedimp @ S_train\n",
        "      err_fedimp = (torch.argmax(Y_train, axis=0) - torch.argmax(Y_pred_fedimp, axis=0)).float()\n",
        "      std_fedimp = torch.std(err_fedimp)\n",
        "      score_fedimp = accuracy_score(torch.argmax(Y_train, axis=0),torch.argmax(Y_pred_fedimp, axis=0))\n",
        "      Y_pred_rand = W_rand @ S_train\n",
        "      err_rand = (torch.argmax(Y_train, axis=0) - torch.argmax(Y_pred_rand, axis=0)).float()\n",
        "      std_rand = torch.std(err_rand)\n",
        "      score_rand = accuracy_score(torch.argmax(Y_train, axis=0),torch.argmax(Y_pred_rand, axis=0))\n",
        "      result['train'] = {\n",
        "          'incfed': score_incfed,\n",
        "          'fedavg': score_fedavg,\n",
        "          'fedimp': score_fedimp,\n",
        "          'random': score_rand,\n",
        "\n",
        "          'std_incfed': std_incfed,\n",
        "          'std_fedavg': std_fedavg,\n",
        "          'std_fedimp': std_fedimp,\n",
        "          'std_rand':   std_rand\n",
        "      }\n",
        "      \n",
        "      Y_pred_incfed = W_incfed @ S_valid\n",
        "      err_incfed = (torch.argmax(Y_valid, axis=0) - torch.argmax(Y_pred_incfed, axis=0)).float()\n",
        "      std_incfed = torch.std(err_incfed)\n",
        "      score_incfed = accuracy_score(torch.argmax(Y_valid, axis=0),torch.argmax(Y_pred_incfed, axis=0))\n",
        "      Y_pred_fedavg = W_fedavg @ S_valid\n",
        "      err_fedavg = (torch.argmax(Y_valid, axis=0) - torch.argmax(Y_pred_fedavg, axis=0)).float()\n",
        "      std_fedavg = torch.std(err_fedavg)\n",
        "      score_fedavg = accuracy_score(torch.argmax(Y_valid, axis=0),torch.argmax(Y_pred_fedavg, axis=0))\n",
        "      Y_pred_fedimp = W_fedimp @ S_valid\n",
        "      err_fedimp = (torch.argmax(Y_valid, axis=0) - torch.argmax(Y_pred_fedimp, axis=0)).float()\n",
        "      std_fedimp = torch.std(err_fedimp)\n",
        "      score_fedimp = accuracy_score(torch.argmax(Y_valid, axis=0),torch.argmax(Y_pred_fedimp, axis=0))\n",
        "      Y_pred_rand = W_rand @ S_valid\n",
        "      err_rand = (torch.argmax(Y_valid, axis=0) - torch.argmax(Y_pred_rand, axis=0)).float()\n",
        "      std_rand = torch.std(err_rand)\n",
        "      score_rand = accuracy_score(torch.argmax(Y_valid, axis=0),torch.argmax(Y_pred_rand, axis=0))\n",
        "      result['valid'] = {\n",
        "          'incfed': score_incfed,\n",
        "          'fedavg': score_fedavg,\n",
        "          'fedimp': score_fedimp,\n",
        "          'random': score_rand,\n",
        "\n",
        "          'std_incfed': std_incfed,\n",
        "          'std_fedavg': std_fedavg,\n",
        "          'std_fedimp': std_fedimp,\n",
        "          'std_rand':   std_rand\n",
        "      }\n",
        "      \n",
        "      Y_pred_incfed = W_incfed @ S_test\n",
        "      err_incfed = (torch.argmax(Y_test, axis=0) - torch.argmax(Y_pred_incfed, axis=0)).float()\n",
        "      std_incfed = torch.std(err_incfed)\n",
        "      score_incfed = accuracy_score(torch.argmax(Y_test, axis=0),torch.argmax(Y_pred_incfed, axis=0))\n",
        "      Y_pred_fedavg = W_fedavg @ S_test\n",
        "      err_fedavg = (torch.argmax(Y_test, axis=0) - torch.argmax(Y_pred_fedavg, axis=0)).float()\n",
        "      std_fedavg = torch.std(err_fedavg)\n",
        "      score_fedavg = accuracy_score(torch.argmax(Y_test, axis=0),torch.argmax(Y_pred_fedavg, axis=0))\n",
        "      Y_pred_fedimp = W_fedimp @ S_test\n",
        "      err_fedimp = (torch.argmax(Y_test, axis=0) - torch.argmax(Y_pred_fedimp, axis=0)).float()\n",
        "      std_fedimp = torch.std(err_fedimp)\n",
        "      score_fedimp = accuracy_score(torch.argmax(Y_test, axis=0),torch.argmax(Y_pred_fedimp, axis=0))\n",
        "      Y_pred_rand = W_rand @ S_test\n",
        "      err_rand = (torch.argmax(Y_test, axis=0) - torch.argmax(Y_pred_rand, axis=0)).float()\n",
        "      std_rand = torch.std(err_rand)\n",
        "      score_rand = accuracy_score(torch.argmax(Y_test, axis=0),torch.argmax(Y_pred_rand, axis=0))\n",
        "      result['test'] = {\n",
        "          'incfed': score_incfed,\n",
        "          'fedavg': score_fedavg,\n",
        "          'fedimp': score_fedimp,\n",
        "          'random': score_rand,\n",
        "\n",
        "          'std_incfed': std_incfed,\n",
        "          'std_fedavg': std_fedavg,\n",
        "          'std_fedimp': std_fedimp,\n",
        "          'std_rand':   std_rand\n",
        "      }\n",
        "      results.append(result)\n",
        "    \n",
        "  return results\n",
        "\n",
        "\n",
        "r = []\n",
        "for a in [0.3, 0.6, 0.8, 1]:\n",
        "    res = exec_experiments(a)\n",
        "    r = r+res\n",
        "\n",
        "\n",
        "results_100 = [e for e in r if e['params']['perc']==100]\n",
        "best_incfed_idx = np.argmax([e['valid']['incfed'] for e in results_100])\n",
        "best_incfed_100 = results_100[best_incfed_idx]\n",
        "best_incfed_100_train       = best_incfed_100['train']['incfed']\n",
        "best_incfed_100_train_std   = best_incfed_100['train']['std_incfed']\n",
        "best_incfed_100_valid       = best_incfed_100['valid']['incfed']\n",
        "best_incfed_100_valid_std   = best_incfed_100['valid']['std_incfed']\n",
        "best_incfed_100_test        = best_incfed_100['test']['incfed']\n",
        "best_incfed_100_test_std    = best_incfed_100['test']['std_incfed']\n",
        "\n",
        "best_incfed_75 = [e for e in r if e['params']['perc']==75 and e['params']['alpha']==best_incfed_100['params']['alpha'] and e['params']['seq_len']==best_incfed_100['params']['seq_len'] and e['params']['Nr']==best_incfed_100['params']['Nr']][0]\n",
        "best_incfed_75_train      = best_incfed_75['train']['incfed']\n",
        "best_incfed_75_train_std  = best_incfed_75['train']['std_incfed']\n",
        "best_incfed_75_valid      = best_incfed_75['valid']['incfed']\n",
        "best_incfed_75_valid_std  = best_incfed_75['valid']['std_incfed']\n",
        "best_incfed_75_test       = best_incfed_75['test']['incfed']\n",
        "best_incfed_75_test_std   = best_incfed_75['test']['std_incfed']\n",
        "\n",
        "best_incfed_50 = [e for e in r if e['params']['perc']==50 and e['params']['alpha']==best_incfed_100['params']['alpha'] and e['params']['seq_len']==best_incfed_100['params']['seq_len'] and e['params']['Nr']==best_incfed_100['params']['Nr']][0]\n",
        "best_incfed_50_train      = best_incfed_50['train']['incfed']\n",
        "best_incfed_50_train_std  = best_incfed_50['train']['std_incfed']\n",
        "best_incfed_50_valid      = best_incfed_50['valid']['incfed']\n",
        "best_incfed_50_valid_std  = best_incfed_50['valid']['std_incfed']\n",
        "best_incfed_50_test       = best_incfed_50['test']['incfed']\n",
        "best_incfed_50_test_std   = best_incfed_50['test']['std_incfed']\n",
        "\n",
        "best_incfed_25 = [e for e in r if e['params']['perc']==25 and e['params']['alpha']==best_incfed_100['params']['alpha'] and e['params']['seq_len']==best_incfed_100['params']['seq_len'] and e['params']['Nr']==best_incfed_100['params']['Nr']][0]\n",
        "best_incfed_25_train      = best_incfed_25['train']['incfed']\n",
        "best_incfed_25_train_std  = best_incfed_25['train']['std_incfed']\n",
        "best_incfed_25_valid      = best_incfed_25['valid']['incfed']\n",
        "best_incfed_25_valid_std  = best_incfed_25['valid']['std_incfed']\n",
        "best_incfed_25_test       = best_incfed_25['test']['incfed']\n",
        "best_incfed_25_test_std   = best_incfed_25['test']['std_incfed']\n",
        "\n",
        "best_fedimp_idx = np.argmax([e['valid']['fedimp'] for e in results_100])\n",
        "best_fedimp_100 = results_100[best_fedimp_idx]\n",
        "best_fedimp_100_train       = best_fedimp_100['train']['fedimp']\n",
        "best_fedimp_100_train_std   = best_fedimp_100['train']['std_fedimp']\n",
        "best_fedimp_100_valid       = best_fedimp_100['valid']['fedimp']\n",
        "best_fedimp_100_valid_std   = best_fedimp_100['valid']['std_fedimp']\n",
        "best_fedimp_100_test        = best_fedimp_100['test']['fedimp']\n",
        "best_fedimp_100_test_std    = best_fedimp_100['test']['std_fedimp']\n",
        "\n",
        "best_fedimp_75 = [e for e in r if e['params']['perc']==75 and e['params']['alpha']==best_fedimp_100['params']['alpha'] and e['params']['seq_len']==best_fedimp_100['params']['seq_len'] and e['params']['Nr']==best_fedimp_100['params']['Nr']][0]\n",
        "best_fedimp_75_train       = best_fedimp_75['train']['fedimp']\n",
        "best_fedimp_75_train_std   = best_fedimp_75['train']['std_fedimp']\n",
        "best_fedimp_75_valid       = best_fedimp_75['valid']['fedimp']\n",
        "best_fedimp_75_valid_std   = best_fedimp_75['valid']['std_fedimp']\n",
        "best_fedimp_75_test        = best_fedimp_75['test']['fedimp']\n",
        "best_fedimp_75_test_std    = best_fedimp_75['test']['std_fedimp']\n",
        "\n",
        "best_fedimp_50 = [e for e in r if e['params']['perc']==50 and e['params']['alpha']==best_fedimp_100['params']['alpha'] and e['params']['seq_len']==best_fedimp_100['params']['seq_len'] and e['params']['Nr']==best_fedimp_100['params']['Nr']][0]\n",
        "best_fedimp_50_train       = best_fedimp_50['train']['fedimp']\n",
        "best_fedimp_50_train_std   = best_fedimp_50['train']['std_fedimp']\n",
        "best_fedimp_50_valid       = best_fedimp_50['valid']['fedimp']\n",
        "best_fedimp_50_valid_std   = best_fedimp_50['valid']['std_fedimp']\n",
        "best_fedimp_50_test        = best_fedimp_50['test']['fedimp']\n",
        "best_fedimp_50_test_std    = best_fedimp_50['test']['std_fedimp']\n",
        "\n",
        "best_fedimp_25 = [e for e in r if e['params']['perc']==25 and e['params']['alpha']==best_fedimp_100['params']['alpha'] and e['params']['seq_len']==best_fedimp_100['params']['seq_len'] and e['params']['Nr']==best_fedimp_100['params']['Nr']][0]\n",
        "best_fedimp_25_train       = best_fedimp_25['train']['fedimp']\n",
        "best_fedimp_25_train_std   = best_fedimp_25['train']['std_fedimp']\n",
        "best_fedimp_25_valid       = best_fedimp_25['valid']['fedimp']\n",
        "best_fedimp_25_valid_std   = best_fedimp_25['valid']['std_fedimp']\n",
        "best_fedimp_25_test        = best_fedimp_25['test']['fedimp']\n",
        "best_fedimp_25_test_std    = best_fedimp_25['test']['std_fedimp']\n",
        "\n",
        "best_rand_idx = np.argmax([e['valid']['random'] for e in results_100])\n",
        "best_rand_100 = results_100[best_rand_idx]\n",
        "best_rand_100_train       = best_rand_100['train']['random']\n",
        "best_rand_100_train_std   = best_rand_100['train']['std_rand']\n",
        "best_rand_100_valid       = best_rand_100['valid']['random']\n",
        "best_rand_100_valid_std   = best_rand_100['valid']['std_rand']\n",
        "best_rand_100_test        = best_rand_100['test']['random']\n",
        "best_rand_100_test_std    = best_rand_100['test']['std_rand']\n",
        "\n",
        "best_rand_75 = [e for e in r if e['params']['perc']==75 and e['params']['alpha']==best_rand_100['params']['alpha'] and e['params']['seq_len']==best_rand_100['params']['seq_len'] and e['params']['Nr']==best_rand_100['params']['Nr']][0]\n",
        "best_rand_75_train       = best_rand_75['train']['random']\n",
        "best_rand_75_train_std   = best_rand_75['train']['std_rand']\n",
        "best_rand_75_valid       = best_rand_75['valid']['random']\n",
        "best_rand_75_valid_std   = best_rand_75['valid']['std_rand']\n",
        "best_rand_75_test        = best_rand_75['test']['random']\n",
        "best_rand_75_test_std    = best_rand_75['test']['std_rand']\n",
        "\n",
        "best_rand_50 = [e for e in r if e['params']['perc']==50 and e['params']['alpha']==best_rand_100['params']['alpha'] and e['params']['seq_len']==best_rand_100['params']['seq_len'] and e['params']['Nr']==best_rand_100['params']['Nr']][0]\n",
        "best_rand_50_train       = best_rand_50['train']['random']\n",
        "best_rand_50_train_std   = best_rand_50['train']['std_rand']\n",
        "best_rand_50_valid       = best_rand_50['valid']['random']\n",
        "best_rand_50_valid_std   = best_rand_50['valid']['std_rand']\n",
        "best_rand_50_test        = best_rand_50['test']['random']\n",
        "best_rand_50_test_std    = best_rand_50['test']['std_rand']\n",
        "\n",
        "best_rand_25 = [e for e in r if e['params']['perc']==25 and e['params']['alpha']==best_rand_100['params']['alpha'] and e['params']['seq_len']==best_rand_100['params']['seq_len'] and e['params']['Nr']==best_rand_100['params']['Nr']][0]\n",
        "best_rand_25_train       = best_rand_25['train']['random']\n",
        "best_rand_25_train_std   = best_rand_25['train']['std_rand']\n",
        "best_rand_25_valid       = best_rand_25['valid']['random']\n",
        "best_rand_25_valid_std   = best_rand_25['valid']['std_rand']\n",
        "best_rand_25_test        = best_rand_25['test']['random']\n",
        "best_rand_25_test_std    = best_rand_25['test']['std_rand']\n",
        "\n",
        "best_fedavg_idx = np.argmax([e['valid']['fedavg'] for e in results_100])\n",
        "best_fedavg_100 = results_100[best_fedavg_idx]\n",
        "best_fedavg_100_train       = best_fedavg_100['train']['fedavg']\n",
        "best_fedavg_100_train_std   = best_fedavg_100['train']['std_fedavg']\n",
        "best_fedavg_100_valid       = best_fedavg_100['valid']['fedavg']\n",
        "best_fedavg_100_valid_std   = best_fedavg_100['valid']['std_fedavg']\n",
        "best_fedavg_100_test        = best_fedavg_100['test']['fedavg']\n",
        "best_fedavg_100_test_std    = best_fedavg_100['test']['std_fedavg']\n",
        "\n",
        "best_fedavg_75 = [e for e in r if e['params']['perc']==75 and e['params']['alpha']==best_fedavg_100['params']['alpha'] and e['params']['seq_len']==best_fedavg_100['params']['seq_len'] and e['params']['Nr']==best_fedavg_100['params']['Nr']][0]\n",
        "best_fedavg_75_train       = best_fedavg_75['train']['fedavg']\n",
        "best_fedavg_75_train_std   = best_fedavg_75['train']['std_fedavg']\n",
        "best_fedavg_75_valid       = best_fedavg_75['valid']['fedavg']\n",
        "best_fedavg_75_valid_std   = best_fedavg_75['valid']['std_fedavg']\n",
        "best_fedavg_75_test        = best_fedavg_75['test']['fedavg']\n",
        "best_fedavg_75_test_std    = best_fedavg_75['test']['std_fedavg']\n",
        "\n",
        "best_fedavg_50 = [e for e in r if e['params']['perc']==50 and e['params']['alpha']==best_fedavg_100['params']['alpha'] and e['params']['seq_len']==best_fedavg_100['params']['seq_len'] and e['params']['Nr']==best_fedavg_100['params']['Nr']][0]\n",
        "best_fedavg_50_train       = best_fedavg_50['train']['fedavg']\n",
        "best_fedavg_50_train_std   = best_fedavg_50['train']['std_fedavg']\n",
        "best_fedavg_50_valid       = best_fedavg_50['valid']['fedavg']\n",
        "best_fedavg_50_valid_std   = best_fedavg_50['valid']['std_fedavg']\n",
        "best_fedavg_50_test        = best_fedavg_50['test']['fedavg']\n",
        "best_fedavg_50_test_std    = best_fedavg_50['test']['std_fedavg']\n",
        "\n",
        "best_fedavg_25 = [e for e in r if e['params']['perc']==25 and e['params']['alpha']==best_fedavg_100['params']['alpha'] and e['params']['seq_len']==best_fedavg_100['params']['seq_len'] and e['params']['Nr']==best_fedavg_100['params']['Nr']][0]\n",
        "best_fedavg_25_train       = best_fedavg_25['train']['fedavg']\n",
        "best_fedavg_25_train_std   = best_fedavg_25['train']['std_fedavg']\n",
        "best_fedavg_25_valid       = best_fedavg_25['valid']['fedavg']\n",
        "best_fedavg_25_valid_std   = best_fedavg_25['valid']['std_fedavg']\n",
        "best_fedavg_25_test        = best_fedavg_25['test']['fedavg']\n",
        "best_fedavg_25_test_std    = best_fedavg_25['test']['std_fedavg']\n",
        "\n",
        "print(f\"Best IncFed params: {best_incfed_100['params']}\")\n",
        "print(f\"Best FedImp params: {best_fedimp_100['params']}\")\n",
        "print(f\"Best Random params: {best_rand_100['params']}\")\n",
        "print(f\"Best FedAvg params: {best_fedavg_100['params']}\")\n",
        "\n",
        "print(f\"\"\"\n",
        "     \\t IncFed \\t\\t\\t\\t FedImp \\t\\t\\t\\t Random \\t\\t\\t\\t FedAvg\n",
        "     \\t TR         | VL        | TS \\t\\t TR         | VL        | TS \\t\\t TR         | VL        | TS \\t\\t TR         | VL        | TS\n",
        "100% \\t {best_incfed_100_train:.4f}\\u00B1{best_incfed_100_train_std:.2f}|{best_incfed_100_valid:.4f}\\u00B1{best_incfed_100_valid_std:.2f}|{best_incfed_100_test:.4f}\\u00B1{best_incfed_100_test_std:.2f} \\t {best_fedimp_100_train:.4f}\\u00B1{best_fedimp_100_train_std:.2f}|{best_fedimp_100_valid:.4f}\\u00B1{best_fedimp_100_valid_std:.2f}|{best_fedimp_100_test:.4f}\\u00B1{best_fedimp_100_test_std:.2f} \\t {best_rand_100_train:.4f}\\u00B1{best_rand_100_train_std:.2f}|{best_rand_100_valid:.4f}\\u00B1{best_rand_100_valid_std:.2f}|{best_rand_100_test:.4f}\\u00B1{best_rand_100_test_std:.2f} \\t {best_fedavg_100_train:.4f}\\u00B1{best_fedavg_100_train_std:.2f}|{best_fedavg_100_valid:.4f}\\u00B1{best_fedavg_100_valid_std:.2f}|{best_fedavg_100_test:.4f}\\u00B1{best_fedavg_100_test_std:.2f}\n",
        "75%  \\t {best_incfed_75_train:.4f}\\u00B1{best_incfed_75_train_std:.2f}|{best_incfed_75_valid:.4f}\\u00B1{best_incfed_75_valid_std:.2f}|{best_incfed_75_test:.4f}\\u00B1{best_incfed_75_test_std:.2f} \\t {best_fedimp_75_train:.4f}\\u00B1{best_fedimp_75_train_std:.2f}|{best_fedimp_75_valid:.4f}\\u00B1{best_fedimp_75_valid_std:.2f}|{best_fedimp_75_test:.4f}\\u00B1{best_fedimp_75_test_std:.2f} \\t {best_rand_75_train:.4f}\\u00B1{best_rand_75_train_std:.2f}|{best_rand_75_valid:.4f}\\u00B1{best_rand_75_valid_std:.2f}|{best_rand_75_test:.4f}\\u00B1{best_rand_75_test_std:.2f} \\t {best_fedavg_75_train:.4f}\\u00B1{best_fedavg_75_train_std:.2f}|{best_fedavg_75_valid:.4f}\\u00B1{best_fedavg_75_valid_std:.2f}|{best_fedavg_75_test:.4f}\\u00B1{best_fedavg_75_test_std:.2f}\n",
        "50%  \\t {best_incfed_50_train:.4f}\\u00B1{best_incfed_50_train_std:.2f}|{best_incfed_50_valid:.4f}\\u00B1{best_incfed_50_valid_std:.2f}|{best_incfed_50_test:.4f}\\u00B1{best_incfed_50_test_std:.2f} \\t {best_fedimp_50_train:.4f}\\u00B1{best_fedimp_50_train_std:.2f}|{best_fedimp_50_valid:.4f}\\u00B1{best_fedimp_50_valid_std:.2f}|{best_fedimp_50_test:.4f}\\u00B1{best_fedimp_50_test_std:.2f} \\t {best_rand_50_train:.4f}\\u00B1{best_rand_50_train_std:.2f}|{best_rand_50_valid:.4f}\\u00B1{best_rand_50_valid_std:.2f}|{best_rand_50_test:.4f}\\u00B1{best_rand_50_test_std:.2f} \\t {best_fedavg_50_train:.4f}\\u00B1{best_fedavg_50_train_std:.2f}|{best_fedavg_50_valid:.4f}\\u00B1{best_fedavg_50_valid_std:.2f}|{best_fedavg_50_test:.4f}\\u00B1{best_fedavg_50_test_std:.2f}\n",
        "25%  \\t {best_incfed_25_train:.4f}\\u00B1{best_incfed_25_train_std:.2f}|{best_incfed_25_valid:.4f}\\u00B1{best_incfed_25_valid_std:.2f}|{best_incfed_25_test:.4f}\\u00B1{best_incfed_25_test_std:.2f} \\t {best_fedimp_25_train:.4f}\\u00B1{best_fedimp_25_train_std:.2f}|{best_fedimp_25_valid:.4f}\\u00B1{best_fedimp_25_valid_std:.2f}|{best_fedimp_25_test:.4f}\\u00B1{best_fedimp_25_test_std:.2f} \\t {best_rand_25_train:.4f}\\u00B1{best_rand_25_train_std:.2f}|{best_rand_25_valid:.4f}\\u00B1{best_rand_25_valid_std:.2f}|{best_rand_25_test:.4f}\\u00B1{best_rand_25_test_std:.2f} \\t {best_fedavg_25_train:.4f}\\u00B1{best_fedavg_25_train_std:.2f}|{best_fedavg_25_valid:.4f}\\u00B1{best_fedavg_25_valid_std:.2f}|{best_fedavg_25_test:.4f}\\u00B1{best_fedavg_25_test_std:.2f}\n",
        "\"\"\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}