auxiliary_synthetic.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "auxiliary_synthetic.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "lltGV8IMEM0h",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Introduction\n",
        "\n",
        "This notebook includes experiments on auxiliary learning. Please see the corresponding [repository](https://github.com/vivien000/auxiliary-learning)  and the associated blog post."
      ]
    },
    {
      "metadata": {
        "id": "RD_eWljBzg2S",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Set to True to save the experiments' results on Google Drive\n",
        "google_drive = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jZbqIVG2e-DE",
        "colab_type": "code",
        "cellView": "form",
        "outputId": "60fa56ba-5f30-49d8-95bd-4851405afd5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "#@title Tensorboard launch and utilities\n",
        "\n",
        "import tarfile\n",
        "import urllib.request\n",
        "import json\n",
        "import os\n",
        "import time\n",
        "from IPython.core.display import display, HTML\n",
        "\n",
        "get_ipython().system_raw('tensorboard --logdir ./log --host 0.0.0.0 --port 6006 &')\n",
        "\n",
        "if os.path.exists('/content'):\n",
        "  get_ipython().system_raw('wget -q https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip')\n",
        "  get_ipython().system_raw('unzip -qq ngrok-stable-linux-amd64.zip')\n",
        "  get_ipython().system_raw('pip install -U -q PyDrive')\n",
        "  get_ipython().system_raw('./ngrok http 6006 &')\n",
        "  loop = True\n",
        "  while loop:\n",
        "    try:\n",
        "      ngrok_details = urllib.request.urlopen('http://localhost:4040/api/tunnels')\n",
        "      url = json.load(ngrok_details)['tunnels'][0]['public_url']\n",
        "      loop = False\n",
        "    except (urllib.error.URLError, IndexError):\n",
        "      time.sleep(1)\n",
        "  if google_drive:\n",
        "    from pydrive.auth import GoogleAuth\n",
        "    from pydrive.drive import GoogleDrive\n",
        "    from google.colab import auth\n",
        "    from oauth2client.client import GoogleCredentials\n",
        "    auth.authenticate_user()\n",
        "    gauth = GoogleAuth()\n",
        "    gauth.credentials = GoogleCredentials.get_application_default()\n",
        "    drive = GoogleDrive(gauth)\n",
        "else:\n",
        "  url = 'http://0.0.0.0:6006'\n",
        "display(HTML(\"<a href='\" + url + \"'>Link to Tensorboard</a>\"))\n",
        "\n",
        "def read_csv(file, *args, **kwargs):\n",
        "  if google_drive:\n",
        "    auth.authenticate_user()\n",
        "    gauth = GoogleAuth()\n",
        "    gauth.credentials = GoogleCredentials.get_application_default()\n",
        "    drive = GoogleDrive(gauth)\n",
        "    listed = drive.ListFile({'q': \"title contains '\"+file+\"'\"}).GetList()\n",
        "    drive.CreateFile({'id': listed[0]['id']}).GetContentFile(file)\n",
        "  return pd.read_csv(file, *args, **kwargs)\n",
        "\n",
        "def append_to_log(file, line, header):\n",
        "  if google_drive:\n",
        "    auth.authenticate_user()\n",
        "    gauth = GoogleAuth()\n",
        "    gauth.credentials = GoogleCredentials.get_application_default()\n",
        "    drive = GoogleDrive(gauth)\n",
        "    listed = drive.ListFile({'q': \"title contains '\"+file+\"'\"}).GetList()\n",
        "    if len(listed) > 0:\n",
        "      file = drive.CreateFile({'id': listed[0]['id']})\n",
        "      content = file.GetContentString() + line + '\\n'\n",
        "    else:\n",
        "      file = drive.CreateFile({'title': file})\n",
        "      content = header + '\\n' + line + '\\n'\n",
        "    file.SetContentString(content)\n",
        "    file.Upload()\n",
        "  else:\n",
        "    if os.path.isfile(file):\n",
        "      with open(file, 'a') as f:  \n",
        "        f.write(line)\n",
        "    else:\n",
        "      with open(file, 'w') as f:  \n",
        "        f.write(header + '\\n' + line + '\\n')\n",
        "\n",
        "def number_log_lines(file):\n",
        "  if google_drive:\n",
        "    auth.authenticate_user()\n",
        "    gauth = GoogleAuth()\n",
        "    gauth.credentials = GoogleCredentials.get_application_default()\n",
        "    drive = GoogleDrive(gauth)\n",
        "    listed = drive.ListFile({'q': \"title contains '\"+file+\"'\"}).GetList()\n",
        "    if len(listed) > 0:\n",
        "      file = drive.CreateFile({'id': listed[0]['id']})\n",
        "      content = file.GetContentString()\n",
        "    else:\n",
        "      return 0\n",
        "  else:\n",
        "    if os.path.isfile(file):\n",
        "      with open(file, 'a') as f:  \n",
        "        content = f.read()\n",
        "    else:\n",
        "      return 0\n",
        "  try:\n",
        "      return len(content.split('\\n'))-2\n",
        "  except ValueError:\n",
        "      return 0        \n",
        "\n",
        "def save_folder(folders, file):\n",
        "  if google_drive:\n",
        "    auth.authenticate_user()\n",
        "    gauth = GoogleAuth()\n",
        "    gauth.credentials = GoogleCredentials.get_application_default()\n",
        "    drive = GoogleDrive(gauth)\n",
        "    with tarfile.open(file, 'w|gz') as tar:\n",
        "      for folder in folders:\n",
        "        tar.add(folder)\n",
        "    listed = drive.ListFile({'q': \"title contains '\"+file+\"'\"}).GetList()\n",
        "    if len(listed) > 0:\n",
        "      uploaded = drive.CreateFile({'id': listed[0]['id']})\n",
        "    else:\n",
        "      uploaded = drive.CreateFile({'title': file})\n",
        "    uploaded.SetContentFile(file)\n",
        "    uploaded.Upload()\n",
        "    os.remove(file)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<a href='https://e6b7b0ac.ngrok.io'>Link to Tensorboard</a>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "t3LApDa61AeR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "np.random.seed(seed=0)\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow.contrib.eager as tfe\n",
        "\n",
        "tf.enable_eager_execution()\n",
        "tf.set_random_seed(0)\n",
        "\n",
        "import pandas as pd\n",
        "import sklearn\n",
        "import matplotlib.cm as cm\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import time\n",
        "import shutil\n",
        "\n",
        "from IPython.display import clear_output\n",
        "\n",
        "try:\n",
        "  step_counter = tf.train.create_global_step()\n",
        "except ValueError:\n",
        "  step_counter.assign(0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cTpw0TU6bunW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "x_train1 = tf.constant(np.random.rand(1000, 250), dtype=tf.float32)\n",
        "x_train2 = tf.constant(np.random.rand(1000, 250), dtype=tf.float32)\n",
        "x_val = tf.constant(np.random.rand(1000, 250), dtype=tf.float32)\n",
        "x_test = tf.constant(np.random.rand(10000, 250), dtype=tf.float32)\n",
        "\n",
        "b = tf.constant(10*np.random.randn(250, 100), dtype=tf.float32)\n",
        "other_b = tf.constant(10*np.random.randn(250, 100), dtype=tf.float32)\n",
        "noisy_b = tf.constant(b + 3.5*np.random.randn(250, 100), dtype=tf.float32)\n",
        "\n",
        "def synthetic_function(x, matrix):\n",
        "  return tf.tanh(tf.tensordot(x, matrix, axes=1))\n",
        "\n",
        "y_train1 = synthetic_function(x_train1, b)\n",
        "y_val = synthetic_function(x_val, b)\n",
        "y_test = synthetic_function(x_test, b)\n",
        "\n",
        "y_train2_same = synthetic_function(x_train2, b)\n",
        "y_train2_other = synthetic_function(x_train2, other_b)\n",
        "y_train2_noisy = synthetic_function(x_train2, noisy_b)\n",
        "\n",
        "ds_train1 = tf.data.Dataset.from_tensor_slices((x_train1, y_train1))\n",
        "ds_train2_same = tf.data.Dataset.from_tensor_slices((x_train2, y_train2_same))\n",
        "ds_train2_other = tf.data.Dataset.from_tensor_slices((x_train2, y_train2_other))\n",
        "ds_train2_noisy = tf.data.Dataset.from_tensor_slices((x_train2, y_train2_noisy))\n",
        "\n",
        "ds_train1 = ds_train1.shuffle(1000).batch(100)\n",
        "ds_train2_same = ds_train2_same.shuffle(1000).batch(100)\n",
        "ds_train2_other = ds_train2_other.shuffle(1000).batch(100)\n",
        "ds_train2_noisy = ds_train2_noisy.shuffle(1000).batch(100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "m0U8uDRFTKUL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class SyntheticModel(tf.keras.Model):\n",
        "  def __init__(self):\n",
        "    super(SyntheticModel, self).__init__()\n",
        "    kwargs = {'activation': 'relu'}\n",
        "    self.dense1 = tf.keras.layers.Dense(100, **kwargs)\n",
        "    self.dense2 = tf.keras.layers.Dense(100, **kwargs)\n",
        "    self.dense3 = tf.keras.layers.Dense(100, **kwargs)\n",
        "    self.dense4 = tf.keras.layers.Dense(100, **kwargs)\n",
        "    self.dense5_primary = tf.keras.layers.Dense(100)\n",
        "    self.dense5_auxiliary = tf.keras.layers.Dense(100)\n",
        "  \n",
        "  def call(self, x):\n",
        "    y = self.dense1(x)\n",
        "    y = self.dense2(y)\n",
        "    y = self.dense3(y)\n",
        "    y = self.dense4(y)\n",
        "    return self.dense5_primary(y), self.dense5_auxiliary(y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eyNUOJsLzxHv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Initializes a series of networks to be used in the experiments, so that each\n",
        "# combination of hyperparameters is tested with the same initial weights\n",
        "if not os.path.exists('synthetic'):\n",
        "  os.mkdir('synthetic')\n",
        "  for i in range(20):\n",
        "    model = SyntheticModel()\n",
        "    _ = model(x_val[:1, :])\n",
        "    model.save_weights('synthetic/model_%i.h5' % i)\n",
        "  del model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FtPDABQzYPz4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def censored_vector(u, v, mode):\n",
        "  \"\"\"Adjusts the auxiliary loss gradient\n",
        "  \n",
        "  Adjusts the auxiliary loss gradient before adding it to the primary loss\n",
        "  gradient and using a gradient descent-based method\n",
        "  \n",
        "  Args:\n",
        "    u: A tensorflow variable representing the auxiliary loss gradient\n",
        "    v: A tensorflow variable representing the primary loss gradient\n",
        "    mode: The method used for the adjustment:\n",
        "      - Single task: the auxiliary loss gradient is ignored\n",
        "      - Multitask: the auxiliary loss gradient is kept as it is\n",
        "      - Unweighted cosine: cf. https://arxiv.org/abs/1812.02224\n",
        "      - Weighted cosine: cf. https://arxiv.org/abs/1812.02224\n",
        "      - Projection: cf. https://github.com/vivien000/auxiliary-learning\n",
        "      - Parameter-wise: same as projection but at the level of each parameter\n",
        "    \n",
        "  Returns:\n",
        "    A tensorflow variable representing the adjusted auxiliary loss gradient\n",
        "  \"\"\"\n",
        "  if mode == 'Single task' or u is None:\n",
        "    return 0  \n",
        "  if mode == 'Multitask' or v is None:\n",
        "    return u\n",
        "  if len(u.shape.as_list()) == 1:\n",
        "    u_dot_v, l_u, l_v = tf.reduce_sum(u*v), tf.norm(u), tf.norm(v)\n",
        "  else:\n",
        "    a, b = tf.reshape(u, [-1]), tf.reshape(v, [-1])\n",
        "    u_dot_v, l_u, l_v = tf.reduce_sum(a*b), tf.norm(a), tf.norm(b)\n",
        "  if l_u.numpy() == 0 or l_v.numpy() == 0:\n",
        "    return u\n",
        "  if mode == 'Unweighted cosine':\n",
        "    return u if u_dot_v > 0 else tf.zeros_like(u)\n",
        "  if mode == 'Weighted cosine':\n",
        "    return tf.maximum(u_dot_v, 0)*u/l_u/l_v\n",
        "  if mode == 'Projection':\n",
        "    return u - tf.minimum(u_dot_v, 0)*v/l_v/l_v\n",
        "  if mode == 'Parameter-wise':\n",
        "    return u*((tf.math.sign(u*v)+1)/2)\n",
        "\n",
        "def combined_grads(primary_grad,\n",
        "                   average_primary_grad,\n",
        "                   auxiliary_grad,\n",
        "                   mode,\n",
        "                   overall=False,\n",
        "                   lam=1):\n",
        "  \"\"\"Combines auxiliary loss gradients and primary loss gradients\n",
        "  \n",
        "  Combines a sequence of auxiliary loss gradients and a sequence of primary\n",
        "  loss gradients before performing a gradient descent step\n",
        "  \n",
        "  Args:\n",
        "    primary_grad: A list of tensorflow variables corresponding to the primary\n",
        "    loss gradient for the network's Keras variables\n",
        "    average_primary_grad: A list of tensorflow variables corresponding to\n",
        "    exponential moving averages of the elements above\n",
        "    auxiliary_grad: A list of tensorflow variables corresponding to the\n",
        "    auxiliary loss gradient for the network's Keras variables\n",
        "    mode: The method used for the adjustment:\n",
        "      - Single task: the auxiliary loss gradient is ignored\n",
        "      - Multitask: the auxiliary loss gradient is kept as it is\n",
        "      - Unweighted cosine: cf. https://arxiv.org/abs/1812.02224\n",
        "      - Weighted cosine: cf. https://arxiv.org/abs/1812.02224\n",
        "      - Projection: cf. https://github.com/vivien000/auxiliary-learning\n",
        "      - Parameter-wise: same as projection but at the level of each parameter\n",
        "    overall: True if the transformation takes place at the level of the whole\n",
        "    parameter vector, i.e. the concatenation of all the Keras variables of the\n",
        "    network\n",
        "    lambda: Float balancing the primary loss and the auxiliary loss\n",
        "    \n",
        "  Returns:\n",
        "    A list of tensorflow variables combining the primary loss gradients and the\n",
        "    auxiliary loss gradients and that can directly be used for the next gradient\n",
        "    descent step\n",
        "  \"\"\"\n",
        "  result = [0]*len(primary_grad)\n",
        "  a = tf.constant([], dtype=tf.float32)\n",
        "  aa = tf.constant([], dtype=tf.float32)\n",
        "  b = tf.constant([], dtype=tf.float32)\n",
        "  shapes = []\n",
        "  for i in range(len(primary_grad)):\n",
        "    if auxiliary_grad[i] is None or mode == 'Single task':\n",
        "      result[i] = primary_grad[i]\n",
        "    elif primary_grad[i] is None:\n",
        "      result[i] = lam*auxiliary_grad[i]\n",
        "    elif mode == 'Multitask':\n",
        "      result[i] = primary_grad[i] + lam*auxiliary_grad[i]\n",
        "    elif not overall:\n",
        "      if average_primary_grad is None:\n",
        "        result[i] = (primary_grad[i]\n",
        "                     + lam*censored_vector(auxiliary_grad[i],\n",
        "                                           primary_grad[i],\n",
        "                                           mode))\n",
        "      else:\n",
        "        result[i] = (primary_grad[i]\n",
        "                     + lam*censored_vector(auxiliary_grad[i],\n",
        "                                           average_primary_grad[i],\n",
        "                                           mode))\n",
        "    else:\n",
        "      a = tf.concat([a, tf.reshape(primary_grad[i], [-1])], axis=0)\n",
        "      if average_primary_grad is not None:\n",
        "        aa = tf.concat([aa, tf.reshape(average_primary_grad[i], [-1])], axis=0)\n",
        "      b = tf.concat([b, tf.reshape(auxiliary_grad[i], [-1])], axis=0)\n",
        "      shapes.append((primary_grad[i].shape,\n",
        "                     np.product(primary_grad[i].shape.as_list()),\n",
        "                     i))\n",
        "\n",
        "  if len(shapes) > 0:\n",
        "    if average_primary_grad is None:\n",
        "      c = a + lam*censored_vector(b, a, mode)\n",
        "    else:\n",
        "      c = a + lam*censored_vector(b, aa, mode)\n",
        "    start = 0\n",
        "    for i in range(len(shapes)):\n",
        "      shape, length, index = shapes[i]\n",
        "      result[index] = tf.reshape(c[start:start+length], shape)\n",
        "      start += length\n",
        "  return result\n",
        "\n",
        "def train_iteration(model,\n",
        "                    average_primary_grad,\n",
        "                    alpha,\n",
        "                    optimizer,\n",
        "                    ds_train2,\n",
        "                    writer,\n",
        "                    step_counter,\n",
        "                    mode,\n",
        "                    overall=False,\n",
        "                    lam=1):\n",
        "\n",
        "  if mode != 'Single task':\n",
        "    train_iterator2 = ds_train2.make_one_shot_iterator()\n",
        "    \n",
        "  with writer.as_default(), tf.contrib.summary.always_record_summaries():\n",
        "    for x1, y1 in ds_train1.make_one_shot_iterator():\n",
        "      if mode != 'Single task':\n",
        "        x2, y2 = train_iterator2.get_next()\n",
        "        \n",
        "      with tf.GradientTape(persistent=True) as tape:\n",
        "        y1_hat = model(x1)[0]\n",
        "        primary_loss = tf.reduce_mean((y1_hat-y1)**2)\n",
        "        if mode != 'Single task':\n",
        "          y2_hat = model(x2)[1]\n",
        "          auxiliary_loss = tf.reduce_mean((y2_hat-y2)**2)\n",
        "\n",
        "      tf.contrib.summary.scalar('primary_loss', primary_loss)\n",
        "      primary_grad = tape.gradient(primary_loss, model.variables)\n",
        "      if mode == 'Single task':       \n",
        "        optimizer.apply_gradients(zip(primary_grad, model.variables),\n",
        "                                  global_step=step_counter)\n",
        "      else:\n",
        "        tf.contrib.summary.scalar('auxiliary_loss', auxiliary_loss)\n",
        "        auxiliary_grad = tape.gradient(auxiliary_loss, model.variables)\n",
        "        \n",
        "        if alpha != 1:\n",
        "          if average_primary_grad is None:\n",
        "            average_primary_grad = primary_grad\n",
        "          else:\n",
        "            for i in range(len(average_primary_grad)):\n",
        "              if primary_grad[i] is not None:\n",
        "                average_primary_grad[i] = ((1 - alpha)*average_primary_grad[i]\n",
        "                                           + alpha*primary_grad[i])\n",
        "    \n",
        "        grad = combined_grads(primary_grad,\n",
        "                              average_primary_grad,\n",
        "                              auxiliary_grad,\n",
        "                              mode,\n",
        "                              overall=overall,\n",
        "                              lam=lam)\n",
        "        optimizer.apply_gradients(zip(grad, model.variables),\n",
        "                                  global_step=step_counter)\n",
        "  return average_primary_grad\n",
        "\n",
        "def get_metrics(dataset,\n",
        "                model,\n",
        "                writer,\n",
        "                step_counter):\n",
        "  x, y = (x_val, y_val) if dataset == 'val' else (x_test, y_test)\n",
        "  with writer.as_default(), tf.contrib.summary.always_record_summaries():\n",
        "    y_hat = model(x)[0]\n",
        "    primary_loss = tf.reduce_mean((y_hat-y)**2)\n",
        "    tf.contrib.summary.scalar('primary_loss', primary_loss)\n",
        "  return primary_loss.numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K2HlhKpHh-Z2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def run_experiment(name, model, alpha, ds_train2, mode, overall, lam, output):\n",
        "  train_writer = tf.contrib.summary.create_file_writer('./log/train/' + name,\n",
        "                                                       flush_millis=10000)\n",
        "  val_writer = tf.contrib.summary.create_file_writer('./log/val/' + name,\n",
        "                                                       flush_millis=10000)\n",
        "  test_writer = tf.contrib.summary.create_file_writer('./log/test/' + name,\n",
        "                                                      flush_millis=10000)\n",
        "  step_counter.assign(0)\n",
        "  optimizer = tf.train.AdamOptimizer()\n",
        "  checkpoint_dir = 'model_synthetic'\n",
        "  shutil.rmtree(checkpoint_dir, ignore_errors=True)\n",
        "  checkpoint_prefix = os.path.join(checkpoint_dir, 'model.ckpt')\n",
        "  root = tf.contrib.eager.Checkpoint(optimizer=optimizer,\n",
        "                                     model=model,\n",
        "                                     optimizer_step=step_counter)\n",
        "\n",
        "  average_primary_grad = None\n",
        "  iteration, not_better, best_loss = 1, 0, np.Inf\n",
        "  while not_better < 10:\n",
        "    average_primary_grad = train_iteration(model,\n",
        "                                           average_primary_grad,\n",
        "                                           alpha,\n",
        "                                           optimizer,\n",
        "                                           ds_train2,\n",
        "                                           train_writer,\n",
        "                                           step_counter,\n",
        "                                           mode,\n",
        "                                           overall=overall,\n",
        "                                           lam=lam)\n",
        "    val_loss = get_metrics('val', model, val_writer, step_counter)\n",
        "    clear_output()\n",
        "    print(output)\n",
        "    print(iteration, val_loss)\n",
        "    if val_loss < best_loss:\n",
        "      not_better, best_loss = 0, val_loss\n",
        "      root.save(file_prefix=checkpoint_prefix)\n",
        "    else:\n",
        "      not_better += 1\n",
        "    iteration += 1\n",
        "\n",
        "  root.restore(tf.train.latest_checkpoint(checkpoint_dir))\n",
        "  return model, get_metrics('test', model, test_writer, step_counter)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LF52tk5h3RAa",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def run_experiments(configs, filename):\n",
        "  already_done = number_log_lines(filename)\n",
        "\n",
        "  runs_list = []\n",
        "  header = 'situation,mode,overall,alpha,lam,test_loss'\n",
        "  output = ''\n",
        "\n",
        "  current_iteration = -1\n",
        "\n",
        "  for iteration, mode, overall, lam, case, alpha in configs[already_done:]:\n",
        "    \n",
        "    start = time.time()\n",
        "    \n",
        "    model = SyntheticModel()\n",
        "    _ = model(x_val[:1, :])\n",
        "    model.load_weights('synthetic/model_%i.h5' % iteration)\n",
        "\n",
        "    if case == 'Same task':\n",
        "      ds_train2 = ds_train2_same\n",
        "    elif case == 'Similar task':\n",
        "      ds_train2 = ds_train2_noisy\n",
        "    elif case == 'Unrelated task':\n",
        "      ds_train2 = ds_train2_other\n",
        "    \n",
        "    output += 'Iteration #%d: %s, %s (overall: %s, %f)\\n' % (iteration,\n",
        "                                                             mode,\n",
        "                                                             case,\n",
        "                                                             overall,\n",
        "                                                             lam)\n",
        "    name = '%s-%s-%s-%s-%f-%f' % (iteration, mode, case, overall, lam, alpha)\n",
        "    test_loss = run_experiment(name,\n",
        "                               model,\n",
        "                               alpha,\n",
        "                               ds_train2,\n",
        "                               mode,\n",
        "                               overall,\n",
        "                               lam,\n",
        "                               output)[1]\n",
        "    template = 'Loss: %.3f (%d seconds)\\n\\n'\n",
        "    output += template % (test_loss, time.time()-start)\n",
        "    dict1 = {'case': case,\n",
        "             'mode': mode,\n",
        "             'lam': lam,\n",
        "             'overall': overall,\n",
        "             'alpha': alpha,\n",
        "             'test_loss': test_loss}\n",
        "    runs_list.append(dict1)\n",
        "    line = '%s,%s,%s,%f,%f,%f' % (case,\n",
        "                                  mode,\n",
        "                                  overall,\n",
        "                                  alpha,\n",
        "                                  lam,\n",
        "                                  test_loss)\n",
        "    append_to_log(filename, line, header)\n",
        "\n",
        "  return pd.DataFrame(runs_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "okA4oGPGz4Wu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "iterations = 20\n",
        "\n",
        "filename = 'synthetic_dataset_experiments.csv'\n",
        "\n",
        "configs = [(iteration, mode, overall, lam, case, alpha)\n",
        "           for iteration in range(iterations)\n",
        "           for lam in [0]\n",
        "           for overall in [True]\n",
        "           for mode in ['Single task']\n",
        "           for case in ['Same task']\n",
        "           for alpha in [1]]\n",
        "\n",
        "configs += [(iteration, mode, overall, lam, case, alpha)\n",
        "            for iteration in range(iterations)\n",
        "            for lam in [.1, .3, 1, 3, 10]\n",
        "            for overall in [True]\n",
        "            for mode in ['Multitask']\n",
        "            for case in ['Same task', 'Similar task', 'Unrelated task']\n",
        "            for alpha in [1]]\n",
        "\n",
        "configs += [(iteration, mode, overall, lam, case, alpha)\n",
        "            for iteration in range(iterations)\n",
        "            for lam in [.1, .3, 1, 3, 10]\n",
        "            for overall in [True]\n",
        "            for mode in ['Projection']\n",
        "            for case in ['Same task', 'Similar task', 'Unrelated task']\n",
        "            for alpha in [0.01, 0.1, 1]]\n",
        "\n",
        "configs += [(iteration, mode, overall, lam, case, alpha)\n",
        "            for iteration in range(iterations)\n",
        "            for lam in [.1, .3, 1, 3, 10]\n",
        "            for overall in [True]\n",
        "            for mode in ['Weighted cosine', 'Unweighted cosine']\n",
        "            for case in ['Same task', 'Similar task', 'Unrelated task']\n",
        "            for alpha in [0.01]]\n",
        "\n",
        "configs += [(iteration, mode, overall, lam, case, alpha)\n",
        "            for iteration in range(iterations)\n",
        "            for lam in [.1, .3, 1, 3, 10]\n",
        "            for overall in [False]\n",
        "            for mode in ['Projection', 'Parameter-wise']\n",
        "            for case in ['Same task', 'Similar task', 'Unrelated task']\n",
        "            for alpha in [0.01]]\n",
        "\n",
        "configs += [(iteration, mode, overall, lam, case, alpha)\n",
        "            for iteration in range(iterations)\n",
        "            for lam in [.1, .3, 1, 3, 10]\n",
        "            for overall in [True]\n",
        "            for mode in ['Weighted cosine', 'Unweighted cosine']\n",
        "            for case in ['Same task', 'Similar task', 'Unrelated task']\n",
        "            for alpha in [0.1, 1]]\n",
        "\n",
        "run_experiments(configs, filename)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}