Colab_Tutorial.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "vrIIciaAlHSl"
   },
   "source": [
    "Copyright (c) 2022, salesforce.com, inc and MILA.  \n",
    "All rights reserved.  \n",
    "SPDX-License-Identifier: BSD-3-Clause  \n",
    "For full license text, see the LICENSE file in the repo root  \n",
    "or https://opensource.org/licenses/BSD-3-Clause  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "U5SXMcYAarz5"
   },
   "source": [
    "# How to use this notebook\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "FOR1FpQi0SLi"
   },
   "source": [
    "- The purpose of this notebook is to walk people through the process \n",
    "- The expected enviornment to run this notebook is [colab](https://colab.research.google.com/)\n",
    "- Change runtime type to [GPU](https://research.google.com/colaboratory/faq.html#gpu-availability) for GPU training, CPU is slower\n",
    "- If GPU : \"Train agents with GPU\"\n",
    "- If CPU : \"Train agents with CPU\"\n",
    "- If restart runtime : rerun \"Install prerequisite packages\" and \"Load dependency\" sections\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [IMPORTANT, DO NOT SKIP] Swtich the python version to 3.7.16"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please run the below block and restart runtime after it finishes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference: https://stackoverflow.com/a/74538231\n",
    "# Import the Python sys module\n",
    "import sys\n",
    "\n",
    "# Get the Python version\n",
    "python_version = sys.version_info\n",
    "\n",
    "# Print the Python version\n",
    "print(f\"Current Python version: {python_version.major}.{python_version.minor}\")\n",
    "\n",
    "# If the Python version is not 3.7, then install Python 3.7 and change the alternatives\n",
    "if python_version.major != 3 or python_version.minor != 7:\n",
    "    print(\"\\n Switching to Python version 3.7. Please restart runtime after finishing running this cell.\")\n",
    "    # Install Python 3.7\n",
    "    !sudo apt-get update -y\n",
    "    !sudo apt-get install python3.7\n",
    "\n",
    "    # Change alternatives\n",
    "    !sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python{python_version.major}.{python_version.minor} 1\n",
    "    !sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2\n",
    "\n",
    "    # Install pip for new python \n",
    "    !sudo apt-get install python3.7-distutils\n",
    "    !wget https://bootstrap.pypa.io/get-pip.py\n",
    "    !python get-pip.py\n",
    "\n",
    "    # Install colab's dependencies\n",
    "    !python -m pip install ipython ipython_genutils ipykernel jupyter_console prompt_toolkit httplib2 astor\n",
    "\n",
    "    # Link to the old google package\n",
    "    !ln -s /usr/local/lib/python{python_version.major}.{python_version.minor}/dist-packages/google \\\n",
    "           /usr/local/lib/python3.7/dist-packages/google\n",
    "\n",
    "    # Modify import statements in Google Colab Python files\n",
    "    !sed -i \"s/from IPython.utils import traitlets as _traitlets/import traitlets as _traitlets/\" /usr/local/lib/python3.7/dist-packages/google/colab/*.py\n",
    "    !sed -i \"s/from IPython.utils import traitlets/import traitlets/\" /usr/local/lib/python3.7/dist-packages/google/colab/*.py\n",
    "\n",
    "    print(\"\\n\\nRestarting the Python runtime! Please (re-)run the cells below.\")\n",
    "    # Restart Runtime\n",
    "    import os\n",
    "    os.kill(os.getpid(), 9)\n",
    "else:\n",
    "    print(\"Python version is already 3.7\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please make sure both the below block returning the Python version of 3.7 and consistency: True. If it does not work in the future, please contact `ai4climatecoop@gmail.com` to update the switch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import subprocess\n",
    "\n",
    "# Get Python version using sys\n",
    "sys_version = sys.version_info\n",
    "sys_version_str = f\"{sys_version.major}.{sys_version.minor}\"\n",
    "\n",
    "# Get Python version using shell command\n",
    "shell_version_bytes = subprocess.check_output([\"python\", \"--version\"])\n",
    "shell_version_str = shell_version_bytes.decode(\"utf-8\").strip().split()[1]\n",
    "shell_major_minor_version_str = \".\".join(shell_version_str.split(\".\")[:2])\n",
    "\n",
    "# Check if the versions are the same\n",
    "are_versions_same = sys_version_str == shell_major_minor_version_str\n",
    "\n",
    "print(f\"Python version from sys: {sys_version_str}\")\n",
    "print(f\"Python version from shell command: {shell_major_minor_version_str}\")\n",
    "print(f\"Are the versions the consistent? {are_versions_same}\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "ytMyQ2OHlHSr"
   },
   "source": [
    "# Install prerequisite packages\n",
    "Running this block takes about 4 minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rxMcwd4VsXjC"
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/mila-iqia/climate-cooperation-competition.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-28T00:18:43.729692Z",
     "start_time": "2022-06-28T00:18:30.752376Z"
    },
    "id": "JnhEwCVKlMUg"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir(os.getcwd()+\"/climate-cooperation-competition\")\n",
    "_ROOT = os.getcwd()\n",
    "!pip install -r requirements.txt\n",
    "!pip install rl_warp_drive==1.7.0 # For troubleshooting, please refer to https://github.com/salesforce/warp-drive\n",
    "!pip install ray[rllib]==1.0.0\n",
    "# !pip install codecarbon"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "ukp1MeR1Q0dG"
   },
   "source": [
    "# Load dependency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-28T00:05:15.112923Z",
     "start_time": "2022-06-28T00:05:14.814685Z"
    },
    "id": "u1kP2TiblHSv"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "_ROOT = os.getcwd()\n",
    "sys.path.append(_ROOT+\"/scripts\")\n",
    "sys.path = [os.path.join(_ROOT, \"/scripts\")] + sys.path\n",
    "\n",
    "from desired_outputs import desired_outputs\n",
    "from importlib import reload\n",
    "# from codecarbon import EmissionsTracker"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "4BIL2upxlHSw"
   },
   "source": [
    "# Train agents with GPU"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "nDPbWnNplHSx"
   },
   "source": [
    "<!-- To train with GPU, you need to make sure that you have an **Nvdia Graphic Card** and be able to install critical packages such as ``warp-drive`` and ``pytorch``. If you don't have an Nvdia Graphic Card, you may refer to the section **Train Agents with CPU** below. -->\n",
    "\n",
    "In this section, two examples of GPU-based training with [WarpDrive](https://github.com/salesforce/warp-drive) are presented. \n",
    "\n",
    "\n",
    "1.   The first example does not include negotiation between regions. Since there is no direct interaction between the different regions without negotiation, total runtime is ~2 minutes.\n",
    "2.   The second example includes negotiations between regions. These negotiations take place according to the negotiation protocol outlined in ``rice.py``. Total runtime is 15~20 minutes.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-06-28T00:05:17.978821Z",
     "start_time": "2022-06-28T00:05:17.505382Z"
    },
    "id": "a0jIeTNPlHSy"
   },
   "outputs": [],
   "source": [
    "import train_with_warp_drive as gpu_trainer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "emCh3VJKlHSz"
   },
   "source": [
    "Here are some suggested baseline parameter values. The training process is done by a single GPU.\n",
    "\n",
    "```python\n",
    "num_envs = 100 # ensemble results with 100 randomly initialized enviornments\n",
    "train_batch_size = 1024 # train with 1024 batch_size\n",
    "num_episodes = 30000 # number of episodes\n",
    "lr = 0.005 # learning rate\n",
    "model_params_save_freq = 5000 # save model for every 5000 steps\n",
    "```\n",
    "Additionally, we specify \n",
    "```python \n",
    "negotiation_on = 0 # no negotiation\n",
    "```\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following codes are for the carbon emission tracking using [codecarbon](https://github.com/mlco2/codecarbon). Please comment them out if you do not wish the codecarbon to track your carbon footprint. Please read more at [here](https://codecarbon.io/) for more details.\n",
    "```python \n",
    "tracker = EmissionsTracker()\n",
    "tracker.start()\n",
    "pass # GPU Intensive code goes here\n",
    "tracker.stop()\n",
    "```"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "GMxOg2TJk6-S"
   },
   "source": [
    "Running this next cell will take approximately 2 minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aw-4gD9WlHS0"
   },
   "outputs": [],
   "source": [
    "# tracker = EmissionsTracker()\n",
    "# tracker.start()\n",
    "\n",
    "gpu_trainer_off, gpu_nego_off_ts = gpu_trainer.trainer(negotiation_on=0, # no negotiation\n",
    "  num_envs=100, \n",
    "  train_batch_size=1024, \n",
    "  num_episodes=3000, \n",
    "  lr=0.0005,\n",
    "  model_params_save_freq=5000, \n",
    "  desired_outputs=desired_outputs, # a list of values that the simulator will output\n",
    "  output_all_envs=False # output the mean of all \"num_envs\" results. Set to True for output all results\n",
    "  )\n",
    "\n",
    "# tracker.stop()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "HfLSuYHjlHS0"
   },
   "source": [
    "To train the agents with negotiation, we modify ``negotiation_on``:\n",
    "\n",
    "```python\n",
    "negotiation_on = 1 # with naive negotiation\n",
    "```\n",
    "A naive negotiation protocol is already implemented, but **participants are expected to modify, improve and/or replace this protocol to maximize climate and economic outcomes**."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "-vSy2J__fman"
   },
   "source": [
    "Running this next cell will take 15~20 minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nm2jt8A-lHS1",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# tracker = EmissionsTracker()\n",
    "# tracker.start()\n",
    "\n",
    "gpu_trainer_on, gpu_nego_on_ts = gpu_trainer.trainer(negotiation_on=1, # with naive negotiation\n",
    "  num_envs=100,\n",
    "  train_batch_size=1024,\n",
    "  num_episodes=30000,\n",
    "  lr=0.0005,\n",
    "  model_params_save_freq=5000,\n",
    "  desired_outputs=desired_outputs, # a list of values that the simulator will output\n",
    "  output_all_envs=False # output the mean of all \"num_envs\" results. Set to True for output all results\n",
    "  )\n",
    "\n",
    "# tracker.stop()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "aLWIZ1kjfjxC"
   },
   "source": [
    "The trainer `gpu_trainer_on` closes gracefully, so `gpu_nego_on_ts` contains the timeseries data from the trainer.\n",
    "\n",
    "\n",
    "If you encounter the following error:\n",
    "\n",
    "```\n",
    "RuntimeError: CUDA out of memory.\n",
    "```\n",
    "reducing ``num_envs`` and ``train_batch_size`` can help to some extent.\n",
    "\n",
    "If you encounter unexpected errors such as \n",
    "\n",
    "```\n",
    "RuntimeError: CUDA error: invalid resource handle\n",
    "CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\n",
    "For debugging consider passing CUDA_LAUNCH_BLOCKING=1.\n",
    "```\n",
    "\n",
    "please try to restart runtime before open an issue."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "gL3uTUGTlHS2"
   },
   "source": [
    "To customize the training script, please check ``gpu_trainer.py`` for more details."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "hrqSp18wlHS2"
   },
   "source": [
    "# Train agents with CPU"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "mq3kgyo9lHS2"
   },
   "source": [
    "CPU-based training can also be done with `rllib`, although it can take much longer depending on the complexity of the negotiation protocol (~3 times longer for the naive negotiation protocol)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vQzgkrwtlHS3"
   },
   "outputs": [],
   "source": [
    "# This is necessary for rllib to get the correct path!\n",
    "os.chdir(_ROOT+\"/scripts\")\n",
    "import train_with_rllib as cpu_trainer"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "kkmFGiccQUUP"
   },
   "source": [
    "Here are some suggested baseline parameter values. The training process is done by a single CPU.\n",
    "\n",
    "```python\n",
    "num_envs = 1 # ensemble results with 100 random intialized enviornments\n",
    "train_batch_size = 1024 # train with 1024 batch_size\n",
    "num_episodes = 30000 # number of episodes\n",
    "lr = 0.005 # learning rate\n",
    "model_params_save_freq = 5000 # save model for every 5000 steps\n",
    "num_workers=1 # a single CPU\n",
    "```\n",
    "Additionally, we specify \n",
    "```python \n",
    "negotiation_on = 0 # no negotiation\n",
    "```\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "mjRpvEmoqLS2"
   },
   "source": [
    "Running this next cell will take ~6 minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpu_trainer = reload(cpu_trainer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "p4cen253lHS3",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cpu_trainer_off, cpu_nego_off_ts = cpu_trainer.trainer(negotiation_on=0,  # no negotiation\n",
    "  num_envs=1, \n",
    "  train_batch_size=1024, \n",
    "  num_episodes=300, \n",
    "  lr=0.0005, \n",
    "  model_params_save_freq=5000, \n",
    "  desired_outputs=desired_outputs, # a list of values that the simulator will output\n",
    "  num_workers=1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "j1mlWgecQZ6s"
   },
   "source": [
    "To train the agents with negotiation, we modify ``negotiation_on``:\n",
    "\n",
    "```python\n",
    "negotiation_on = 1 # with naive negotiation\n",
    "```\n",
    "A naive negotiation protocol is already implemented, but **participants are expected to modify, improve and/or replace this protocol to maximize climate and economic outcomes**."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "UN1EHJtVp-co"
   },
   "source": [
    "Running this next cell will take  ~33 minutes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpu_trainer = reload(cpu_trainer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "egkatPyElHS3"
   },
   "outputs": [],
   "source": [
    "cpu_trainer_on, cpu_nego_on_ts = cpu_trainer.trainer(negotiation_on=1, # with naive negotiation\n",
    "  num_envs=1, \n",
    "  train_batch_size=1024, \n",
    "  num_episodes=300, \n",
    "  lr=0.0005, \n",
    "  model_params_save_freq=5000, \n",
    "  desired_outputs=desired_outputs, # a list of values that the simulator will output\n",
    "  num_workers=1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "7q_PO6bilqCU"
   },
   "source": [
    "The trainer `cpu_trainer_on` closes gracefully, so `cpu_nego_on_ts` contains the timeseries data from the trainer.\n",
    "\n",
    "If the process is killed during training, reducing ``num_envs`` and ``train_batch_size`` can help to some extent."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "OF4mXVOHlHS3"
   },
   "source": [
    "# Save or load from previous training results"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "80g85HbZlHS4"
   },
   "source": [
    "This section is for saving and loading the results of training (not the trainer itself)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "K9zxtDGzlHS4"
   },
   "outputs": [],
   "source": [
    "from opt_helper import save, load"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "03fOZB9fpHAs"
   },
   "source": [
    "To save the output timeseries: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lYF6UDHKlHS4"
   },
   "outputs": [],
   "source": [
    "# [uncomment below to save]\n",
    "# save({\"nego_off\":gpu_nego_off_ts, \"nego_on\":gpu_nego_on_ts}, \"filename.pkl\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "0vG1JZ75pIa7"
   },
   "source": [
    "To load the output timeseries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4TEE7CvHlHS4"
   },
   "outputs": [],
   "source": [
    "# [uncomment below to load]\n",
    "# dict_ts = load(\"filename.pkl\")\n",
    "# nego_off_ts, nego_on_ts = dict_ts[\"nego_off\"], dict_ts[\"nego_on\"]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot training procedures"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One may want to plot the some metrics such as `mean reward` which are logged during the training procedure.\n",
    "\n",
    "```python\n",
    "metrics = ['Iterations Completed',\n",
    " 'VF loss coefficient',\n",
    " 'Entropy coefficient',\n",
    " 'Total loss',\n",
    " 'Policy loss',\n",
    " 'Value function loss',\n",
    " 'Mean rewards',\n",
    " 'Max. rewards',\n",
    " 'Min. rewards',\n",
    " 'Mean value function',\n",
    " 'Mean advantages',\n",
    " 'Mean (norm.) advantages',\n",
    " 'Mean (discounted) returns',\n",
    " 'Mean normalized returns',\n",
    " 'Mean entropy',\n",
    " 'Variance explained by the value function',\n",
    " 'Gradient norm',\n",
    " 'Learning rate',\n",
    " 'Mean episodic reward',\n",
    " 'Mean policy eval time per iter (ms)',\n",
    " 'Mean action sample time per iter (ms)',\n",
    " 'Mean env. step time per iter (ms)',\n",
    " 'Mean training time per iter (ms)',\n",
    " 'Mean total time per iter (ms)',\n",
    " 'Mean steps per sec (policy eval)',\n",
    " 'Mean steps per sec (action sample)',\n",
    " 'Mean steps per sec (env. step)',\n",
    " 'Mean steps per sec (training time)',\n",
    " 'Mean steps per sec (total)'\n",
    " ]\n",
    "```"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To check out the logged submissions, please run the following block."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "glob(os.path.join(_ROOT,\"Submissions/*.zip\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If previous trainings are finished and logged properly, this should give a list of `*.zip` files where the logs are included. \n",
    "\n",
    "We picked one of the submissions and the metric `Mean episodic reward` as an example, please check the code below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from opt_helper import get_training_curve, plot_training_curve\n",
    "\n",
    "log_zip = glob(os.path.join(_ROOT,\"Submissions/*.zip\"))[0]\n",
    "plot_training_curve(None, 'Mean episodic reward', log_zip)\n",
    "\n",
    "# to check the raw logging dictionary, uncomment below\n",
    "# logs = get_training_curve(log_zip)\n",
    "# logs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "0BCG5IYWlHS5"
   },
   "source": [
    "# Plot results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oZW6-QJGlHS5"
   },
   "outputs": [],
   "source": [
    "from desired_outputs import desired_outputs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "AdxL0JanlHS5"
   },
   "source": [
    "One may want to check the performance of the agents by plotting graphs. Below, we list all the logged variables. One may change the ``desired_outputs.py`` to add more variables of interest.\n",
    "\n",
    "```python\n",
    "desired_outputs = ['global_temperature', \n",
    "  'global_carbon_mass', \n",
    "  'capital_all_regions', \n",
    "  'labor_all_regions', \n",
    "  'production_factor_all_regions', \n",
    "  'intensity_all_regions', \n",
    "  'global_exogenous_emissions', \n",
    "  'global_land_emissions', \n",
    "  'timestep', \n",
    "  'activity_timestep', \n",
    "  'capital_depreciation_all_regions', \n",
    "  'savings_all_regions', \n",
    "  'mitigation_rate_all_regions', \n",
    "  'max_export_limit_all_regions', \n",
    "  'mitigation_cost_all_regions', \n",
    "  'damages_all_regions', \n",
    "  'abatement_cost_all_regions', \n",
    "  'utility_all_regions', \n",
    "  'social_welfare_all_regions', \n",
    "  'reward_all_regions', \n",
    "  'consumption_all_regions', \n",
    "  'current_balance_all_regions', \n",
    "  'gross_output_all_regions', \n",
    "  'investment_all_regions', \n",
    "  'production_all_regions', \n",
    "  'tariffs', \n",
    "  'future_tariffs', \n",
    "  'scaled_imports', \n",
    "  'desired_imports', \n",
    "  'tariffed_imports', \n",
    "  'stage', \n",
    "  'minimum_mitigation_rate_all_regions', \n",
    "  'promised_mitigation_rate', \n",
    "  'requested_mitigation_rate', \n",
    "  'proposal_decisions',\n",
    "  'global_consumption',\n",
    "  'global_production']\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rYLsNHRjlHS5"
   },
   "outputs": [],
   "source": [
    "from opt_helper import plot_result"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "-Ab3Dd-zlHS5"
   },
   "source": [
    "`plot_result()` plots the time series of logged variables.\n",
    "\n",
    "```python\n",
    "plot_result(variables, nego_off, nego_on, k)\n",
    "```\n",
    "* ``variables`` can be either a single variable of interest or a list of variable names from the above list. \n",
    "* The ``nego_off_ts`` and ``nego_on_ts`` are the logged time series for these variables, with and without negotiation. \n",
    "* ``k`` represents the dimension of the variable of interest ( it should be ``0`` by default for most situations)."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "mrn13h2V2jBQ"
   },
   "source": [
    "Here's an example of plotting a single variable of interest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nDdreoz2lHS6"
   },
   "outputs": [],
   "source": [
    "plot_result(\"global_temperature\", \n",
    "  nego_off=gpu_nego_off_ts, # change it to cpu_nego_off_ts if using CPU\n",
    "  nego_on=gpu_nego_on_ts, \n",
    "  k=0)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "VNXPz9kk2meL"
   },
   "source": [
    "Here's an example of plotting a list of variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Z76yOBp3lHS5",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plot_result(desired_outputs[0:3], # truncated for demonstration purposes\n",
    "  nego_off=gpu_nego_off_ts, \n",
    "  nego_on=gpu_nego_on_ts, \n",
    "  k=0)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "6sJSQ5gdxCni"
   },
   "source": [
    "If one only want to plot negotiation-off plots, feel free to set `nego_on=None`. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FkuU8kV2xCnj"
   },
   "outputs": [],
   "source": [
    "plot_result(desired_outputs[0:3], # truncated for demonstration purposes\n",
    "  nego_off=gpu_nego_off_ts, \n",
    "  nego_on=None, \n",
    "  k=0)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "yDI4p7cqlHS6"
   },
   "source": [
    "# How to quickly evaluate the results"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "6TSbZcQzlHS6"
   },
   "source": [
    "This section to for evaluating the trained agents. One can edit the evaluation function ``eval metrics`` in ``evaluate_submission.py`` to include more metrics of interest.\n",
    "\n",
    "The evaluation script requires as input:\n",
    "1. The trainer\n",
    "2. The logged_variables\n",
    "3. The framework of the trainer. If using GPU-based training, it should be ``warpdrive``. If using CPU-based training, it should be ``rllib``.\n",
    "\n",
    "We give one example below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(os.path.join(_ROOT,\"scripts\"))\n",
    "from evaluate_submission import val_metrics\n",
    "val_metrics(logged_ts=gpu_nego_off_ts, framework=\"warpdrive\") # for GPU\n",
    "# val_metrics(logged_ts=cpu_nego_off_ts, framework=\"rllib\") # for CPU"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you want to evaluate a specific zip submission. You may do the followings. Please replace the `FILENAME.zip` with your zip filename."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluate_submission import perform_evaluation, get_results_dir\n",
    "unzip_path, _ = get_results_dir(\"/content/climate-cooperation-competition/Submissions/FILENAME.zip\")\n",
    "from run_unittests import _BASE_CODE_PATH, _BASE_RICE_PATH, _BASE_RICE_HELPERS_PATH, _BASE_RICE_BUILD_PATH, _BASE_CONSISTENCY_CHECKER_PATH\n",
    "perform_evaluation(unzip_path)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "LuF76W4FlHS6"
   },
   "source": [
    "# Code pieces that can be modified"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "pXdnxXJ7m9Va"
   },
   "source": [
    "As a running example, we use the bilateral negotiation protocol. For more examples, please see section 5.3 in [the white paper](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/White_Paper.pdf)."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "mu48DrgvlHS6"
   },
   "source": [
    "## Introduction of environment codes"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "nOO9JTyHlHS6"
   },
   "source": [
    "[``rice.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py), [``rice_cuda.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_cuda.py), [``rice_step.cu``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_step.cu) and [``rice_helpers.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_helpers.py) are responsible for the GPU code.\n",
    "\n",
    "* [``rice.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py) includes interactions between the agents and the environment. **[``rice.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py) is the main script to be modified.**\n",
    "\n",
    "* [``rice_helpers.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_helpers.py) includes all the socioeconomic and climate dynamics. [``rice_helpers.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_helpers.py) should not be changed.\n",
    "\n",
    "* [GPU needed] [``rice_cuda.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_cuda.py) connects the data between the python script and CUDA code.\n",
    "\n",
    "* [GPU needed] [``rice_step.cu``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_step.cu) is the CUDA version of the code which contains the socioeconomic and climate dynamics, as well as the interactions between the agents and the environment. **To use GPU-based training, the CUDA code in ``rice_step.cu`` must have the same logic as the python code in [``rice.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py) and [``rice_helpers.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice_helpers.py).** The CUDA code mostly follows the grammar of C++. Please refer to [here](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html) for more details.\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "wERUqHqJlHS7"
   },
   "source": [
    "## How to add extra observations"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "OmERwCAjlHS7"
   },
   "source": [
    "To add extra observations or make changes to the observation space, at least two functions must be modified.\n",
    "1.   [`generate_observation()`](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L379)\n",
    "2.   [`reset()`](https://github.com/mila-iqia/climate-cooperation-competition/blob/)\n",
    "\n",
    "As an example, [here](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L428) are the features added when the naive bilateral negotiation protocol is enabled in the simulator: \n",
    "\n",
    "``` python\n",
    "        if self.negotiation_on:\n",
    "            global_features += [\"stage\"]\n",
    "\n",
    "            public_features += []\n",
    "\n",
    "            private_features += [\n",
    "                \"minimum_mitigation_rate_all_regions\",\n",
    "            ]\n",
    "\n",
    "            bilateral_features += [\n",
    "                \"promised_mitigation_rate\",\n",
    "                \"requested_mitigation_rate\",\n",
    "                \"proposal_decisions\",\n",
    "            ]\n",
    "\n",
    "        shared_features = np.array([])\n",
    "        for feature in global_features + public_features:\n",
    "            shared_features = np.append(\n",
    "                shared_features,\n",
    "                self.flatten_array(\n",
    "                    self.global_state[feature][\"value\"][self.timestep]\n",
    "                    / self.global_state[feature][\"norm\"]\n",
    "                ),\n",
    "            )\n",
    "\n",
    "\n",
    "```\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "l1D5ZPREoPLG"
   },
   "source": [
    "## How to add actions"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "_zdRJ2-yqlAq"
   },
   "source": [
    "By default, agents' actions are contained in [`self.actions_nvec`](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L136) during [`init()`](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L64):\n",
    "\n",
    "```python\n",
    "        self.actions_nvec = (\n",
    "            self.savings_action_nvec\n",
    "            + self.mitigation_rate_action_nvec\n",
    "            + self.export_action_nvec\n",
    "            + self.import_actions_nvec\n",
    "            + self.tariff_actions_nvec\n",
    "        )\n",
    "\n",
    "```"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "M-lrkKZqoRKM"
   },
   "source": [
    "Extra actions related to the negotiation protocol can be appended to `self.actions_nvec`.\n",
    "It is important that extra actions be appended at the **end** of `self.actions_nvec`.\n",
    "``` python \n",
    "            # Each region proposes to each other region\n",
    "            # self mitigation and their mitigation values\n",
    "            self.proposal_actions_nvec = (\n",
    "                [self.num_discrete_action_levels] * 2 * self.num_regions\n",
    "            )\n",
    "\n",
    "            # Each region evaluates a proposal from every other region,\n",
    "            # either accept or reject.\n",
    "            self.evaluation_actions_nvec = [2] * self.num_regions\n",
    "\n",
    "            # extra actions are appended to the end of self.actions_nvec\n",
    "            self.actions_nvec += (\n",
    "                self.proposal_actions_nvec + self.evaluation_actions_nvec\n",
    "            )\n",
    "\n",
    "```"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "FjAU4P1elHS7"
   },
   "source": [
    "## How to implement the logic for negotiation protocols"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "7JwolIYnlHS7"
   },
   "source": [
    "The baseline logic for bilateral negotiation actions is a naive bargain process with two steps:\n",
    "1. A [``proposal_step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L536) for each agent to propose certains actions to other agents, for example a minimum mitigation rate.\n",
    "2. An [``evaluation_step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L585) for each agent to evaluation other agents' proposals. \n",
    "\n",
    "These functions describe how the negotiations actions affect the observation space and the action masking (for more, see the next section).\n",
    "Both steps are done sequentially in the [``step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L346) function in [``rice.py``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py): \n",
    "\n",
    "```python\n",
    "        if self.negotiation_on:\n",
    "            # Note: The '+1` below is for the climate_and_economy_simulation_step\n",
    "            self.stage = self.timestep % (self.num_negotiation_stages + 1)\n",
    "            self.set_global_state(\n",
    "                \"stage\", self.stage, self.timestep, dtype=self.int_dtype\n",
    "            )\n",
    "            if self.stage == 1:\n",
    "                return self.proposal_step(actions)\n",
    "\n",
    "            if self.stage == 2:\n",
    "                return self.evaluation_step(actions)\n",
    "\n",
    "        return self.climate_and_economy_simulation_step(actions)\n",
    "\n",
    "```\n",
    "Once the stages of the negotiation protocol are concluded, then the [`climate_and_economy_simulation_step()`](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L651) implements the socioeconomic and climate dynamics associated with the updated observation space and masked actions.\n",
    "\n",
    "We expect competitors to propose different mechanisms to encourage global cooperation along climate and economic objectives.\n",
    "Participants should therefore modify this code to match the logic of their proposed negotiation protocol, even proposing new functions to replace [``proposal_step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L536), [``evaluation_step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L585) and the code above.\n",
    "\n",
    "For example, competitors could propose a mechanism to form [dynamic climate clubs](https://williamnordhaus.com/publications/climate-clubs-overcoming-free-riding-international-climate-policy), where admittance is based on a minimum mitigation rate. Club members enjoy lower tariffs when trading with other club members, while non-members, who do not have to contribute to mitigation, suffer heavy tariffs when trading with club members.\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "A1OGL7JAlHS7"
   },
   "source": [
    "## What is masking?"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "3ww_LsgvlHS7"
   },
   "source": [
    "Action masking determines the feasible subspace of the action space according to the negotiation protocol. Action masks are set before agents choose their actions, so the agent explicitly chooses from the feasible action subspace.\n",
    "To implement this logic, actions masks are modified in the [``evaluation_step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L585), after the [``proposal_step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L536) and [``evaluation_step()``](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L585), but before the [`climate_and_economy_simulation_step()`](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L651). This way, the regions are prohibited from taking actions outside of the feasible action subspace.\n",
    "\n",
    "For example, during the bilateral negotiation process, regions that agree to implement minimum mitigation rates are required to do so. \n",
    "\n",
    "```python\n",
    "        for region_id in range(self.num_regions):\n",
    "            outgoing_accepted_mitigation_rates = [\n",
    "                self.global_state[\"promised_mitigation_rate\"][\"value\"][\n",
    "                    self.timestep, region_id, j\n",
    "                ]\n",
    "                * self.global_state[\"proposal_decisions\"][\"value\"][\n",
    "                    self.timestep, j, region_id\n",
    "                ]\n",
    "                for j in range(self.num_regions)\n",
    "            ]\n",
    "            incoming_accepted_mitigation_rates = [\n",
    "                self.global_state[\"requested_mitigation_rate\"][\"value\"][\n",
    "                    self.timestep, j, region_id\n",
    "                ]\n",
    "                * self.global_state[\"proposal_decisions\"][\"value\"][\n",
    "                    self.timestep, region_id, j\n",
    "                ]\n",
    "                for j in range(self.num_regions)\n",
    "            ]\n",
    "\n",
    "            self.global_state[\"minimum_mitigation_rate_all_regions\"][\"value\"][\n",
    "                self.timestep, region_id\n",
    "            ] = max(\n",
    "                outgoing_accepted_mitigation_rates + incoming_accepted_mitigation_rates\n",
    "            )\n",
    "\n",
    "```\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "G37-_TDLlHS7"
   },
   "source": [
    "## How to implement and/or modify the logic of action masking?"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "The logic behind action masks is implemented in [`generate_action_mask()`](https://github.com/mila-iqia/climate-cooperation-competition/blob/main/rice.py#L506).\n",
    "`mask_dict` gives the mapping for each region to its corresponding action `mask`. In the current implementation, `mask` is a binary vector where `0` indicates an action that is not allowed, and `1` indicates an action that is allowed.\n",
    "\n",
    "For example, in the bilateral negotiation protocol, the action mask is based on the minimum mitigation rate for each region (see code below).\n",
    "```python\n",
    "    def generate_action_mask(self):\n",
    "        \"\"\"\n",
    "        Generate action masks.\n",
    "        \"\"\"\n",
    "        mask_dict = {region_id: None for region_id in range(self.num_regions)}\n",
    "        for region_id in range(self.num_regions):\n",
    "            mask = self.default_agent_action_mask.copy()\n",
    "            if self.negotiation_on:\n",
    "                minimum_mitigation_rate = int(round(\n",
    "                    self.global_state[\"minimum_mitigation_rate_all_regions\"][\"value\"][\n",
    "                        self.timestep, region_id\n",
    "                    ]\n",
    "                    * self.num_discrete_action_levels\n",
    "                ))\n",
    "                mitigation_mask = np.array(\n",
    "                    [0 for _ in range(minimum_mitigation_rate)]\n",
    "                    + [\n",
    "                        1\n",
    "                        for _ in range(\n",
    "                            self.num_discrete_action_levels - minimum_mitigation_rate\n",
    "                        )\n",
    "                    ]\n",
    "                )\n",
    "                mask_start = sum(self.savings_action_nvec)\n",
    "                mask_end = mask_start + sum(self.mitigation_rate_action_nvec)\n",
    "                mask[mask_start:mask_end] = mitigation_mask\n",
    "            mask_dict[region_id] = mask\n",
    "\n",
    "        return mask_dict\n",
    "\n",
    "```"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "ytMyQ2OHlHSr",
    "ukp1MeR1Q0dG",
    "4BIL2upxlHSw",
    "hrqSp18wlHS2",
    "OF4mXVOHlHS3",
    "0BCG5IYWlHS5",
    "yDI4p7cqlHS6",
    "LuF76W4FlHS6"
   ],
   "name": "Copy of Colab_Tutorial.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  },
  "vscode": {
   "interpreter": {
    "hash": "27fe5b6ee35739f11838d1f2aa805cea7439f8f105925cbccb33ad3076b40a51"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}