DQN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "DQN.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "hL0qgt0K9RO_",
        "zz94oU2p6BEc",
        "78F0UtjRJzXf",
        "3XhI8FXGJTg5",
        "CkTTjNITVLiL",
        "YIoquaHUTLwz",
        "WCJgSqXHcybf",
        "CtVUzAk6VoZf",
        "rdgnYP_DVSET",
        "8SXqVwQfTAPi"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sarah-wookey/RL_NetHack_2020/blob/main/DQN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Wuek_rQIct1"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hL0qgt0K9RO_"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZcEuWgSH7gr7",
        "outputId": "50603757-2070-46d9-f000-bcdf8528dcbc"
      },
      "source": [
        "# install prerequisites for nle\n",
        "!sudo apt-get install -y build-essential autoconf libtool pkg-config \\\n",
        "    python3-dev python3-pip python3-numpy git libncurses5-dev \\\n",
        "    libzmq3-dev flex bison"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "build-essential is already the newest version (12.4ubuntu1).\n",
            "pkg-config is already the newest version (0.29.1-0ubuntu2).\n",
            "python3-numpy is already the newest version (1:1.13.3-2ubuntu1).\n",
            "python3-numpy set to manually installed.\n",
            "git is already the newest version (1:2.17.1-1ubuntu0.7).\n",
            "libncurses5-dev is already the newest version (6.1-1ubuntu1.18.04).\n",
            "python3-dev is already the newest version (3.6.7-1~18.04).\n",
            "libzmq3-dev is already the newest version (4.2.5-1ubuntu0.2).\n",
            "The following additional packages will be installed:\n",
            "  automake autotools-dev file libbison-dev libfl-dev libfl2 libmagic-mgc\n",
            "  libmagic1 libsigsegv2 m4 python-pip-whl python3-asn1crypto\n",
            "  python3-cffi-backend python3-crypto python3-cryptography python3-idna\n",
            "  python3-keyring python3-keyrings.alt python3-pkg-resources\n",
            "  python3-secretstorage python3-setuptools python3-six python3-wheel\n",
            "  python3-xdg\n",
            "Suggested packages:\n",
            "  autoconf-archive gnu-standards autoconf-doc gettext bison-doc flex-doc\n",
            "  libtool-doc gcj-jdk m4-doc python-crypto-doc python-cryptography-doc\n",
            "  python3-cryptography-vectors gnome-keyring libkf5wallet-bin\n",
            "  gir1.2-gnomekeyring-1.0 python-secretstorage-doc python-setuptools-doc\n",
            "The following NEW packages will be installed:\n",
            "  autoconf automake autotools-dev bison file flex libbison-dev libfl-dev\n",
            "  libfl2 libmagic-mgc libmagic1 libsigsegv2 libtool m4 python-pip-whl\n",
            "  python3-asn1crypto python3-cffi-backend python3-crypto python3-cryptography\n",
            "  python3-idna python3-keyring python3-keyrings.alt python3-pip\n",
            "  python3-pkg-resources python3-secretstorage python3-setuptools python3-six\n",
            "  python3-wheel python3-xdg\n",
            "0 upgraded, 29 newly installed, 0 to remove and 14 not upgraded.\n",
            "Need to get 5,371 kB of archives.\n",
            "After this operation, 22.4 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsigsegv2 amd64 2.12-1 [14.7 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 m4 amd64 1.4.18-1 [197 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 flex amd64 2.6.4-6 [316 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic-mgc amd64 1:5.32-2ubuntu0.4 [184 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic1 amd64 1:5.32-2ubuntu0.4 [68.6 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 file amd64 1:5.32-2ubuntu0.4 [22.1 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 autoconf all 2.69-11 [322 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 autotools-dev all 20180224.1 [39.6 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic/main amd64 automake all 1:1.15.1-3ubuntu2 [509 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 libbison-dev amd64 2:3.0.4.dfsg-1build1 [339 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu bionic/main amd64 bison amd64 2:3.0.4.dfsg-1build1 [266 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 libfl2 amd64 2.6.4-6 [11.4 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic/main amd64 libfl-dev amd64 2.6.4-6 [6,320 B]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic/main amd64 libtool all 2.4.6-2 [194 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python-pip-whl all 9.0.1-2.3~ubuntu1.18.04.4 [1,653 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-asn1crypto all 0.24.0-1 [72.8 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-cffi-backend amd64 1.11.5-1 [64.6 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-crypto amd64 2.6.1-8ubuntu2 [244 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-idna all 2.6-1 [32.5 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-six all 1.11.0-2 [11.4 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-cryptography amd64 2.1.4-1ubuntu1.4 [220 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-secretstorage all 2.3.1-2 [12.1 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-keyring all 10.6.0-1 [26.7 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-keyrings.alt all 3.0-1 [16.6 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python3-pip all 9.0.1-2.3~ubuntu1.18.04.4 [114 kB]\n",
            "Get:26 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pkg-resources all 39.0.1-2 [98.8 kB]\n",
            "Get:27 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-setuptools all 39.0.1-2 [248 kB]\n",
            "Get:28 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-wheel all 0.30.0-0.2 [36.5 kB]\n",
            "Get:29 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-xdg all 0.25-4ubuntu1 [31.4 kB]\n",
            "Fetched 5,371 kB in 1s (5,158 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 29.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libsigsegv2:amd64.\n",
            "(Reading database ... 144865 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libsigsegv2_2.12-1_amd64.deb ...\n",
            "Unpacking libsigsegv2:amd64 (2.12-1) ...\n",
            "Selecting previously unselected package m4.\n",
            "Preparing to unpack .../01-m4_1.4.18-1_amd64.deb ...\n",
            "Unpacking m4 (1.4.18-1) ...\n",
            "Selecting previously unselected package flex.\n",
            "Preparing to unpack .../02-flex_2.6.4-6_amd64.deb ...\n",
            "Unpacking flex (2.6.4-6) ...\n",
            "Selecting previously unselected package libmagic-mgc.\n",
            "Preparing to unpack .../03-libmagic-mgc_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmagic1:amd64.\n",
            "Preparing to unpack .../04-libmagic1_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package file.\n",
            "Preparing to unpack .../05-file_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking file (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package autoconf.\n",
            "Preparing to unpack .../06-autoconf_2.69-11_all.deb ...\n",
            "Unpacking autoconf (2.69-11) ...\n",
            "Selecting previously unselected package autotools-dev.\n",
            "Preparing to unpack .../07-autotools-dev_20180224.1_all.deb ...\n",
            "Unpacking autotools-dev (20180224.1) ...\n",
            "Selecting previously unselected package automake.\n",
            "Preparing to unpack .../08-automake_1%3a1.15.1-3ubuntu2_all.deb ...\n",
            "Unpacking automake (1:1.15.1-3ubuntu2) ...\n",
            "Selecting previously unselected package libbison-dev:amd64.\n",
            "Preparing to unpack .../09-libbison-dev_2%3a3.0.4.dfsg-1build1_amd64.deb ...\n",
            "Unpacking libbison-dev:amd64 (2:3.0.4.dfsg-1build1) ...\n",
            "Selecting previously unselected package bison.\n",
            "Preparing to unpack .../10-bison_2%3a3.0.4.dfsg-1build1_amd64.deb ...\n",
            "Unpacking bison (2:3.0.4.dfsg-1build1) ...\n",
            "Selecting previously unselected package libfl2:amd64.\n",
            "Preparing to unpack .../11-libfl2_2.6.4-6_amd64.deb ...\n",
            "Unpacking libfl2:amd64 (2.6.4-6) ...\n",
            "Selecting previously unselected package libfl-dev:amd64.\n",
            "Preparing to unpack .../12-libfl-dev_2.6.4-6_amd64.deb ...\n",
            "Unpacking libfl-dev:amd64 (2.6.4-6) ...\n",
            "Selecting previously unselected package libtool.\n",
            "Preparing to unpack .../13-libtool_2.4.6-2_all.deb ...\n",
            "Unpacking libtool (2.4.6-2) ...\n",
            "Selecting previously unselected package python-pip-whl.\n",
            "Preparing to unpack .../14-python-pip-whl_9.0.1-2.3~ubuntu1.18.04.4_all.deb ...\n",
            "Unpacking python-pip-whl (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Selecting previously unselected package python3-asn1crypto.\n",
            "Preparing to unpack .../15-python3-asn1crypto_0.24.0-1_all.deb ...\n",
            "Unpacking python3-asn1crypto (0.24.0-1) ...\n",
            "Selecting previously unselected package python3-cffi-backend.\n",
            "Preparing to unpack .../16-python3-cffi-backend_1.11.5-1_amd64.deb ...\n",
            "Unpacking python3-cffi-backend (1.11.5-1) ...\n",
            "Selecting previously unselected package python3-crypto.\n",
            "Preparing to unpack .../17-python3-crypto_2.6.1-8ubuntu2_amd64.deb ...\n",
            "Unpacking python3-crypto (2.6.1-8ubuntu2) ...\n",
            "Selecting previously unselected package python3-idna.\n",
            "Preparing to unpack .../18-python3-idna_2.6-1_all.deb ...\n",
            "Unpacking python3-idna (2.6-1) ...\n",
            "Selecting previously unselected package python3-six.\n",
            "Preparing to unpack .../19-python3-six_1.11.0-2_all.deb ...\n",
            "Unpacking python3-six (1.11.0-2) ...\n",
            "Selecting previously unselected package python3-cryptography.\n",
            "Preparing to unpack .../20-python3-cryptography_2.1.4-1ubuntu1.4_amd64.deb ...\n",
            "Unpacking python3-cryptography (2.1.4-1ubuntu1.4) ...\n",
            "Selecting previously unselected package python3-secretstorage.\n",
            "Preparing to unpack .../21-python3-secretstorage_2.3.1-2_all.deb ...\n",
            "Unpacking python3-secretstorage (2.3.1-2) ...\n",
            "Selecting previously unselected package python3-keyring.\n",
            "Preparing to unpack .../22-python3-keyring_10.6.0-1_all.deb ...\n",
            "Unpacking python3-keyring (10.6.0-1) ...\n",
            "Selecting previously unselected package python3-keyrings.alt.\n",
            "Preparing to unpack .../23-python3-keyrings.alt_3.0-1_all.deb ...\n",
            "Unpacking python3-keyrings.alt (3.0-1) ...\n",
            "Selecting previously unselected package python3-pip.\n",
            "Preparing to unpack .../24-python3-pip_9.0.1-2.3~ubuntu1.18.04.4_all.deb ...\n",
            "Unpacking python3-pip (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Selecting previously unselected package python3-pkg-resources.\n",
            "Preparing to unpack .../25-python3-pkg-resources_39.0.1-2_all.deb ...\n",
            "Unpacking python3-pkg-resources (39.0.1-2) ...\n",
            "Selecting previously unselected package python3-setuptools.\n",
            "Preparing to unpack .../26-python3-setuptools_39.0.1-2_all.deb ...\n",
            "Unpacking python3-setuptools (39.0.1-2) ...\n",
            "Selecting previously unselected package python3-wheel.\n",
            "Preparing to unpack .../27-python3-wheel_0.30.0-0.2_all.deb ...\n",
            "Unpacking python3-wheel (0.30.0-0.2) ...\n",
            "Selecting previously unselected package python3-xdg.\n",
            "Preparing to unpack .../28-python3-xdg_0.25-4ubuntu1_all.deb ...\n",
            "Unpacking python3-xdg (0.25-4ubuntu1) ...\n",
            "Setting up python-pip-whl (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Setting up python3-cffi-backend (1.11.5-1) ...\n",
            "Setting up libsigsegv2:amd64 (2.12-1) ...\n",
            "Setting up python3-crypto (2.6.1-8ubuntu2) ...\n",
            "Setting up python3-idna (2.6-1) ...\n",
            "Setting up python3-xdg (0.25-4ubuntu1) ...\n",
            "Setting up python3-six (1.11.0-2) ...\n",
            "Setting up python3-wheel (0.30.0-0.2) ...\n",
            "Setting up m4 (1.4.18-1) ...\n",
            "Setting up python3-pkg-resources (39.0.1-2) ...\n",
            "Setting up libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Setting up python3-asn1crypto (0.24.0-1) ...\n",
            "Setting up autotools-dev (20180224.1) ...\n",
            "Setting up python3-pip (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Setting up libbison-dev:amd64 (2:3.0.4.dfsg-1build1) ...\n",
            "Setting up libfl2:amd64 (2.6.4-6) ...\n",
            "Setting up bison (2:3.0.4.dfsg-1build1) ...\n",
            "update-alternatives: using /usr/bin/bison.yacc to provide /usr/bin/yacc (yacc) in auto mode\n",
            "Setting up python3-setuptools (39.0.1-2) ...\n",
            "Setting up python3-cryptography (2.1.4-1ubuntu1.4) ...\n",
            "Setting up flex (2.6.4-6) ...\n",
            "Setting up python3-keyrings.alt (3.0-1) ...\n",
            "Setting up autoconf (2.69-11) ...\n",
            "Setting up file (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libfl-dev:amd64 (2.6.4-6) ...\n",
            "Setting up automake (1:1.15.1-3ubuntu2) ...\n",
            "update-alternatives: using /usr/bin/automake-1.15 to provide /usr/bin/automake (automake) in auto mode\n",
            "Setting up python3-secretstorage (2.3.1-2) ...\n",
            "Setting up libtool (2.4.6-2) ...\n",
            "Setting up python3-keyring (10.6.0-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.2) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rwou6Htk7raf",
        "outputId": "442f2ff9-930e-479f-b5d9-36917c659de5"
      },
      "source": [
        "# download, build and install flatbuffers\n",
        "\n",
        "!git clone https://github.com/google/flatbuffers.git\n",
        "# all these commands have to be run in the same directory and !cd doesn't change\n",
        "# the directory permanently in colab see: \n",
        "# https://stackoverflow.com/questions/48298146/changing-directory-in-google-colab-breaking-out-of-the-python-interpreter\n",
        "!cd flatbuffers && cmake -G \"Unix Makefiles\" && make && sudo make install"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'flatbuffers'...\n",
            "remote: Enumerating objects: 19411, done.\u001b[K\n",
            "remote: Total 19411 (delta 0), reused 0 (delta 0), pack-reused 19411\u001b[K\n",
            "Receiving objects: 100% (19411/19411), 11.73 MiB | 26.12 MiB/s, done.\n",
            "Resolving deltas: 100% (13497/13497), done.\n",
            "-- The C compiler identification is GNU 7.5.0\n",
            "-- The CXX compiler identification is GNU 7.5.0\n",
            "-- Check for working C compiler: /usr/bin/cc\n",
            "-- Check for working C compiler: /usr/bin/cc -- works\n",
            "-- Detecting C compiler ABI info\n",
            "-- Detecting C compiler ABI info - done\n",
            "-- Detecting C compile features\n",
            "-- Detecting C compile features - done\n",
            "-- Check for working CXX compiler: /usr/bin/c++\n",
            "-- Check for working CXX compiler: /usr/bin/c++ -- works\n",
            "-- Detecting CXX compiler ABI info\n",
            "-- Detecting CXX compiler ABI info - done\n",
            "-- Detecting CXX compile features\n",
            "-- Detecting CXX compile features - done\n",
            "-- Looking for strtof_l\n",
            "-- Looking for strtof_l - found\n",
            "-- Looking for strtoull_l\n",
            "-- Looking for strtoull_l - found\n",
            "-- `tests/monster_test.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/monster_test.fbs`: add generation of binary (.bfbs) schema\n",
            "-- `tests/namespace_test/namespace_test1.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/namespace_test/namespace_test2.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/union_vector/union_vector.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/optional_scalars.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/native_type_test.fbs`: add generation of C++ code with ''\n",
            "-- `tests/arrays_test.fbs`: add generation of C++ code with '--scoped-enums;--gen-compare'\n",
            "-- `tests/arrays_test.fbs`: add generation of binary (.bfbs) schema\n",
            "-- `tests/monster_test.fbs`: add generation of C++ embedded binary schema code with '--no-includes;--gen-compare'\n",
            "-- `tests/monster_extra.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `samples/monster.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `samples/monster.fbs`: add generation of binary (.bfbs) schema\n",
            "Proceeding with version: 1.12.0.219\n",
            "-- Configuring done\n",
            "-- Generating done\n",
            "-- Build files have been written to: /content/flatbuffers\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatbuffers\u001b[0m\n",
            "[  1%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[  2%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[  3%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/reflection.cpp.o\u001b[0m\n",
            "[  4%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/util.cpp.o\u001b[0m\n",
            "[  5%] \u001b[32m\u001b[1mLinking CXX static library libflatbuffers.a\u001b[0m\n",
            "[  5%] Built target flatbuffers\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatc\u001b[0m\n",
            "[  6%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[  7%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[  9%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 10%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/util.cpp.o\u001b[0m\n",
            "[ 11%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_cpp.cpp.o\u001b[0m\n",
            "[ 12%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_csharp.cpp.o\u001b[0m\n",
            "[ 13%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_dart.cpp.o\u001b[0m\n",
            "[ 14%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_kotlin.cpp.o\u001b[0m\n",
            "[ 15%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_go.cpp.o\u001b[0m\n",
            "[ 17%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_java.cpp.o\u001b[0m\n",
            "[ 18%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_js_ts.cpp.o\u001b[0m\n",
            "[ 19%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_php.cpp.o\u001b[0m\n",
            "[ 20%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_python.cpp.o\u001b[0m\n",
            "[ 21%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_lobster.cpp.o\u001b[0m\n",
            "[ 22%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_lua.cpp.o\u001b[0m\n",
            "[ 23%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_rust.cpp.o\u001b[0m\n",
            "[ 25%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_fbs.cpp.o\u001b[0m\n",
            "[ 26%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_grpc.cpp.o\u001b[0m\n",
            "[ 27%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_json_schema.cpp.o\u001b[0m\n",
            "[ 28%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_swift.cpp.o\u001b[0m\n",
            "[ 29%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/flatc.cpp.o\u001b[0m\n",
            "[ 30%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/flatc_main.cpp.o\u001b[0m\n",
            "[ 31%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/code_generators.cpp.o\u001b[0m\n",
            "[ 32%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/cpp_generator.cc.o\u001b[0m\n",
            "[ 34%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/go_generator.cc.o\u001b[0m\n",
            "[ 35%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/java_generator.cc.o\u001b[0m\n",
            "[ 36%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/python_generator.cc.o\u001b[0m\n",
            "[ 37%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/swift_generator.cc.o\u001b[0m\n",
            "[ 38%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/ts_generator.cc.o\u001b[0m\n",
            "[ 39%] \u001b[32m\u001b[1mLinking CXX executable flatc\u001b[0m\n",
            "[ 39%] Built target flatc\n",
            "\u001b[35m\u001b[1mScanning dependencies of target generated_code\u001b[0m\n",
            "[ 40%] \u001b[34m\u001b[1mRun generation: 'samples/monster.bfbs'\u001b[0m\n",
            "[ 42%] \u001b[34m\u001b[1mRun generation: 'tests/monster_test_generated.h'\u001b[0m\n",
            "[ 43%] \u001b[34m\u001b[1mRun generation: 'tests/monster_test.bfbs'\u001b[0m\n",
            "[ 44%] \u001b[34m\u001b[1mRun generation: 'tests/namespace_test/namespace_test1_generated.h'\u001b[0m\n",
            "[ 45%] \u001b[34m\u001b[1mRun generation: 'tests/namespace_test/namespace_test2_generated.h'\u001b[0m\n",
            "[ 46%] \u001b[34m\u001b[1mRun generation: 'tests/union_vector/union_vector_generated.h'\u001b[0m\n",
            "[ 47%] \u001b[34m\u001b[1mRun generation: 'tests/optional_scalars_generated.h'\u001b[0m\n",
            "[ 48%] \u001b[34m\u001b[1mRun generation: 'tests/native_type_test_generated.h'\u001b[0m\n",
            "[ 50%] \u001b[34m\u001b[1mRun generation: 'tests/arrays_test_generated.h'\u001b[0m\n",
            "[ 51%] \u001b[34m\u001b[1mRun generation: 'tests/arrays_test.bfbs'\u001b[0m\n",
            "[ 52%] \u001b[34m\u001b[1mRun generation: 'tests/monster_test_bfbs_generated.h'\u001b[0m\n",
            "[ 53%] \u001b[34m\u001b[1mRun generation: 'tests/monster_extra_generated.h'\u001b[0m\n",
            "[ 54%] \u001b[34m\u001b[1mRun generation: 'samples/monster_generated.h'\u001b[0m\n",
            "[ 55%] \u001b[34m\u001b[1mAll generated files were updated.\u001b[0m\n",
            "[ 55%] Built target generated_code\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flattests\u001b[0m\n",
            "[ 56%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[ 57%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[ 59%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 60%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/util.cpp.o\u001b[0m\n",
            "[ 61%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/idl_gen_fbs.cpp.o\u001b[0m\n",
            "[ 62%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/test.cpp.o\u001b[0m\n",
            "[ 63%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/test_assert.cpp.o\u001b[0m\n",
            "[ 64%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/test_builder.cpp.o\u001b[0m\n",
            "[ 65%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/native_type_test_impl.cpp.o\u001b[0m\n",
            "[ 67%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/code_generators.cpp.o\u001b[0m\n",
            "[ 68%] \u001b[32m\u001b[1mLinking CXX executable flattests\u001b[0m\n",
            "[ 78%] Built target flattests\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatsamplebinary\u001b[0m\n",
            "[ 79%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebinary.dir/samples/sample_binary.cpp.o\u001b[0m\n",
            "[ 80%] \u001b[32m\u001b[1mLinking CXX executable flatsamplebinary\u001b[0m\n",
            "[ 81%] Built target flatsamplebinary\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flathash\u001b[0m\n",
            "[ 82%] \u001b[32mBuilding CXX object CMakeFiles/flathash.dir/src/flathash.cpp.o\u001b[0m\n",
            "[ 84%] \u001b[32m\u001b[1mLinking CXX executable flathash\u001b[0m\n",
            "[ 84%] Built target flathash\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatsampletext\u001b[0m\n",
            "[ 85%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[ 86%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[ 87%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 88%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/util.cpp.o\u001b[0m\n",
            "[ 89%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/samples/sample_text.cpp.o\u001b[0m\n",
            "[ 90%] \u001b[32m\u001b[1mLinking CXX executable flatsampletext\u001b[0m\n",
            "[ 92%] Built target flatsampletext\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatsamplebfbs\u001b[0m\n",
            "[ 93%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[ 94%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[ 95%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 96%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/util.cpp.o\u001b[0m\n",
            "[ 97%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/samples/sample_bfbs.cpp.o\u001b[0m\n",
            "[ 98%] \u001b[32m\u001b[1mLinking CXX executable flatsamplebfbs\u001b[0m\n",
            "[100%] Built target flatsamplebfbs\n",
            "[  5%] Built target flatbuffers\n",
            "[ 39%] Built target flatc\n",
            "[ 40%] \u001b[34m\u001b[1mAll generated files were updated.\u001b[0m\n",
            "[ 55%] Built target generated_code\n",
            "[ 78%] Built target flattests\n",
            "[ 81%] Built target flatsamplebinary\n",
            "[ 84%] Built target flathash\n",
            "[ 92%] Built target flatsampletext\n",
            "[100%] Built target flatsamplebfbs\n",
            "\u001b[36mInstall the project...\u001b[0m\n",
            "-- Install configuration: \"\"\n",
            "-- Installing: /usr/local/include/flatbuffers\n",
            "-- Installing: /usr/local/include/flatbuffers/pch\n",
            "-- Installing: /usr/local/include/flatbuffers/pch/flatc_pch.h\n",
            "-- Installing: /usr/local/include/flatbuffers/pch/pch.h\n",
            "-- Installing: /usr/local/include/flatbuffers/flexbuffers.h\n",
            "-- Installing: /usr/local/include/flatbuffers/flatc.h\n",
            "-- Installing: /usr/local/include/flatbuffers/grpc.h\n",
            "-- Installing: /usr/local/include/flatbuffers/registry.h\n",
            "-- Installing: /usr/local/include/flatbuffers/minireflect.h\n",
            "-- Installing: /usr/local/include/flatbuffers/code_generators.h\n",
            "-- Installing: /usr/local/include/flatbuffers/flatbuffers.h\n",
            "-- Installing: /usr/local/include/flatbuffers/hash.h\n",
            "-- Installing: /usr/local/include/flatbuffers/util.h\n",
            "-- Installing: /usr/local/include/flatbuffers/base.h\n",
            "-- Installing: /usr/local/include/flatbuffers/idl.h\n",
            "-- Installing: /usr/local/include/flatbuffers/reflection_generated.h\n",
            "-- Installing: /usr/local/include/flatbuffers/stl_emulation.h\n",
            "-- Installing: /usr/local/include/flatbuffers/reflection.h\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersConfig.cmake\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersConfigVersion.cmake\n",
            "-- Installing: /usr/local/lib/libflatbuffers.a\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersTargets.cmake\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersTargets-noconfig.cmake\n",
            "-- Installing: /usr/local/bin/flatc\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatcTargets.cmake\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatcTargets-noconfig.cmake\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jbORi6RC7uCE",
        "outputId": "94ccc2ca-e0da-4fca-ee7e-f1d74e0955ba"
      },
      "source": [
        "# the next step requires a version of cmake > 3.14.0\n",
        "!pip install cmake==3.15.3"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting cmake==3.15.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ff/34/0a311fedffcc7a153bbc0390ef4c378dbc7f09f9865247137f82d62f8e7a/cmake-3.15.3-py3-none-manylinux2010_x86_64.whl (16.5MB)\n",
            "\u001b[K     |████████████████████████████████| 16.5MB 208kB/s \n",
            "\u001b[?25hInstalling collected packages: cmake\n",
            "  Found existing installation: cmake 3.12.0\n",
            "    Uninstalling cmake-3.12.0:\n",
            "      Successfully uninstalled cmake-3.12.0\n",
            "Successfully installed cmake-3.15.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "exofv2jL-3U8",
        "outputId": "2e8da37c-d789-477f-ae48-ce69651b8d0f"
      },
      "source": [
        "# add -v for verbose if there are any errors\n",
        "!pip install nle==0.5.2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting nle==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/70/6c/412567cf66bd4a9009ff8fdc077b2b71f1e660f16d2c06c68921fd0dbe48/nle-0.5.2.tar.gz (6.7MB)\n",
            "\u001b[K     |████████████████████████████████| 6.7MB 13.4MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from nle==0.5.2) (1.18.5)\n",
            "Collecting pybind11>=2.2\n",
            "  Using cached https://files.pythonhosted.org/packages/00/84/fc9dc13ee536ba5e6b8fd10ce368fea5b738fe394c3b296cde7c9b144a92/pybind11-2.6.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: gym>=0.15 in /usr/local/lib/python3.6/dist-packages (from nle==0.5.2) (0.17.3)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle==0.5.2) (1.3.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle==0.5.2) (1.4.1)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle==0.5.2) (1.5.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym>=0.15->nle==0.5.2) (0.16.0)\n",
            "Building wheels for collected packages: nle\n",
            "  Building wheel for nle (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nle: filename=nle-0.5.2-cp36-cp36m-linux_x86_64.whl size=2836551 sha256=0aee4f052896be66d057f0e6b18b939dc76e3c165f9aa1f927567731ca3980e8\n",
            "  Stored in directory: /root/.cache/pip/wheels/3c/e1/2e/ece6eaf3e2382f04888155cb96fcb005f94c988d5f0c315d00\n",
            "Successfully built nle\n",
            "Installing collected packages: pybind11, nle\n",
            "Successfully installed nle-0.5.2 pybind11-2.6.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iky0Xmi1-L5k",
        "outputId": "6bdedd70-9de5-463a-a776-f6a1ba629a49"
      },
      "source": [
        "!pip install \"nle[agent]\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: nle[agent] in /usr/local/lib/python3.6/dist-packages (0.5.2)\n",
            "Requirement already satisfied: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (1.18.5)\n",
            "Requirement already satisfied: pybind11>=2.2 in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (2.6.1)\n",
            "Requirement already satisfied: gym>=0.15 in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (0.17.3)\n",
            "Requirement already satisfied: torch>=1.3.1; extra == \"agent\" in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (1.7.0+cu101)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle[agent]) (1.5.0)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle[agent]) (1.3.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle[agent]) (1.4.1)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch>=1.3.1; extra == \"agent\"->nle[agent]) (0.8)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.3.1; extra == \"agent\"->nle[agent]) (0.16.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch>=1.3.1; extra == \"agent\"->nle[agent]) (3.7.4.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eee2-Pfr_KUz",
        "outputId": "95ac1eb1-214d-49cf-a08c-b9cf2dc5bab0"
      },
      "source": [
        "!pip install pyvirtualdisplay"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyvirtualdisplay\n",
            "  Downloading https://files.pythonhosted.org/packages/d0/8a/643043cc70791367bee2d19eb20e00ed1a246ac48e5dbe57bbbcc8be40a9/PyVirtualDisplay-1.3.2-py2.py3-none-any.whl\n",
            "Collecting EasyProcess\n",
            "  Downloading https://files.pythonhosted.org/packages/48/3c/75573613641c90c6d094059ac28adb748560d99bd27ee6f80cce398f404e/EasyProcess-0.3-py2.py3-none-any.whl\n",
            "Installing collected packages: EasyProcess, pyvirtualdisplay\n",
            "Successfully installed EasyProcess-0.3 pyvirtualdisplay-1.3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zz94oU2p6BEc"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nsHNDA4d5Mkn"
      },
      "source": [
        "\n",
        "import gym\n",
        "from gym import logger as gymlogger\n",
        "from gym.wrappers import Monitor\n",
        "gymlogger.set_level(40) #error only\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import random\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "# %matplotlib inline\n",
        "import math\n",
        "import glob\n",
        "import io\n",
        "import os\n",
        "import base64\n",
        "from IPython.display import HTML\n",
        "\n",
        "from IPython import display as ipythondisplay\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torch import multiprocessing as mp\n",
        "import torch.distributions as distributions\n",
        "from torchvision import datasets, transforms\n",
        "\n",
        "from nle import nethack\n",
        "\n",
        "from pyvirtualdisplay import Display\n",
        "# display = Display(visible=0, size=(1400, 900))\n",
        "# display.start()\n",
        "from collections import defaultdict\n",
        "\n",
        "Colab = True\n",
        "\n",
        "if Colab:\n",
        "    # Imports specifically for google colab/drive\n",
        "    from google.colab import files, drive\n",
        "    drive.mount('/content/gdrive')\n",
        "    dir = '/content/gdrive/My Drive/Colab Notebooks/Nethack/'\n",
        "    os.makedirs(dir, exist_ok=True)\n",
        "else:\n",
        "    dir = './'\n",
        "\n",
        "path = lambda fname: os.path.join(dir, fname)\n",
        "\n",
        "from tqdm import tqdm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fWeMBF2MB0-i"
      },
      "source": [
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "print(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "78F0UtjRJzXf"
      },
      "source": [
        "# Crop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eMHCmWZ0aG-y"
      },
      "source": [
        "\n",
        "class Crop(nn.Module):\n",
        "    \"\"\"Helper class for NetHackNet below.\"\"\"\n",
        "\n",
        "    def __init__(self, height, width, height_target, width_target):\n",
        "        super(Crop, self).__init__()\n",
        "        self.width = width\n",
        "        self.height = height\n",
        "        self.width_target = width_target\n",
        "        self.height_target = height_target\n",
        "        width_grid = _step_to_range(2 / (self.width - 1), self.width_target)[\n",
        "            None, :\n",
        "        ].expand(self.height_target, -1)\n",
        "        height_grid = _step_to_range(2 / (self.height - 1), height_target)[\n",
        "            :, None\n",
        "        ].expand(-1, self.width_target)\n",
        "\n",
        "        # \"clone\" necessary, https://github.com/pytorch/pytorch/issues/34880\n",
        "        self.register_buffer(\"width_grid\", width_grid.clone())\n",
        "        self.register_buffer(\"height_grid\", height_grid.clone())\n",
        "\n",
        "    def forward(self, inputs, coordinates):\n",
        "        \"\"\"Calculates centered crop around given x,y coordinates.\n",
        "        Args:\n",
        "           inputs [B x H x W]\n",
        "           coordinates [B x 2] x,y coordinates\n",
        "        Returns:\n",
        "           [B x H' x W'] inputs cropped and centered around x,y coordinates.\n",
        "        \"\"\"\n",
        "        assert inputs.shape[1] == self.height, f'{ inputs.shape[1],self.height}'\n",
        "        assert inputs.shape[2] == self.width\n",
        "\n",
        "        inputs = inputs[:, None, :, :].float()\n",
        "\n",
        "        x = coordinates[:, 0]\n",
        "        y = coordinates[:, 1]\n",
        "\n",
        "        x_shift = 2 / (self.width - 1) * (x.float() - self.width // 2)\n",
        "        y_shift = 2 / (self.height - 1) * (y.float() - self.height // 2)\n",
        "\n",
        "        grid = torch.stack(\n",
        "            [\n",
        "                self.width_grid[None, :, :] + x_shift[:, None, None],\n",
        "                self.height_grid[None, :, :] + y_shift[:, None, None],\n",
        "            ],\n",
        "            dim=3,\n",
        "        )\n",
        "\n",
        "        # TODO: only cast to int if original tensor was int\n",
        "        return (\n",
        "            torch.round(F.grid_sample(inputs, grid, align_corners=True))\n",
        "            .squeeze(1)\n",
        "            .long()\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3XhI8FXGJTg5"
      },
      "source": [
        "# Replay Buffer\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CqQ-PZPSJYBM"
      },
      "source": [
        "import os\n",
        "import matplotlib as mp\n",
        "if os.environ.get('DISPLAY','') == '':\n",
        "    print('no display found. Using non-interactive Agg backend')\n",
        "    mp.use('Agg')\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import numpy as np\n",
        "import gym\n",
        "import nle\n",
        "import random\n",
        "from gym import spaces\n",
        "\n",
        "from collections import deque\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch\n",
        "import random\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "\"\"\"replay buffer\"\"\"\n",
        "class ReplayBuffer:\n",
        "    \"\"\"\n",
        "    Simple storage for transitions from an environment.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, size):\n",
        "        \"\"\"\n",
        "        Initialise a buffer of a given size for storing transitions\n",
        "        :param size: the maximum number of transitions that can be stored\n",
        "        \"\"\"\n",
        "        self._storage = []\n",
        "        self._maxsize = size\n",
        "        self._next_idx = 0\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self._storage)\n",
        "\n",
        "    def add(self, state, action, reward, next_state, done):\n",
        "        \"\"\"\n",
        "        Add a transition to the buffer. Old transitions will be overwritten if the buffer is full.\n",
        "        :param state: the agent's initial state\n",
        "        :param action: the action taken by the agent\n",
        "        :param reward: the reward the agent received\n",
        "        :param next_state: the subsequent state\n",
        "        :param done: whether the episode terminated\n",
        "        \"\"\"\n",
        "        data = (state, action, reward, next_state, done)\n",
        "        # print(\"state type\",state.type)\n",
        "        if self._next_idx >= len(self._storage):\n",
        "            self._storage.append(data)\n",
        "        else:\n",
        "            self._storage[self._next_idx] = data\n",
        "        self._next_idx = (self._next_idx + 1) % self._maxsize\n",
        "\n",
        "    def _encode_sample(self, indices):\n",
        "        states, actions, rewards, next_states, dones = {}, [], [], [], []\n",
        "        glyphs, blstats, glyphs_ns, blstats_ns = [], [], [], []\n",
        "        ## come back and fix this\n",
        "        for i in indices:\n",
        "            data = self._storage[i]\n",
        "            state, action, reward, next_state, done = data\n",
        "            # states.append(dict_to_device(state))\n",
        "            glyphs.append(state['glyphs'].to(device))\n",
        "            blstats.append(state['blstats'].to(device))\n",
        "            glyphs_ns.append(next_state['glyphs'].to(device))\n",
        "            blstats_ns.append(next_state['blstats'].to(device))            \n",
        "            actions.append(action)\n",
        "            rewards.append(reward)\n",
        "            # next_states.append(dict_to_device(next_state))\n",
        "            dones.append(done)\n",
        "        \n",
        "        dones = torch.stack(dones)\n",
        "        states = {'glyphs': torch.cat(glyphs), 'blstats': torch.cat(blstats)}\n",
        "        next_states = {'glyphs': torch.cat(glyphs_ns), 'blstats': torch.cat(blstats_ns)}\n",
        "\n",
        "        return (\n",
        "            (states),\n",
        "            (actions),\n",
        "            (rewards),\n",
        "            (next_states),\n",
        "            (dones),\n",
        "        )\n",
        "\n",
        "    def sample(self, batch_size):\n",
        "        \"\"\"\n",
        "        Randomly sample a batch of transitions from the buffer.\n",
        "        :param batch_size: the number of transitions to sample\n",
        "        :return: a mini-batch of sampled transitions\n",
        "        \"\"\"\n",
        "        indices = np.random.randint(0, len(self._storage) - 1, size=batch_size)\n",
        "        return self._encode_sample(indices)\n",
        "\n",
        "def to_state(state):\n",
        "    glyphs_matrix = state['glyphs'] # T x B x H x W\n",
        "    agent_stat = state['blstats']\n",
        "    agent_row = state['blstats'][1]\n",
        "    agent_col = state['blstats'][0]\n",
        "\n",
        "    around_agent = np.zeros([9,9])\n",
        "    row = agent_row - 4\n",
        "    col = agent_col -4\n",
        "    for i in range(9):\n",
        "        for j in range(9):\n",
        "            if row>0 and row<glyphs_matrix.shape[0] and col>0 and col<glyphs_matrix.shape[1]:\n",
        "                around_agent[i][j] = glyphs_matrix[row][col]\n",
        "            col+=1\n",
        "        col = agent_col -4\n",
        "        row+=1\n",
        "        \n",
        "    return glyphs_matrix.copy(),around_agent.copy(),agent_stat.copy()\n",
        "    \n",
        "    \n",
        "class AbstractAgent:\n",
        "    \"\"\"\n",
        "    AbstractAgent\n",
        "\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        raise NotImplementedError()\n",
        "\n",
        "    def act(self, observation):\n",
        "        raise NotImplementedError()\n",
        "\n",
        "\n",
        "class MyAgent(AbstractAgent):\n",
        "    def __init__(self, observation_space, action_space):\n",
        "        self.observation_space = observation_space\n",
        "        self.action_space = action_space\n",
        "        # TODO Initialise your agent's models\n",
        "        self.model = DQN(observation_space,action_space).to(device)\n",
        "\n",
        "        # for example, if your agent had a Pytorch model it must be load here\n",
        "        self.model.load_state_dict(torch.load( 'models/dqn_nethack.pth', map_location=torch.device(device)))\n",
        "\n",
        "    def act(self, observation):\n",
        "        # Perform processing to observation\n",
        "\n",
        "        # TODO: return selected action\n",
        "        state = format_observations(observation)\n",
        "        # glyphs_matrix,around_agent,agent_stat = to_state(observation)\n",
        "        # glyphs_matrix = torch.from_numpy(glyphs_matrix).type(torch.FloatTensor).to(device).unsqueeze(0).unsqueeze(0)/5991.0\n",
        "        # agent_stat = torch.from_numpy(agent_stat).type(torch.FloatTensor).to(device).unsqueeze(0)\n",
        "        # around_agent = torch.from_numpy(around_agent).type(torch.FloatTensor).to(device).unsqueeze(0).unsqueeze(0)/5991.0\n",
        "        # state = (glyphs_matrix,around_agent,agent_stat)\n",
        "\n",
        "        with torch.no_grad():\n",
        "            outputs, core_state = self.model(state, core_state, dones)\n",
        "            action = torch.argmax(outputs)\n",
        "        return action.item()\n",
        "\n",
        "class RandomAgent(AbstractAgent):\n",
        "    def __init__(self, observation_space, action_space):\n",
        "        self.observation_space = observation_space\n",
        "        self.action_space = action_space\n",
        "\n",
        "    def act(self, observation):\n",
        "        return self.action_space.sample()\n",
        "\n",
        "\n",
        "def run_episode(env):\n",
        "    # create instance of MyAgent\n",
        "    # from MyAgent import MyAgent\n",
        "    agent = MyAgent(env.observation_space,env.action_space.n)\n",
        "\n",
        "    done = False\n",
        "    episode_return = 0.0\n",
        "    state = env.reset()\n",
        "\n",
        "    save_dir = './animations_nethack_dqn/'\n",
        "    if not os.path.exists(save_dir):\n",
        "        os.makedirs(save_dir)\n",
        "\n",
        "    try:\n",
        "        env = gym.wrappers.Monitor(\n",
        "        env, save_dir, video_callable=lambda episode_id: True)\n",
        "    except gym.error.Error as e:\n",
        "        print(e)\n",
        "\n",
        "    while not done:\n",
        "        # pass state to agent and let agent decide action\n",
        "        action = agent.act(state)\n",
        "        env.render()\n",
        "        new_state, reward, done, _ = env.step(action)\n",
        "        episode_return += reward\n",
        "        state = new_state\n",
        "    return episode_return\n",
        "\n",
        "\n",
        "\"\"\"model\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVQo-8K7Vkab"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CkTTjNITVLiL"
      },
      "source": [
        "# DQN Agent"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_X5WYGVAVDpQ"
      },
      "source": [
        "\n",
        "class DQNAgent:\n",
        "    def __init__(self,observation_space: spaces.Box,action_space,replay_buffer,lr,batch_size,gamma):\n",
        "        \"\"\"\n",
        "        Initialise the DQN algorithm using the Adam optimiser\n",
        "        :param action_space: the action space of the environment\n",
        "        :param observation_space: the state space of the environment\n",
        "        :param replay_buffer: storage for experience replay\n",
        "        :param lr: the learning rate for Adam\n",
        "        :param batch_size: the batch size\n",
        "        :param gamma: the discount factor\n",
        "        \"\"\"\n",
        "\n",
        "        self.grad_norm_clipping = 40.0\n",
        "\n",
        "        self.action_space = action_space\n",
        "        self.observation_space = observation_space\n",
        "        self.dqn = DQN(observation_space,action_space)\n",
        "        self.target = DQN(observation_space,action_space)\n",
        "        self.criterion = nn.L1Loss()\n",
        "        self.optimizer = optim.RMSprop(self.dqn.parameters(), lr=lr,eps=0.000001)\n",
        "        self.dqn.to(device)\n",
        "        self.target.to(device)\n",
        "        self.replay_buffer = replay_buffer\n",
        "        self.batch_size = batch_size\n",
        "        self.gamma = gamma\n",
        "        self.update_target_network()\n",
        "        self.core_state = self.dqn.initial_state()\n",
        "\n",
        "    def optimise_td_loss(self):\n",
        "        \"\"\"\n",
        "        Optimise the TD-error over a single minibatch of transitions\n",
        "        :return: the loss\n",
        "        \"\"\"\n",
        "        states, actions, rewards, next_states, dones = self.replay_buffer.sample(self.batch_size)\n",
        "\n",
        "        # actions = torch.from_numpy(np.array(actions)).to(device)\n",
        "        # rewards = torch.from_numpy(np.array(rewards)).to(device)\n",
        "        # dones = torch.from_numpy(np.array(dones)).to(device)\n",
        "        \n",
        "        # matrixs = torch.from_numpy(matrixs).type(torch.FloatTensor).to(device).unsqueeze(1)/5991.0\n",
        "        # agent_stats = torch.from_numpy(agent_stats).type(torch.FloatTensor).to(device)\n",
        "        # around_agents = torch.from_numpy(around_agents).type(torch.FloatTensor).to(device).unsqueeze(1)/5991.0\n",
        "        # states = (matrixs,around_agents,agent_stats)\n",
        "        \n",
        "        # matrix_ps = torch.from_numpy(matrix_ps).type(torch.FloatTensor).to(device).unsqueeze(1)/5991.0\n",
        "        # agent_stat_ps = torch.from_numpy(agent_stat_ps).type(torch.FloatTensor).to(device)\n",
        "        # around_agent_ps = torch.from_numpy(around_agent_ps).type(torch.FloatTensor).to(device).unsqueeze(1)/5991.0\n",
        "        # next_states = (matrix_ps,around_agent_ps,agent_stat_ps)\n",
        "        # states = list_dict_to_device(states)\n",
        "        # next_states =  dict_to_device(next_states)\n",
        "        \n",
        "        actions = list_to_device(actions)\n",
        "        rewards = list_to_device(rewards)\n",
        "       \n",
        "        dones = list_to_device(dones)\n",
        "        prediction, self.core_state = self.dqn(states, self.core_state, dones)\n",
        "        current_q_value = prediction['policy_logits'].squeeze().gather(1,actions.unsqueeze(1))\n",
        "\n",
        "\n",
        "        with torch.no_grad():\n",
        "            target_prediction, _ = self.target(next_states, self.core_state, dones)\n",
        "            target_prediction = target_prediction\n",
        "            max_q,_ = torch.max(target_prediction['policy_logits'].squeeze(),1)\n",
        "            target_q_value = rewards.squeeze() + ((1-dones.squeeze().float())*self.gamma*max_q.squeeze())\n",
        "            target_q_value = target_q_value.unsqueeze(1)\n",
        "\n",
        "\n",
        "        loss = nn.functional.smooth_l1_loss(current_q_value, target_q_value)\n",
        "        total_loss = loss.clone().detach().requires_grad_(True)\n",
        "\n",
        "        self.optimizer.zero_grad()\n",
        "        total_loss.backward()\n",
        "        nn.utils.clip_grad_norm_(self.dqn.parameters(), self.grad_norm_clipping)\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return total_loss.item()\n",
        "        \n",
        "    def update_target_network(self):\n",
        "        \"\"\"\n",
        "        Update the target Q-network by copying the weights from the current Q-network\n",
        "        \"\"\"\n",
        "        # TODO update target_network parameters with policy_network parameters\n",
        "        self.target.load_state_dict(self.dqn.state_dict())\n",
        "    \n",
        "    def act(self, state, done):\n",
        "        \"\"\"\n",
        "        Select an action greedily from the Q-network given the state\n",
        "        :param state: the current state\n",
        "        :return: the action to take\n",
        "        \"\"\" \n",
        "        # state = format_observations(state)\n",
        "        # self.DQN(dict_to_device(state), list_to_device(self.agent_state))\n",
        "        with torch.no_grad():\n",
        "            outputs, self.core_state = self.dqn(dict_to_device(state), list_to_device(self.core_state), done.to(device))\n",
        "            action = torch.argmax(outputs['action'])\n",
        "        return action.item()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YIoquaHUTLwz"
      },
      "source": [
        "# Other "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tnMb6S4aJP_1"
      },
      "source": [
        "def _step_to_range(delta, num_steps):\n",
        "    \"\"\"Range of `num_steps` integers with distance `delta` centered around zero.\"\"\"\n",
        "    return delta * torch.arange(-num_steps // 2, num_steps // 2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Sf8_Ffq64MB"
      },
      "source": [
        "def list_dict_to_device(list_dict):\n",
        "    return [ dict_to_device(dict_) for dict_ in list_dict ]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "piQkUlw8OV_i"
      },
      "source": [
        "def dict_to_device(dic):\n",
        "    for key, value in dic.items():\n",
        "        dic[key] = dic[key].to(device)\n",
        "    return dic"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1zats62_OYWk"
      },
      "source": [
        "def list_to_device(l):\n",
        "    l = torch.stack([ t.to(device) for t in l ])\n",
        "    return l"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7cl-pulpHORN"
      },
      "source": [
        "def format_observations(observation, keys=(\"glyphs\", \"blstats\")):\n",
        "    observations = {}\n",
        "    for key in keys:\n",
        "        entry = observation[key]\n",
        "        entry = torch.from_numpy(entry)\n",
        "        entry = entry.view((1, 1) + entry.shape)  # (...) -> (T,B,...).\n",
        "        observations[key] = entry\n",
        "    return observations"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jXcebRy4Fpf9"
      },
      "source": [
        "def save_checkpoint(\n",
        "    fname, agent, step_num,\n",
        "    episode_rewards = None, num_episode_step_counter = None,\n",
        "    train_loss=None, episode_loss=None, episode_loss_av = None,\n",
        "    download=False):\n",
        "    print('saving')\n",
        "    checkpoint = {\n",
        "        'observation_space': agent.observation_space,\n",
        "        'step_num': step_num,\n",
        "        'episode_rewards': episode_rewards,\n",
        "        'train_loss': train_loss,\n",
        "        'num_episode_step_counter':num_episode_step_counter,\n",
        "        # 'episode_loss':episode_loss,\n",
        "        # 'episode_loss_av':episode_loss_av,\n",
        "        'replay_buffer': agent.replay_buffer,\n",
        "        'model_state_dict': agent.dqn.state_dict(),\n",
        "        'target_state_dict': agent.target.state_dict(),\n",
        "        'optimizer_state_dict': agent.optimizer.state_dict(),\n",
        "        'core_state': agent.core_state}\n",
        "\n",
        "    torch.save(checkpoint, path(fname))\n",
        "    if download and Colab: files.download(path) # not working well \n",
        "\n",
        "def load_checkpoint(fname, agent):\n",
        "    # well, this function is just badly written\n",
        "    # checkpoint = torch.load(path(fname), map_location=torch.device(device))\n",
        "    checkpoint = torch.load(path(fname))\n",
        "    agent.dqn.load_state_dict(checkpoint['model_state_dict'])\n",
        "    #  self.model.load_state_dict(torch.load(PATH, map_location=torch.device(device)))\n",
        "    agent.replay_buffer = (checkpoint['replay_buffer'])\n",
        "    agent.target.load_state_dict(checkpoint['target_state_dict'])\n",
        "    agent.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "    agent.dqn.to(device)\n",
        "    agent.target.to(device)\n",
        "    agent.core_state = checkpoint['core_state']\n",
        "\n",
        "    return checkpoint"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WCJgSqXHcybf"
      },
      "source": [
        "# Network old"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JDQiU9OkZyca"
      },
      "source": [
        "\n",
        "# class DQN(nn.Module):\n",
        "#     def __init__(\n",
        "#         self,\n",
        "#         observation_space,\n",
        "#         num_actions,\n",
        "#         use_lstm = True,\n",
        "#         embedding_dim=32,\n",
        "#         crop_dim=9,\n",
        "#         num_layers=5\n",
        "#     ):\n",
        "#         \"\"\"\n",
        "#         Initialise the DQN\n",
        "#         :param observation_space: the state space of the environment\n",
        "#         :param action_space: the action space of the environment\n",
        "#         \"\"\"\n",
        "#         super(DQN,self).__init__()\n",
        "\n",
        "#         self.glyph_shape = observation_space[\"glyphs\"].shape\n",
        "#         self.blstats_size = observation_space[\"blstats\"].shape[0]\n",
        "\n",
        "#         self.num_actions = num_actions\n",
        "#         self.use_lstm = use_lstm\n",
        "\n",
        "#         self.H = self.glyph_shape[0]\n",
        "#         self.W = self.glyph_shape[1]\n",
        "\n",
        "#         self.k_dim = embedding_dim\n",
        "#         self.h_dim = 512\n",
        "\n",
        "#         self.crop_dim = crop_dim\n",
        "\n",
        "#         self.crop = Crop(self.H, self.W, self.crop_dim, self.crop_dim)\n",
        "\n",
        "#         self.embed = nn.Embedding(nethack.MAX_GLYPH, self.k_dim)\n",
        "\n",
        "#         K = embedding_dim  # number of input filters\n",
        "#         F = 3  # filter dimensions # kernel_size\n",
        "#         S = 1  # stride\n",
        "#         P = 1  # padding\n",
        "#         M = 16  # number of intermediate filters\n",
        "#         Y = 8  # number of output filters\n",
        "#         L = num_layers  # number of convnet layers\n",
        "\n",
        "#         self.extract_representation = nn.Sequential\n",
        "#         (\n",
        "#             nn.Conv2d(1, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#         )\n",
        "\n",
        "#         self.extract_crop_representation = nn.Sequential\n",
        "#         (\n",
        "#             nn.Conv2d(1, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#             nn.Conv2d(M, M, kernel_size=F, stride=S,padding = K),\n",
        "#             nn.ELU(),\n",
        "#         )\n",
        "\n",
        "#         out_dim = self.k_dim # embedding_dimension = 32\n",
        "#         out_dim += self.H * self.W * Y # CNN over full glyph map\n",
        "#         out_dim += self.crop_dim ** 2 * Y # CNN crop model.\n",
        "\n",
        "#         self.embed_blstats = nn.Sequential\n",
        "#         (\n",
        "#             nn.Linear(self.blstats_size, self.k_dim),\n",
        "#             nn.ReLU(),\n",
        "#             nn.Linear(self.k_dim, self.k_dim),\n",
        "#             nn.ReLU(),\n",
        "#         )\n",
        "\n",
        "#         self.fc = nn.Sequential\n",
        "#         (\n",
        "#             nn.Linear(out_dim, self.h_dim),\n",
        "#             nn.ReLU(),\n",
        "#             nn.Linear(self.h_dim, self.h_dim),\n",
        "#             nn.ReLU(),\n",
        "#         )\n",
        "\n",
        "#         if self.use_lstm:\n",
        "#             self.core = nn.LSTM(self.h_dim, self.h_dim, num_layers=1)\n",
        "\n",
        "#         self.policy = nn.Linear(self.h_dim, self.num_actions)\n",
        "    \n",
        "\n",
        "#     def initial_state(self, batch_size=1):\n",
        "#         if not self.use_lstm:\n",
        "#             return tuple()\n",
        "#         return tuple((\n",
        "#             torch.zeros(self.core.num_layers, batch_size, self.core.hidden_size),\n",
        "#             torch.zeros(self.core.num_layers, batch_size, self.core.hidden_size)\n",
        "#         ))\n",
        "\n",
        "#     def forward(self, state, core_state, dones=None):\n",
        "#         # -- [T x B x H x W] # \n",
        "#         glyphs = state[\"glyphs\"]\n",
        "\n",
        "#         # -- [T x B x F]\n",
        "#         blstats = state[\"blstats\"]\n",
        "\n",
        "#         T, B, *_ = glyphs.shape\n",
        "\n",
        "#         # -- [B' x H x W]\n",
        "#         glyphs = torch.flatten(glyphs, 0, 1)  # Merge time and batch.\n",
        "\n",
        "#         # -- [B' x F]\n",
        "#         blstats = blstats.view(T * B, -1).float()\n",
        "\n",
        "#         # -- [B x H x W]\n",
        "#         glyphs = glyphs.long()\n",
        "#         # -- [B x 2] x,y coordinates\n",
        "#         coordinates = blstats[:, :2]\n",
        "#         # TODO ???\n",
        "#         # coordinates[:, 0].add_(-1)\n",
        "\n",
        "#         # -- [B x F]\n",
        "#         # FIXME: hack to use compatible blstats to before\n",
        "#         # blstats = blstats[:, [0, 1, 21, 10, 11]]\n",
        "\n",
        "#         blstats = blstats.view(T * B, -1).float()\n",
        "#         # -- [B x K]\n",
        "#         blstats_emb = self.embed_blstats(blstats)\n",
        "\n",
        "#         assert blstats_emb.shape[0] == T * B\n",
        "\n",
        "#         reps = [blstats_emb] # representations\n",
        "\n",
        "#         # -- [B x H' x W']\n",
        "#         crop = self.crop(glyphs, coordinates)\n",
        "\n",
        "#         # print(\"crop\", crop)\n",
        "#         # print(\"at_xy\", glyphs[:, coordinates[:, 1].long(), coordinates[:, 0].long()])\n",
        "\n",
        "#         ## self.extract_crop_representation and self.extract_representation forward pass the same\n",
        "#         # -- [B x H' x W' x K]\n",
        "#         crop_emb = self._select(self.embed, crop)\n",
        "\n",
        "#         # CNN crop model.\n",
        "#         # -- [B x K x W' x H']\n",
        "#         crop_emb = crop_emb.transpose(1, 3)  # -- TODO: slow?\n",
        "#         # -- [B x W' x H' x K]\n",
        "#         crop_rep = self.extract_crop_representation(crop_emb)\n",
        "\n",
        "#         # -- [B x K']\n",
        "#         crop_rep = crop_rep.view(T * B, -1)\n",
        "#         assert crop_rep.shape[0] == T * B\n",
        "\n",
        "#         reps.append(crop_rep)\n",
        "\n",
        "#         # -- [B x H x W x K]\n",
        "#         glyphs_emb = self._select(self.embed, glyphs)\n",
        "#         # glyphs_emb = self.embed(glyphs)\n",
        "#         # -- [B x K x W x H]\n",
        "#         glyphs_emb = glyphs_emb.transpose(1, 3)  # -- TODO: slow?\n",
        "#         # -- [B x W x H x K]\n",
        "#         glyphs_rep = self.extract_representation(glyphs_emb)\n",
        "\n",
        "#         # -- [B x K']\n",
        "#         glyphs_rep = glyphs_rep.view(T * B, -1)\n",
        "#         assert glyphs_rep.shape[0] == T * B\n",
        "\n",
        "#         # -- [B x K'']\n",
        "#         reps.append(glyphs_rep)\n",
        "\n",
        "#         st = torch.cat(reps, dim=1)\n",
        "\n",
        "#         # -- [B x K]\n",
        "#         st = self.fc(st)\n",
        "\n",
        "#         if self.use_lstm:\n",
        "#             core_input = st.view(T, B, -1)\n",
        "#             core_output_list = []\n",
        "#             notdone = (~done).float()\n",
        "#             for input, nd in zip(core_input.unbind(), notdone.unbind()):\n",
        "#                 # Reset core state to zero whenever an episode ended.\n",
        "#                 # Make `done` broadcastable with (num_layers, B, hidden_size)\n",
        "#                 # states:\n",
        "#                 nd = nd.view(1, -1, 1)\n",
        "#                 core_state = tuple(nd * s for s in core_state)\n",
        "#                 output, core_state = self.core(input.unsqueeze(0), core_state)\n",
        "#                 core_output_list.append(output)\n",
        "#             core_output = torch.flatten(torch.cat(core_output_list), 0, 1)\n",
        "#         else:\n",
        "#             core_output = st\n",
        "\n",
        "#         # -- [B x A]\n",
        "#         policy_logits = self.policy(core_output)\n",
        "#         # -- [B x A]\n",
        "        \n",
        "#         print(F.softmax(policy_logits, dim=1))\n",
        "#         if self.training:\n",
        "#             action = torch.multinomial(F.softmax(policy_logits, dim=1), num_samples=1)\n",
        "#         else:\n",
        "#             # Don't sample when testing.\n",
        "#             action = torch.argmax(policy_logits, dim=1)\n",
        "\n",
        "#         policy_logits = policy_logits.view(T, B, self.num_actions)\n",
        "        \n",
        "#         action = action.view(T, B)\n",
        "\n",
        "#         return (\n",
        "#             dict(policy_logits=policy_logits,  action=action),\n",
        "#             core_state,\n",
        "#         )\n",
        "# \"\"\"agent\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CtVUzAk6VoZf"
      },
      "source": [
        "# Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2y_T8TQVlBV"
      },
      "source": [
        "\n",
        "class DQN(nn.Module):\n",
        "    def __init__(\n",
        "        self,\n",
        "        observation_shape,\n",
        "        num_actions,\n",
        "        use_lstm = True,\n",
        "        embedding_dim=32,\n",
        "        crop_dim=9,\n",
        "        num_layers=5\n",
        "    ):\n",
        "        \"\"\"\n",
        "        Initialise the DQN\n",
        "        :param observation_space: the state space of the environment\n",
        "        :param action_space: the action space of the environment\n",
        "        \"\"\"\n",
        "        super(DQN,self).__init__()\n",
        "\n",
        "        self.glyph_shape = observation_shape[\"glyphs\"].shape\n",
        "        self.blstats_size = observation_shape[\"blstats\"].shape[0]\n",
        "\n",
        "        self.num_actions = num_actions\n",
        "        self.use_lstm = use_lstm\n",
        "\n",
        "        self.H = self.glyph_shape[0]\n",
        "        self.W = self.glyph_shape[1]\n",
        "\n",
        "        self.k_dim = embedding_dim\n",
        "        self.h_dim = 512\n",
        "\n",
        "        self.crop_dim = crop_dim\n",
        "\n",
        "        self.crop = Crop(self.H, self.W, self.crop_dim, self.crop_dim)\n",
        "\n",
        "        self.embed = nn.Embedding(nethack.MAX_GLYPH, self.k_dim)\n",
        "\n",
        "        K = embedding_dim  # number of input filters\n",
        "        F = 3  # filter dimensions\n",
        "        S = 1  # stride\n",
        "        P = 1  # padding\n",
        "        M = 16  # number of intermediate filters\n",
        "        Y = 8  # number of output filters\n",
        "        L = num_layers  # number of convnet layers\n",
        "\n",
        "        in_channels = [K] + [M] * (L - 1)\n",
        "        out_channels = [M] * (L - 1) + [Y]\n",
        "\n",
        "        def interleave(xs, ys):\n",
        "            return [val for pair in zip(xs, ys) for val in pair]\n",
        "\n",
        "        conv_extract = [\n",
        "            nn.Conv2d(\n",
        "                in_channels=in_channels[i],\n",
        "                out_channels=out_channels[i],\n",
        "                kernel_size=(F, F),\n",
        "                stride=S,\n",
        "                padding=P,\n",
        "            )\n",
        "            for i in range(L)\n",
        "        ]\n",
        "\n",
        "        self.extract_representation = nn.Sequential(\n",
        "            *interleave(conv_extract, [nn.ELU()] * len(conv_extract))\n",
        "        )\n",
        "\n",
        "        # CNN crop model.\n",
        "        conv_extract_crop = [\n",
        "            nn.Conv2d(\n",
        "                in_channels=in_channels[i],\n",
        "                out_channels=out_channels[i],\n",
        "                kernel_size=(F, F),\n",
        "                stride=S,\n",
        "                padding=P,\n",
        "            )\n",
        "            for i in range(L)\n",
        "        ]\n",
        "\n",
        "        self.extract_crop_representation = nn.Sequential(\n",
        "            *interleave(conv_extract_crop, [nn.ELU()] * len(conv_extract))\n",
        "        )\n",
        "\n",
        "        out_dim = self.k_dim\n",
        "        # CNN over full glyph map\n",
        "        out_dim += self.H * self.W * Y\n",
        "\n",
        "        # CNN crop model.\n",
        "        out_dim += self.crop_dim ** 2 * Y\n",
        "\n",
        "        self.embed_blstats = nn.Sequential(\n",
        "            nn.Linear(self.blstats_size, self.k_dim),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(self.k_dim, self.k_dim),\n",
        "            nn.ReLU(),\n",
        "        )\n",
        "\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(out_dim, self.h_dim),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(self.h_dim, self.h_dim),\n",
        "            nn.ReLU(),\n",
        "        )\n",
        "\n",
        "        if self.use_lstm:\n",
        "            self.core = nn.LSTM(self.h_dim, self.h_dim, num_layers=1)\n",
        "\n",
        "        self.policy = nn.Linear(self.h_dim, self.num_actions)\n",
        "        self.baseline = nn.Linear(self.h_dim, 1)\n",
        "\n",
        "    def _select(self, embed, x):\n",
        "        # Work around slow backward pass of nn.Embedding, see\n",
        "        # https://github.com/pytorch/pytorch/issues/24912\n",
        "        out = embed.weight.index_select(0, x.reshape(-1))\n",
        "        return out.reshape(x.shape + (-1,))\n",
        "\n",
        "    def initial_state(self, batch_size=1):\n",
        "        if not self.use_lstm:\n",
        "            return tuple()\n",
        "        return tuple((\n",
        "            torch.zeros(self.core.num_layers, batch_size, self.core.hidden_size).to(device),\n",
        "            torch.zeros(self.core.num_layers, batch_size, self.core.hidden_size).to(device)\n",
        "        ))\n",
        "\n",
        "    def forward(self, state, core_state, dones=None):\n",
        "        # -- [T x B x H x W] # \n",
        "        glyphs = state[\"glyphs\"]\n",
        "\n",
        "        # -- [T x B x F]\n",
        "        blstats = state[\"blstats\"]\n",
        "\n",
        "        T, B, *_ = glyphs.shape\n",
        "\n",
        "        # -- [B' x H x W]\n",
        "        glyphs = torch.flatten(glyphs, 0, 1)  # Merge time and batch.\n",
        "\n",
        "        # -- [B' x F]\n",
        "        blstats = blstats.view(T * B, -1).float()\n",
        "\n",
        "        # -- [B x H x W]\n",
        "        glyphs = glyphs.long()\n",
        "        # -- [B x 2] x,y coordinates\n",
        "        coordinates = blstats[:, :2]\n",
        "        # TODO ???\n",
        "        # coordinates[:, 0].add_(-1)\n",
        "\n",
        "        # -- [B x F]\n",
        "        # FIXME: hack to use compatible blstats to before\n",
        "        # blstats = blstats[:, [0, 1, 21, 10, 11]]\n",
        "\n",
        "        blstats = blstats.view(T * B, -1).float()\n",
        "        # -- [B x K]\n",
        "        blstats_emb = self.embed_blstats(blstats)\n",
        "\n",
        "        assert blstats_emb.shape[0] == T * B\n",
        "\n",
        "        reps = [blstats_emb] # representations\n",
        "\n",
        "        # -- [B x H' x W']\n",
        "        crop = self.crop(glyphs, coordinates)\n",
        "\n",
        "        # print(\"crop\", crop)\n",
        "        # print(\"at_xy\", glyphs[:, coordinates[:, 1].long(), coordinates[:, 0].long()])\n",
        "\n",
        "        ## self.extract_crop_representation and self.extract_representation forward pass the same\n",
        "        # -- [B x H' x W' x K]\n",
        "        crop_emb = self._select(self.embed, crop)\n",
        "\n",
        "        # CNN crop model.\n",
        "        # -- [B x K x W' x H']\n",
        "        crop_emb = crop_emb.transpose(1, 3)  # -- TODO: slow?\n",
        "        # -- [B x W' x H' x K]\n",
        "        crop_rep = self.extract_crop_representation(crop_emb)\n",
        "\n",
        "        # -- [B x K']\n",
        "        crop_rep = crop_rep.view(T * B, -1)\n",
        "        assert crop_rep.shape[0] == T * B\n",
        "\n",
        "        reps.append(crop_rep)\n",
        "\n",
        "        # -- [B x H x W x K]\n",
        "        glyphs_emb = self._select(self.embed, glyphs)\n",
        "        # glyphs_emb = self.embed(glyphs)\n",
        "        # -- [B x K x W x H]\n",
        "        glyphs_emb = glyphs_emb.transpose(1, 3)  # -- TODO: slow?\n",
        "        # -- [B x W x H x K]\n",
        "        glyphs_rep = self.extract_representation(glyphs_emb)\n",
        "\n",
        "        # -- [B x K']\n",
        "        glyphs_rep = glyphs_rep.view(T * B, -1)\n",
        "        assert glyphs_rep.shape[0] == T * B\n",
        "\n",
        "        # -- [B x K'']\n",
        "        reps.append(glyphs_rep)\n",
        "\n",
        "        st = torch.cat(reps, dim=1)\n",
        "\n",
        "        # -- [B x K]\n",
        "        st = self.fc(st)\n",
        "\n",
        "        if self.use_lstm:\n",
        "            core_input = st.view(T, B, -1)\n",
        "            core_output_list = []\n",
        "            notdone = (dones)\n",
        "            for input, nd in zip(core_input.unbind(), notdone.unbind()):\n",
        "                # Reset core state to zero whenever an episode ended.\n",
        "                # Make `done` broadcastable with (num_layers, B, hidden_size)\n",
        "                # states:\n",
        "                nd = nd.view(1, -1, 1)\n",
        "                core_state = tuple(nd * s for s in core_state)\n",
        "                output, core_state = self.core(input.unsqueeze(0), core_state)\n",
        "                core_output_list.append(output)\n",
        "            core_output = torch.flatten(torch.cat(core_output_list), 0, 1)\n",
        "        else:\n",
        "            core_output = st\n",
        "\n",
        "        # -- [B x A]\n",
        "        policy_logits = self.policy(core_output)\n",
        "        # -- [B x A]\n",
        "        \n",
        "        # print(F.softmax(policy_logits, dim=1))\n",
        "        if self.training:\n",
        "            action = torch.multinomial(F.softmax(policy_logits, dim=1), num_samples=1)\n",
        "        else:\n",
        "            # Don't sample when testing.\n",
        "            action = torch.argmax(policy_logits, dim=1)\n",
        "\n",
        "        policy_logits = policy_logits.view(T, B, self.num_actions)\n",
        "        \n",
        "        action = action.view(T, B)\n",
        "\n",
        "        return (\n",
        "            dict(policy_logits=policy_logits,  action=action),\n",
        "            core_state,\n",
        "        )\n",
        "\"\"\"agent\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rdgnYP_DVSET"
      },
      "source": [
        "# Main"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4YXV8j71JQ-B"
      },
      "source": [
        "def main(model_name = None):\n",
        "    # Seed\n",
        "    seeds = [1,2,3,4,5]\n",
        "\n",
        "    # Initialise environment\n",
        "    env = gym.make(\"NetHackScore-v0\",observation_keys=(\"glyphs\", \"blstats\"))\n",
        "    env.seed(10)\n",
        "    hyper_params = {\n",
        "        \"replay-buffer-size\": 1000,  # replay buffer size\n",
        "        \"learning-rate\": 0.00048,  # learning rate for RMSprob\n",
        "        \"discount-factor\": 0.99,  # discount factor\n",
        "        \"num-steps\": int(1e7),  # total number of steps to run the environment for\n",
        "        \"batch-size\": 256,  # number of transitions to optimize at the same time\n",
        "        \"learning-starts\": 10000,  # number of steps before learning starts\n",
        "        \"learning-freq\": 5,  # number of iterations between every optimization step\n",
        "        \"use-double-dqn\": True,  # use double deep Q-learning\n",
        "        \"target-update-freq\": 1000,  # number of iterations between every target network update\n",
        "        \"eps-start\": 1.0,  # e-greedy start threshold\n",
        "        \"eps-end\": 0.01,  # e-greedy end threshold\n",
        "        \"eps-fraction\": .5,  # fraction of num-steps\n",
        "        \"print-freq\":1,\n",
        "        \"save-freq\":2,\n",
        "        \"save-name\":'dqn_2',\n",
        "        \"model_name\": model_name\n",
        "    }\n",
        "    rewards,mean_rewards,loss,path = train_dqn(env,hyper_params)\n",
        "\n",
        "    plt.figure(figsize=(15,15))\n",
        "    plt.plot(rewards)\n",
        "    plt.ylabel(\"reward\")\n",
        "    plt.xlabel(\"episode\")\n",
        "    plt.savefig('img/nethack_dqn_reward.png')\n",
        "    \n",
        "    plt.figure(figsize=(15,15))\n",
        "    plt.plot(mean_rewards)\n",
        "    plt.ylabel(\"mean 100 episode reward\")\n",
        "    plt.xlabel(\"episode\")\n",
        "    plt.savefig('img/nethack_dqn_mean_reward.png')\n",
        "    \n",
        "    plt.figure(figsize=(15,15))\n",
        "    plt.plot(loss)\n",
        "    plt.ylabel(\"loss\")\n",
        "    plt.savefig('img/nethack_dqn_loss.png')\n",
        "    \n",
        "    #Number of times each seed will be run\n",
        "    num_runs = 10\n",
        "\n",
        "    #Run a few episodes on each seed\n",
        "    rewards = []\n",
        "    for seed in seeds:\n",
        "        env.seed(seed)\n",
        "        seed_rewards = []\n",
        "        for i in range(num_runs):\n",
        "            seed_rewards.append(run_episode(env))\n",
        "        rewards.append(np.mean(seed_rewards))\n",
        "\n",
        "    # Close environment and print average reward\n",
        "    env.close()\n",
        "    print(\"Average Reward: %f\" %(np.mean(rewards)))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8SXqVwQfTAPi"
      },
      "source": [
        "# Train DQN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aiLYPNF6S-dz"
      },
      "source": [
        "def train_dqn(env,hyper_params):\n",
        "    \n",
        "\n",
        "\n",
        "    replay_buffer = ReplayBuffer(hyper_params[\"replay-buffer-size\"])\n",
        "    agent=DQNAgent(env.observation_space,env.action_space.n,replay_buffer,\\\n",
        "                   hyper_params[\"learning-rate\"],hyper_params[\"batch-size\"],hyper_params[\"discount-factor\"])\n",
        "    \n",
        "    eps_timesteps = hyper_params[\"eps-fraction\"] * float(hyper_params[\"num-steps\"])\n",
        "    episode_rewards = [torch.zeros(1, 1)]\n",
        "    num_episode_step_counter = [0]\n",
        "\n",
        "    episode_loss_len = 0\n",
        "    episode_loss = defaultdict(lambda: torch.zeros(1, 1))\n",
        "    episode_loss_av = {}\n",
        "    train_loss = {}\n",
        "    t=0\n",
        "    \n",
        "    if hyper_params['model_name'] is not None:\n",
        "        checkpoints = load_checkpoint(hyper_params['model_name'], agent)\n",
        "        step_num = checkpoints['step_num']\n",
        "        episode_rewards = checkpoints['episode_rewards'] + [torch.zeros(1, 1)]\n",
        "        train_loss = checkpoints['train_loss']\n",
        "        t = checkpoints['step_num']+1\n",
        "        \n",
        "        \n",
        "        ##commentend out as old checkpont function foirgot to save these but took them as imputs\n",
        "        ##uncomment if loading a new dqn \n",
        "        # episode_loss = checkpoints['episode_loss']\n",
        "        # episode_loss_av = checkpoints['episode_loss_av']\n",
        "        # num_episode_step_counter=checkpoints['num_episode_step_counter'] + [0] #  not even sure why we have this. \n",
        "        # if episode_loss ==None:\n",
        "        #     episode_loss = defaultdict(lambda: torch.zeros(1, 1))\n",
        "        # if episode_loss_av==None:\n",
        "        #     pisode_loss_av = {}\n",
        "       \n",
        "\n",
        "    state = env.reset()\n",
        "    state=format_observations(state)\n",
        "    done = torch.ones(1, 1, dtype=torch.uint8)\n",
        "    pbar = tqdm((range(t, hyper_params[\"num-steps\"])))\n",
        "    for t in (range(t,hyper_params[\"num-steps\"])):\n",
        "        fraction = min(1.0, float(t) / eps_timesteps)\n",
        "        eps_threshold=hyper_params[\"eps-start\"]+fraction*(hyper_params[\"eps-end\"]-hyper_params[\"eps-start\"])\n",
        "        sample = random.random()\n",
        "        if sample <= eps_threshold:\n",
        "            action = env.action_space.sample()\n",
        "        else:\n",
        "            action = agent.act(state, done)\n",
        "\n",
        "        #take step in env\n",
        "        state_prime,reward,done,_ = env.step(action)\n",
        "        # clip rewards\n",
        "        # reward = np.tanh(reward/100)\n",
        "        # reward = torch.clamp(reward, -1, 1)\n",
        "        action=torch.tensor(action)\n",
        "        reward=torch.tensor(reward).view(1, 1)\n",
        "        done=torch.tensor(done).view(1, 1)\n",
        "        next_state=format_observations(state_prime)\n",
        "        agent.replay_buffer.add(state,action,reward,next_state, (~done))\n",
        "        state = next_state.copy()\n",
        "        episode_rewards[-1] += reward\n",
        "        num_episode_step_counter[-1] += 1\n",
        "        num_episodes = len(episode_rewards)\n",
        "\n",
        "        if (t>hyper_params[\"learning-starts\"] and t%hyper_params[\"learning-freq\"]==0):\n",
        "            train_loss[t] = (agent.optimise_td_loss())\n",
        "            episode_loss[len(episode_rewards)] += train_loss[t]\n",
        "            episode_loss_len += 1\n",
        "        \n",
        "        if (t>hyper_params[\"learning-starts\"] and len(episode_rewards)%hyper_params[\"print-freq\"]==0 and done):\n",
        "            mean_100ep_reward = round(np.mean(episode_rewards[-25:-1]), 1)\n",
        "            mean_100ep_loss = round(np.mean(list(train_loss.values())[-25:-1]), 1)\n",
        "            # pbar.set_description(\n",
        "            #     \"steps: {}\".format(t) +\n",
        "            #     \" episodes: {}\".format(num_episodes) +\n",
        "            #     \" ep reward: {}\".format(episode_rewards[-1]) +\n",
        "            #     \" last loss: {}\".format((train_loss[-1])) +\n",
        "            #     (f\" mean_reward: {mean_100ep_reward:.4f}\") +\n",
        "            #     (f\" mean_loss: {mean_100ep_loss:.7f}\") +\n",
        "            #     \" explore: {}\".format(int(100 * eps_threshold))\n",
        "            # )\n",
        "            print(\n",
        "                \"steps: {}\".format(t),\n",
        "                \"episodes: {}\".format(num_episodes),\n",
        "                \"ep reward: {}\".format(episode_rewards[-1]),\n",
        "                \"last loss: {}\".format((list(train_loss.values())[-1])),\n",
        "                (f\"mean_reward: {mean_100ep_reward:.4f}\"),\n",
        "                (f\"mean_loss: {mean_100ep_loss:.7f}\"),\n",
        "                \"explore: {}\".format(int(100 * eps_threshold)))\n",
        "            \n",
        "        if (len(episode_rewards) % hyper_params['save-freq'] == 0 and done):\n",
        "            save_checkpoint(\n",
        "                path(hyper_params['save-name']+f'-{len(episode_rewards)}.pt'),\n",
        "                agent, t, episode_rewards, num_episode_step_counter,\n",
        "                train_loss, episode_loss, episode_loss_av)\n",
        "\n",
        "        if done:\n",
        "            state = env.reset()\n",
        "            state = format_observations(state)\n",
        "            if (t > hyper_params[\"learning-starts\"]):\n",
        "                episode_loss_av[len(episode_rewards)] = (episode_loss[len(episode_rewards)]/episode_loss_len)\n",
        "            episode_rewards.append(torch.zeros(1, 1))\n",
        "            num_episode_step_counter.append(0)\n",
        "            episode_loss_len = 0\n",
        "            # print(\"length of ep reward\", len(episode_rewards))\n",
        "\n",
        "        if (t>hyper_params[\"learning-starts\"] and t%hyper_params[\"target-update-freq\"]==0):\n",
        "            agent.update_target_network()\n",
        "\n",
        "    PATH = 'models/dqn_nethack.pth'\n",
        "    torch.save(agent.dqn.state_dict(), PATH)\n",
        "    return episode_rewards.copy(),mean_100ep_reward.copy(),train_loss.copy(),PATH\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BmP8fmzoVXTC"
      },
      "source": [
        "# Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "MK-VBEylJiXx"
      },
      "source": [
        "main('dqn_2-528.pt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ljVc9gkbKB9a"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "p6xW8ENYBMm6"
      },
      "source": [
        "def plot_episode(episode, fname, label):\n",
        "    np.save(path(fname), episode)\n",
        "    plt.figure(figsize=(12,8))\n",
        "    plt.plot(np.arange(len(episode)), episode)\n",
        "    plt.xlabel('Training episode')\n",
        "    plt.ylabel(F'The {label} per episode')\n",
        "    plt.title(F' Nethack Training curves tracking the agent’s {label} per episode')\n",
        "    plt.show()\n",
        "    plt.savefig(path(fname + '.png'))\n",
        "\n",
        "def plot_loss(losses, fname, label, label2):\n",
        "    # redundent function (to an extent)\n",
        "    np.save(path(fname), list(losses.values()))\n",
        "    plt.figure(figsize=(12,8))\n",
        "    plt.plot(list(losses.keys()), list(losses.values()))\n",
        "    plt.xlabel(F'Training {label2}')\n",
        "    plt.ylabel(F'The {label} per {label2}')\n",
        "    plt.title(F'Training curves tracking the {label} per {label2}')\n",
        "    plt.show()\n",
        "    plt.savefig(path('steps-loss.png'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "nT3-kGY2CpLH"
      },
      "source": [
        "checkpoints = load_checkpoint('dqn_1-125.pt', agent)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "_lPblXRzClUx"
      },
      "source": [
        "checkpoints = load_checkpoint('dqn_1-125.pt', agent)\n",
        "step_num = checkpoints['step_num']\n",
        "episode_rewards = checkpoints['episode_rewards']\n",
        "train_loss = checkpoints['train_loss']\n",
        "train_loss = checkpoints['train_loss']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "WIyCHORkCwDM"
      },
      "source": [
        "plot_episode(np.array(episode_rewards[:-1]), 'episode-rewards', 'score')\n",
        "# plot_loss(episode_loss_av, 'episode-loss', 'average loss', 'episode')\n",
        "# plot_loss(episode_loss, 'average-episode-loss', 'loss', 'episode')\n",
        "plot_loss(train_loss, 'steps-loss', 'loss', 'step')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iFV3m9n10xK9"
      },
      "source": [
        "# To Do:\n",
        "\n",
        "\n",
        "* Write the report\n",
        "* Actor Critic\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "TUf4S-iJIy1W"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}