Actor_Critic_2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Actor_Critic_2.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "hL0qgt0K9RO_"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sarah-wookey/RL_NetHack_2020/blob/main/Actor_Critic_2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hL0qgt0K9RO_"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZcEuWgSH7gr7",
        "outputId": "f68712ac-1778-41b8-d952-ccfa933a13b2"
      },
      "source": [
        "# install prerequisites for nle\n",
        "!sudo apt-get install -y build-essential autoconf libtool pkg-config \\\n",
        "    python3-dev python3-pip python3-numpy git libncurses5-dev \\\n",
        "    libzmq3-dev flex bison"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "build-essential is already the newest version (12.4ubuntu1).\n",
            "pkg-config is already the newest version (0.29.1-0ubuntu2).\n",
            "python3-numpy is already the newest version (1:1.13.3-2ubuntu1).\n",
            "python3-numpy set to manually installed.\n",
            "git is already the newest version (1:2.17.1-1ubuntu0.7).\n",
            "libncurses5-dev is already the newest version (6.1-1ubuntu1.18.04).\n",
            "python3-dev is already the newest version (3.6.7-1~18.04).\n",
            "libzmq3-dev is already the newest version (4.2.5-1ubuntu0.2).\n",
            "The following additional packages will be installed:\n",
            "  automake autotools-dev file libbison-dev libfl-dev libfl2 libmagic-mgc\n",
            "  libmagic1 libsigsegv2 m4 python-pip-whl python3-asn1crypto\n",
            "  python3-cffi-backend python3-crypto python3-cryptography python3-idna\n",
            "  python3-keyring python3-keyrings.alt python3-pkg-resources\n",
            "  python3-secretstorage python3-setuptools python3-six python3-wheel\n",
            "  python3-xdg\n",
            "Suggested packages:\n",
            "  autoconf-archive gnu-standards autoconf-doc gettext bison-doc flex-doc\n",
            "  libtool-doc gcj-jdk m4-doc python-crypto-doc python-cryptography-doc\n",
            "  python3-cryptography-vectors gnome-keyring libkf5wallet-bin\n",
            "  gir1.2-gnomekeyring-1.0 python-secretstorage-doc python-setuptools-doc\n",
            "The following NEW packages will be installed:\n",
            "  autoconf automake autotools-dev bison file flex libbison-dev libfl-dev\n",
            "  libfl2 libmagic-mgc libmagic1 libsigsegv2 libtool m4 python-pip-whl\n",
            "  python3-asn1crypto python3-cffi-backend python3-crypto python3-cryptography\n",
            "  python3-idna python3-keyring python3-keyrings.alt python3-pip\n",
            "  python3-pkg-resources python3-secretstorage python3-setuptools python3-six\n",
            "  python3-wheel python3-xdg\n",
            "0 upgraded, 29 newly installed, 0 to remove and 14 not upgraded.\n",
            "Need to get 5,371 kB of archives.\n",
            "After this operation, 22.4 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsigsegv2 amd64 2.12-1 [14.7 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 m4 amd64 1.4.18-1 [197 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 flex amd64 2.6.4-6 [316 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic-mgc amd64 1:5.32-2ubuntu0.4 [184 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic1 amd64 1:5.32-2ubuntu0.4 [68.6 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 file amd64 1:5.32-2ubuntu0.4 [22.1 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 autoconf all 2.69-11 [322 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 autotools-dev all 20180224.1 [39.6 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic/main amd64 automake all 1:1.15.1-3ubuntu2 [509 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 libbison-dev amd64 2:3.0.4.dfsg-1build1 [339 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu bionic/main amd64 bison amd64 2:3.0.4.dfsg-1build1 [266 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 libfl2 amd64 2.6.4-6 [11.4 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic/main amd64 libfl-dev amd64 2.6.4-6 [6,320 B]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic/main amd64 libtool all 2.4.6-2 [194 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python-pip-whl all 9.0.1-2.3~ubuntu1.18.04.4 [1,653 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-asn1crypto all 0.24.0-1 [72.8 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-cffi-backend amd64 1.11.5-1 [64.6 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-crypto amd64 2.6.1-8ubuntu2 [244 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-idna all 2.6-1 [32.5 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-six all 1.11.0-2 [11.4 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 python3-cryptography amd64 2.1.4-1ubuntu1.4 [220 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-secretstorage all 2.3.1-2 [12.1 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-keyring all 10.6.0-1 [26.7 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-keyrings.alt all 3.0-1 [16.6 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 python3-pip all 9.0.1-2.3~ubuntu1.18.04.4 [114 kB]\n",
            "Get:26 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-pkg-resources all 39.0.1-2 [98.8 kB]\n",
            "Get:27 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-setuptools all 39.0.1-2 [248 kB]\n",
            "Get:28 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python3-wheel all 0.30.0-0.2 [36.5 kB]\n",
            "Get:29 http://archive.ubuntu.com/ubuntu bionic/main amd64 python3-xdg all 0.25-4ubuntu1 [31.4 kB]\n",
            "Fetched 5,371 kB in 1s (4,114 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 29.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libsigsegv2:amd64.\n",
            "(Reading database ... 144793 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libsigsegv2_2.12-1_amd64.deb ...\n",
            "Unpacking libsigsegv2:amd64 (2.12-1) ...\n",
            "Selecting previously unselected package m4.\n",
            "Preparing to unpack .../01-m4_1.4.18-1_amd64.deb ...\n",
            "Unpacking m4 (1.4.18-1) ...\n",
            "Selecting previously unselected package flex.\n",
            "Preparing to unpack .../02-flex_2.6.4-6_amd64.deb ...\n",
            "Unpacking flex (2.6.4-6) ...\n",
            "Selecting previously unselected package libmagic-mgc.\n",
            "Preparing to unpack .../03-libmagic-mgc_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmagic1:amd64.\n",
            "Preparing to unpack .../04-libmagic1_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package file.\n",
            "Preparing to unpack .../05-file_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking file (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package autoconf.\n",
            "Preparing to unpack .../06-autoconf_2.69-11_all.deb ...\n",
            "Unpacking autoconf (2.69-11) ...\n",
            "Selecting previously unselected package autotools-dev.\n",
            "Preparing to unpack .../07-autotools-dev_20180224.1_all.deb ...\n",
            "Unpacking autotools-dev (20180224.1) ...\n",
            "Selecting previously unselected package automake.\n",
            "Preparing to unpack .../08-automake_1%3a1.15.1-3ubuntu2_all.deb ...\n",
            "Unpacking automake (1:1.15.1-3ubuntu2) ...\n",
            "Selecting previously unselected package libbison-dev:amd64.\n",
            "Preparing to unpack .../09-libbison-dev_2%3a3.0.4.dfsg-1build1_amd64.deb ...\n",
            "Unpacking libbison-dev:amd64 (2:3.0.4.dfsg-1build1) ...\n",
            "Selecting previously unselected package bison.\n",
            "Preparing to unpack .../10-bison_2%3a3.0.4.dfsg-1build1_amd64.deb ...\n",
            "Unpacking bison (2:3.0.4.dfsg-1build1) ...\n",
            "Selecting previously unselected package libfl2:amd64.\n",
            "Preparing to unpack .../11-libfl2_2.6.4-6_amd64.deb ...\n",
            "Unpacking libfl2:amd64 (2.6.4-6) ...\n",
            "Selecting previously unselected package libfl-dev:amd64.\n",
            "Preparing to unpack .../12-libfl-dev_2.6.4-6_amd64.deb ...\n",
            "Unpacking libfl-dev:amd64 (2.6.4-6) ...\n",
            "Selecting previously unselected package libtool.\n",
            "Preparing to unpack .../13-libtool_2.4.6-2_all.deb ...\n",
            "Unpacking libtool (2.4.6-2) ...\n",
            "Selecting previously unselected package python-pip-whl.\n",
            "Preparing to unpack .../14-python-pip-whl_9.0.1-2.3~ubuntu1.18.04.4_all.deb ...\n",
            "Unpacking python-pip-whl (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Selecting previously unselected package python3-asn1crypto.\n",
            "Preparing to unpack .../15-python3-asn1crypto_0.24.0-1_all.deb ...\n",
            "Unpacking python3-asn1crypto (0.24.0-1) ...\n",
            "Selecting previously unselected package python3-cffi-backend.\n",
            "Preparing to unpack .../16-python3-cffi-backend_1.11.5-1_amd64.deb ...\n",
            "Unpacking python3-cffi-backend (1.11.5-1) ...\n",
            "Selecting previously unselected package python3-crypto.\n",
            "Preparing to unpack .../17-python3-crypto_2.6.1-8ubuntu2_amd64.deb ...\n",
            "Unpacking python3-crypto (2.6.1-8ubuntu2) ...\n",
            "Selecting previously unselected package python3-idna.\n",
            "Preparing to unpack .../18-python3-idna_2.6-1_all.deb ...\n",
            "Unpacking python3-idna (2.6-1) ...\n",
            "Selecting previously unselected package python3-six.\n",
            "Preparing to unpack .../19-python3-six_1.11.0-2_all.deb ...\n",
            "Unpacking python3-six (1.11.0-2) ...\n",
            "Selecting previously unselected package python3-cryptography.\n",
            "Preparing to unpack .../20-python3-cryptography_2.1.4-1ubuntu1.4_amd64.deb ...\n",
            "Unpacking python3-cryptography (2.1.4-1ubuntu1.4) ...\n",
            "Selecting previously unselected package python3-secretstorage.\n",
            "Preparing to unpack .../21-python3-secretstorage_2.3.1-2_all.deb ...\n",
            "Unpacking python3-secretstorage (2.3.1-2) ...\n",
            "Selecting previously unselected package python3-keyring.\n",
            "Preparing to unpack .../22-python3-keyring_10.6.0-1_all.deb ...\n",
            "Unpacking python3-keyring (10.6.0-1) ...\n",
            "Selecting previously unselected package python3-keyrings.alt.\n",
            "Preparing to unpack .../23-python3-keyrings.alt_3.0-1_all.deb ...\n",
            "Unpacking python3-keyrings.alt (3.0-1) ...\n",
            "Selecting previously unselected package python3-pip.\n",
            "Preparing to unpack .../24-python3-pip_9.0.1-2.3~ubuntu1.18.04.4_all.deb ...\n",
            "Unpacking python3-pip (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Selecting previously unselected package python3-pkg-resources.\n",
            "Preparing to unpack .../25-python3-pkg-resources_39.0.1-2_all.deb ...\n",
            "Unpacking python3-pkg-resources (39.0.1-2) ...\n",
            "Selecting previously unselected package python3-setuptools.\n",
            "Preparing to unpack .../26-python3-setuptools_39.0.1-2_all.deb ...\n",
            "Unpacking python3-setuptools (39.0.1-2) ...\n",
            "Selecting previously unselected package python3-wheel.\n",
            "Preparing to unpack .../27-python3-wheel_0.30.0-0.2_all.deb ...\n",
            "Unpacking python3-wheel (0.30.0-0.2) ...\n",
            "Selecting previously unselected package python3-xdg.\n",
            "Preparing to unpack .../28-python3-xdg_0.25-4ubuntu1_all.deb ...\n",
            "Unpacking python3-xdg (0.25-4ubuntu1) ...\n",
            "Setting up python-pip-whl (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Setting up python3-cffi-backend (1.11.5-1) ...\n",
            "Setting up libsigsegv2:amd64 (2.12-1) ...\n",
            "Setting up python3-crypto (2.6.1-8ubuntu2) ...\n",
            "Setting up python3-idna (2.6-1) ...\n",
            "Setting up python3-xdg (0.25-4ubuntu1) ...\n",
            "Setting up python3-six (1.11.0-2) ...\n",
            "Setting up python3-wheel (0.30.0-0.2) ...\n",
            "Setting up m4 (1.4.18-1) ...\n",
            "Setting up python3-pkg-resources (39.0.1-2) ...\n",
            "Setting up libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Setting up python3-asn1crypto (0.24.0-1) ...\n",
            "Setting up autotools-dev (20180224.1) ...\n",
            "Setting up python3-pip (9.0.1-2.3~ubuntu1.18.04.4) ...\n",
            "Setting up libbison-dev:amd64 (2:3.0.4.dfsg-1build1) ...\n",
            "Setting up libfl2:amd64 (2.6.4-6) ...\n",
            "Setting up bison (2:3.0.4.dfsg-1build1) ...\n",
            "update-alternatives: using /usr/bin/bison.yacc to provide /usr/bin/yacc (yacc) in auto mode\n",
            "Setting up python3-setuptools (39.0.1-2) ...\n",
            "Setting up python3-cryptography (2.1.4-1ubuntu1.4) ...\n",
            "Setting up flex (2.6.4-6) ...\n",
            "Setting up python3-keyrings.alt (3.0-1) ...\n",
            "Setting up autoconf (2.69-11) ...\n",
            "Setting up file (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libfl-dev:amd64 (2.6.4-6) ...\n",
            "Setting up automake (1:1.15.1-3ubuntu2) ...\n",
            "update-alternatives: using /usr/bin/automake-1.15 to provide /usr/bin/automake (automake) in auto mode\n",
            "Setting up python3-secretstorage (2.3.1-2) ...\n",
            "Setting up libtool (2.4.6-2) ...\n",
            "Setting up python3-keyring (10.6.0-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.2) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rwou6Htk7raf",
        "outputId": "380e4e18-28da-46ba-9c5e-f98fe8e45ac1"
      },
      "source": [
        "# download, build and install flatbuffers\n",
        "\n",
        "!git clone https://github.com/google/flatbuffers.git\n",
        "# all these commands have to be run in the same directory and !cd doesn't change\n",
        "# the directory permanently in colab see: \n",
        "# https://stackoverflow.com/questions/48298146/changing-directory-in-google-colab-breaking-out-of-the-python-interpreter\n",
        "!cd flatbuffers && cmake -G \"Unix Makefiles\" && make && sudo make install"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'flatbuffers'...\n",
            "remote: Enumerating objects: 36, done.\u001b[K\n",
            "remote: Counting objects: 100% (36/36), done.\u001b[K\n",
            "remote: Compressing objects: 100% (33/33), done.\u001b[K\n",
            "remote: Total 19405 (delta 8), reused 14 (delta 1), pack-reused 19369\u001b[K\n",
            "Receiving objects: 100% (19405/19405), 11.82 MiB | 23.88 MiB/s, done.\n",
            "Resolving deltas: 100% (13474/13474), done.\n",
            "-- The C compiler identification is GNU 7.5.0\n",
            "-- The CXX compiler identification is GNU 7.5.0\n",
            "-- Check for working C compiler: /usr/bin/cc\n",
            "-- Check for working C compiler: /usr/bin/cc -- works\n",
            "-- Detecting C compiler ABI info\n",
            "-- Detecting C compiler ABI info - done\n",
            "-- Detecting C compile features\n",
            "-- Detecting C compile features - done\n",
            "-- Check for working CXX compiler: /usr/bin/c++\n",
            "-- Check for working CXX compiler: /usr/bin/c++ -- works\n",
            "-- Detecting CXX compiler ABI info\n",
            "-- Detecting CXX compiler ABI info - done\n",
            "-- Detecting CXX compile features\n",
            "-- Detecting CXX compile features - done\n",
            "-- Looking for strtof_l\n",
            "-- Looking for strtof_l - found\n",
            "-- Looking for strtoull_l\n",
            "-- Looking for strtoull_l - found\n",
            "-- `tests/monster_test.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/monster_test.fbs`: add generation of binary (.bfbs) schema\n",
            "-- `tests/namespace_test/namespace_test1.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/namespace_test/namespace_test2.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/union_vector/union_vector.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/optional_scalars.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `tests/native_type_test.fbs`: add generation of C++ code with ''\n",
            "-- `tests/arrays_test.fbs`: add generation of C++ code with '--scoped-enums;--gen-compare'\n",
            "-- `tests/arrays_test.fbs`: add generation of binary (.bfbs) schema\n",
            "-- `tests/monster_test.fbs`: add generation of C++ embedded binary schema code with '--no-includes;--gen-compare'\n",
            "-- `tests/monster_extra.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `samples/monster.fbs`: add generation of C++ code with '--no-includes;--gen-compare'\n",
            "-- `samples/monster.fbs`: add generation of binary (.bfbs) schema\n",
            "Proceeding with version: 1.12.0.218\n",
            "-- Configuring done\n",
            "-- Generating done\n",
            "-- Build files have been written to: /content/flatbuffers\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatbuffers\u001b[0m\n",
            "[  1%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[  2%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[  3%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/reflection.cpp.o\u001b[0m\n",
            "[  4%] \u001b[32mBuilding CXX object CMakeFiles/flatbuffers.dir/src/util.cpp.o\u001b[0m\n",
            "[  5%] \u001b[32m\u001b[1mLinking CXX static library libflatbuffers.a\u001b[0m\n",
            "[  5%] Built target flatbuffers\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatc\u001b[0m\n",
            "[  6%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[  7%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[  9%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 10%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/util.cpp.o\u001b[0m\n",
            "[ 11%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_cpp.cpp.o\u001b[0m\n",
            "[ 12%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_csharp.cpp.o\u001b[0m\n",
            "[ 13%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_dart.cpp.o\u001b[0m\n",
            "[ 14%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_kotlin.cpp.o\u001b[0m\n",
            "[ 15%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_go.cpp.o\u001b[0m\n",
            "[ 17%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_java.cpp.o\u001b[0m\n",
            "[ 18%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_js_ts.cpp.o\u001b[0m\n",
            "[ 19%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_php.cpp.o\u001b[0m\n",
            "[ 20%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_python.cpp.o\u001b[0m\n",
            "[ 21%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_lobster.cpp.o\u001b[0m\n",
            "[ 22%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_lua.cpp.o\u001b[0m\n",
            "[ 23%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_rust.cpp.o\u001b[0m\n",
            "[ 25%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_fbs.cpp.o\u001b[0m\n",
            "[ 26%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_grpc.cpp.o\u001b[0m\n",
            "[ 27%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_json_schema.cpp.o\u001b[0m\n",
            "[ 28%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/idl_gen_swift.cpp.o\u001b[0m\n",
            "[ 29%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/flatc.cpp.o\u001b[0m\n",
            "[ 30%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/flatc_main.cpp.o\u001b[0m\n",
            "[ 31%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/src/code_generators.cpp.o\u001b[0m\n",
            "[ 32%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/cpp_generator.cc.o\u001b[0m\n",
            "[ 34%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/go_generator.cc.o\u001b[0m\n",
            "[ 35%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/java_generator.cc.o\u001b[0m\n",
            "[ 36%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/python_generator.cc.o\u001b[0m\n",
            "[ 37%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/swift_generator.cc.o\u001b[0m\n",
            "[ 38%] \u001b[32mBuilding CXX object CMakeFiles/flatc.dir/grpc/src/compiler/ts_generator.cc.o\u001b[0m\n",
            "[ 39%] \u001b[32m\u001b[1mLinking CXX executable flatc\u001b[0m\n",
            "[ 39%] Built target flatc\n",
            "\u001b[35m\u001b[1mScanning dependencies of target generated_code\u001b[0m\n",
            "[ 40%] \u001b[34m\u001b[1mRun generation: 'samples/monster.bfbs'\u001b[0m\n",
            "[ 42%] \u001b[34m\u001b[1mRun generation: 'tests/monster_test_generated.h'\u001b[0m\n",
            "[ 43%] \u001b[34m\u001b[1mRun generation: 'tests/monster_test.bfbs'\u001b[0m\n",
            "[ 44%] \u001b[34m\u001b[1mRun generation: 'tests/namespace_test/namespace_test1_generated.h'\u001b[0m\n",
            "[ 45%] \u001b[34m\u001b[1mRun generation: 'tests/namespace_test/namespace_test2_generated.h'\u001b[0m\n",
            "[ 46%] \u001b[34m\u001b[1mRun generation: 'tests/union_vector/union_vector_generated.h'\u001b[0m\n",
            "[ 47%] \u001b[34m\u001b[1mRun generation: 'tests/optional_scalars_generated.h'\u001b[0m\n",
            "[ 48%] \u001b[34m\u001b[1mRun generation: 'tests/native_type_test_generated.h'\u001b[0m\n",
            "[ 50%] \u001b[34m\u001b[1mRun generation: 'tests/arrays_test_generated.h'\u001b[0m\n",
            "[ 51%] \u001b[34m\u001b[1mRun generation: 'tests/arrays_test.bfbs'\u001b[0m\n",
            "[ 52%] \u001b[34m\u001b[1mRun generation: 'tests/monster_test_bfbs_generated.h'\u001b[0m\n",
            "[ 53%] \u001b[34m\u001b[1mRun generation: 'tests/monster_extra_generated.h'\u001b[0m\n",
            "[ 54%] \u001b[34m\u001b[1mRun generation: 'samples/monster_generated.h'\u001b[0m\n",
            "[ 55%] \u001b[34m\u001b[1mAll generated files were updated.\u001b[0m\n",
            "[ 55%] Built target generated_code\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flattests\u001b[0m\n",
            "[ 56%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[ 57%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[ 59%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 60%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/util.cpp.o\u001b[0m\n",
            "[ 61%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/idl_gen_fbs.cpp.o\u001b[0m\n",
            "[ 62%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/test.cpp.o\u001b[0m\n",
            "[ 63%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/test_assert.cpp.o\u001b[0m\n",
            "[ 64%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/test_builder.cpp.o\u001b[0m\n",
            "[ 65%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/tests/native_type_test_impl.cpp.o\u001b[0m\n",
            "[ 67%] \u001b[32mBuilding CXX object CMakeFiles/flattests.dir/src/code_generators.cpp.o\u001b[0m\n",
            "[ 68%] \u001b[32m\u001b[1mLinking CXX executable flattests\u001b[0m\n",
            "[ 78%] Built target flattests\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatsamplebinary\u001b[0m\n",
            "[ 79%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebinary.dir/samples/sample_binary.cpp.o\u001b[0m\n",
            "[ 80%] \u001b[32m\u001b[1mLinking CXX executable flatsamplebinary\u001b[0m\n",
            "[ 81%] Built target flatsamplebinary\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flathash\u001b[0m\n",
            "[ 82%] \u001b[32mBuilding CXX object CMakeFiles/flathash.dir/src/flathash.cpp.o\u001b[0m\n",
            "[ 84%] \u001b[32m\u001b[1mLinking CXX executable flathash\u001b[0m\n",
            "[ 84%] Built target flathash\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatsampletext\u001b[0m\n",
            "[ 85%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[ 86%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[ 87%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 88%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/src/util.cpp.o\u001b[0m\n",
            "[ 89%] \u001b[32mBuilding CXX object CMakeFiles/flatsampletext.dir/samples/sample_text.cpp.o\u001b[0m\n",
            "[ 90%] \u001b[32m\u001b[1mLinking CXX executable flatsampletext\u001b[0m\n",
            "[ 92%] Built target flatsampletext\n",
            "\u001b[35m\u001b[1mScanning dependencies of target flatsamplebfbs\u001b[0m\n",
            "[ 93%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/idl_parser.cpp.o\u001b[0m\n",
            "[ 94%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/idl_gen_text.cpp.o\u001b[0m\n",
            "[ 95%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/reflection.cpp.o\u001b[0m\n",
            "[ 96%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/src/util.cpp.o\u001b[0m\n",
            "[ 97%] \u001b[32mBuilding CXX object CMakeFiles/flatsamplebfbs.dir/samples/sample_bfbs.cpp.o\u001b[0m\n",
            "[ 98%] \u001b[32m\u001b[1mLinking CXX executable flatsamplebfbs\u001b[0m\n",
            "[100%] Built target flatsamplebfbs\n",
            "[  5%] Built target flatbuffers\n",
            "[ 39%] Built target flatc\n",
            "[ 40%] \u001b[34m\u001b[1mAll generated files were updated.\u001b[0m\n",
            "[ 55%] Built target generated_code\n",
            "[ 78%] Built target flattests\n",
            "[ 81%] Built target flatsamplebinary\n",
            "[ 84%] Built target flathash\n",
            "[ 92%] Built target flatsampletext\n",
            "[100%] Built target flatsamplebfbs\n",
            "\u001b[36mInstall the project...\u001b[0m\n",
            "-- Install configuration: \"\"\n",
            "-- Installing: /usr/local/include/flatbuffers\n",
            "-- Installing: /usr/local/include/flatbuffers/stl_emulation.h\n",
            "-- Installing: /usr/local/include/flatbuffers/util.h\n",
            "-- Installing: /usr/local/include/flatbuffers/minireflect.h\n",
            "-- Installing: /usr/local/include/flatbuffers/reflection.h\n",
            "-- Installing: /usr/local/include/flatbuffers/pch\n",
            "-- Installing: /usr/local/include/flatbuffers/pch/pch.h\n",
            "-- Installing: /usr/local/include/flatbuffers/pch/flatc_pch.h\n",
            "-- Installing: /usr/local/include/flatbuffers/base.h\n",
            "-- Installing: /usr/local/include/flatbuffers/registry.h\n",
            "-- Installing: /usr/local/include/flatbuffers/flatc.h\n",
            "-- Installing: /usr/local/include/flatbuffers/flexbuffers.h\n",
            "-- Installing: /usr/local/include/flatbuffers/flatbuffers.h\n",
            "-- Installing: /usr/local/include/flatbuffers/reflection_generated.h\n",
            "-- Installing: /usr/local/include/flatbuffers/grpc.h\n",
            "-- Installing: /usr/local/include/flatbuffers/code_generators.h\n",
            "-- Installing: /usr/local/include/flatbuffers/hash.h\n",
            "-- Installing: /usr/local/include/flatbuffers/idl.h\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersConfig.cmake\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersConfigVersion.cmake\n",
            "-- Installing: /usr/local/lib/libflatbuffers.a\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersTargets.cmake\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatbuffersTargets-noconfig.cmake\n",
            "-- Installing: /usr/local/bin/flatc\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatcTargets.cmake\n",
            "-- Installing: /usr/local/lib/cmake/flatbuffers/FlatcTargets-noconfig.cmake\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jbORi6RC7uCE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "932f50b5-1aea-46e9-b740-263350387237"
      },
      "source": [
        "# the next step requires a version of cmake > 3.14.0\n",
        "!pip install cmake==3.15.3"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting cmake==3.15.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ff/34/0a311fedffcc7a153bbc0390ef4c378dbc7f09f9865247137f82d62f8e7a/cmake-3.15.3-py3-none-manylinux2010_x86_64.whl (16.5MB)\n",
            "\u001b[K     |████████████████████████████████| 16.5MB 187kB/s \n",
            "\u001b[?25hInstalling collected packages: cmake\n",
            "  Found existing installation: cmake 3.12.0\n",
            "    Uninstalling cmake-3.12.0:\n",
            "      Successfully uninstalled cmake-3.12.0\n",
            "Successfully installed cmake-3.15.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exofv2jL-3U8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b27eeb37-31b9-406c-c9a5-057d0bdebb9c"
      },
      "source": [
        "# add -v for verbose if there are any errors\n",
        "!pip install nle "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting nle\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6e/8d/04662ffda71aa358cc8ee0a04e70d6a769fe8140371ec006d8555a7a1e87/nle-0.6.0.tar.gz (6.7MB)\n",
            "\u001b[K     |████████████████████████████████| 6.7MB 5.6MB/s \n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: gym>=0.15 in /usr/local/lib/python3.6/dist-packages (from nle) (0.17.3)\n",
            "Requirement already satisfied: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from nle) (1.18.5)\n",
            "Collecting pybind11>=2.2\n",
            "  Using cached https://files.pythonhosted.org/packages/00/84/fc9dc13ee536ba5e6b8fd10ce368fea5b738fe394c3b296cde7c9b144a92/pybind11-2.6.1-py2.py3-none-any.whl\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle) (1.5.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle) (1.4.1)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle) (1.3.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym>=0.15->nle) (0.16.0)\n",
            "Building wheels for collected packages: nle\n",
            "  Building wheel for nle (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nle: filename=nle-0.6.0-cp36-cp36m-linux_x86_64.whl size=2847878 sha256=918dd39743ba30a5c1bc08aadf26e0a8774b4b77639a1545171ea70190012126\n",
            "  Stored in directory: /root/.cache/pip/wheels/b5/4d/7c/e4c74b776f945ec1bc9bf01dc94bc226e452cf7dd2aba347a2\n",
            "Successfully built nle\n",
            "Installing collected packages: pybind11, nle\n",
            "Successfully installed nle-0.6.0 pybind11-2.6.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iky0Xmi1-L5k",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9768928c-b06c-4ce2-fc72-c41e5abfdbe5"
      },
      "source": [
        "!pip install \"nle[agent]\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: nle[agent] in /usr/local/lib/python3.6/dist-packages (0.6.0)\n",
            "Requirement already satisfied: numpy>=1.16 in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (1.18.5)\n",
            "Requirement already satisfied: gym>=0.15 in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (0.17.3)\n",
            "Requirement already satisfied: pybind11>=2.2 in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (2.6.1)\n",
            "Requirement already satisfied: torch>=1.3.1; extra == \"agent\" in /usr/local/lib/python3.6/dist-packages (from nle[agent]) (1.7.0+cu101)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle[agent]) (1.4.1)\n",
            "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle[agent]) (1.3.0)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.15->nle[agent]) (1.5.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.3.1; extra == \"agent\"->nle[agent]) (0.16.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch>=1.3.1; extra == \"agent\"->nle[agent]) (3.7.4.3)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch>=1.3.1; extra == \"agent\"->nle[agent]) (0.8)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eee2-Pfr_KUz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e66e6eba-1450-4d90-dd90-125d9a5e5218"
      },
      "source": [
        "!pip install pyvirtualdisplay"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyvirtualdisplay\n",
            "  Downloading https://files.pythonhosted.org/packages/d0/8a/643043cc70791367bee2d19eb20e00ed1a246ac48e5dbe57bbbcc8be40a9/PyVirtualDisplay-1.3.2-py2.py3-none-any.whl\n",
            "Collecting EasyProcess\n",
            "  Downloading https://files.pythonhosted.org/packages/48/3c/75573613641c90c6d094059ac28adb748560d99bd27ee6f80cce398f404e/EasyProcess-0.3-py2.py3-none-any.whl\n",
            "Installing collected packages: EasyProcess, pyvirtualdisplay\n",
            "Successfully installed EasyProcess-0.3 pyvirtualdisplay-1.3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lYooMQjRqRty"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eWeRbH06m5LD"
      },
      "source": [
        "import os\n",
        "import matplotlib as mp\n",
        "# if os.environ.get('DISPLAY','') == '':\n",
        "#     print('no display found. Using non-interactive Agg backend')\n",
        "#     mp.use('Agg')\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import numpy as np\n",
        "import gym\n",
        "import nle\n",
        "import random\n",
        "from gym import spaces\n",
        "from nle import nethack\n",
        "from collections import deque\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import torch\n",
        "from torch.distributions import Categorical\n",
        "import random\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wJjvhC5lm-yT"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "cwd = os.getcwd()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Mv4nxMPbbb_"
      },
      "source": [
        "Colab = False\n",
        "\n",
        "if Colab:\n",
        "    # Imports specifically for google colab/drive\n",
        "    from google.colab import files, drive\n",
        "    drive.mount('/content/gdrive')\n",
        "    dir = '/content/gdrive/My Drive/Colab Notebooks/Nethack-AC/'\n",
        "    os.makedirs(dir, exist_ok=True)\n",
        "else:\n",
        "    dir = './'\n",
        "\n",
        "path = lambda fname: os.path.join(dir, fname)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_mCJY5WbrBiG"
      },
      "source": [
        "# .."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yahWmIPdm5IU"
      },
      "source": [
        "\n",
        "\n",
        "# convert the observation from env to the -> glyphs_matrix,around_agent,agent_stat\n",
        "def format_observations(observation, keys=(\"glyphs\", \"blstats\")):\n",
        "    observations = {}\n",
        "    for key in keys:\n",
        "        entry = observation[key]\n",
        "        entry = torch.from_numpy(entry)\n",
        "        entry = entry.view((1, 1) + entry.shape)  # (...) -> (T,B,...).\n",
        "        observations[key] = entry\n",
        "    return observations\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RF7T3pr_rCnQ"
      },
      "source": [
        "# Agents"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lyP4ZGgrm5Fb"
      },
      "source": [
        "class AbstractAgent:\n",
        "    \"\"\"\n",
        "    AbstractAgent\n",
        "    \"\"\"\n",
        "    def __init__(self):\n",
        "        raise NotImplementedError()\n",
        "\n",
        "    def act(self, observation):\n",
        "        raise NotImplementedError()\n",
        "\n",
        "\n",
        "class MyAgent(AbstractAgent):\n",
        "    def __init__(self, observation_space, action_space):\n",
        "        self.observation_space = observation_space\n",
        "        self.action_space = action_space\n",
        "        # TODO Initialise your agent's models\n",
        "        self.model = PolicyValueNetwork(action_space).to(device)\n",
        "\n",
        "        # for example, if your agent had a Pytorch model it must be load here\n",
        "        PATH = cwd+'/models/Nethackac.pth'\n",
        "        self.model.load_state_dict(torch.load(PATH, map_location=torch.device(device)))\n",
        "\n",
        "    def act(self, observation):\n",
        "        # Perform processing to observation\n",
        "\n",
        "        # TODO: return selected action\n",
        "        state = format_observations(observation)\n",
        " \n",
        "\n",
        "        with torch.no_grad():\n",
        "            dist, value = self.model(state)\n",
        "            action = dist.sample().item()\n",
        "        return action.item()\n",
        "\n",
        "class RandomAgent(AbstractAgent):\n",
        "    def __init__(self, observation_space, action_space):\n",
        "        self.observation_space = observation_space\n",
        "        self.action_space = action_space\n",
        "\n",
        "    def act(self, observation):\n",
        "        return self.action_space.sample()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "emgCAXytrKwY"
      },
      "source": [
        "# Network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PFR1t8LirJ6c"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S_CnJvSjm5DA"
      },
      "source": [
        "def run_episode(env):\n",
        "    # create instance of MyAgent\n",
        "    # from MyAgent import MyAgent\n",
        "    agent = MyAgent(env.observation_space, env.action_space.n)\n",
        "\n",
        "    done = False\n",
        "    episode_return = 0.0\n",
        "    state = env.reset()\n",
        "\n",
        "    while not done:\n",
        "        # pass state to agent and let agent decide action\n",
        "        action = agent.act(state)\n",
        "        new_state, reward, done, _ = env.step(action)\n",
        "        episode_return += reward\n",
        "        state = new_state\n",
        "    return episode_return"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S9Iccligm4_M"
      },
      "source": [
        "class PolicyValueNetwork(nn.Module):\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        observation_shape,\n",
        "        num_actions,\n",
        "        use_lstm = True,\n",
        "        embedding_dim=32,\n",
        "        crop_dim=9,\n",
        "        num_layers=5):\n",
        "\n",
        "\n",
        "        super(PolicyValueNetwork, self).__init__()\n",
        "\n",
        "        self.glyph_shape = observation_shape[\"glyphs\"].shape\n",
        "        self.blstats_size = observation_shape[\"blstats\"].shape[0]\n",
        "\n",
        "        self.num_actions = num_actions\n",
        "        self.use_lstm = use_lstm\n",
        "\n",
        "        self.H = self.glyph_shape[0]\n",
        "        self.W = self.glyph_shape[1]\n",
        "\n",
        "        self.k_dim = embedding_dim\n",
        "        self.h_dim = 512\n",
        "\n",
        "        self.crop_dim = crop_dim\n",
        "\n",
        "        self.crop = Crop(self.H, self.W, self.crop_dim, self.crop_dim)\n",
        "\n",
        "        self.embed = nn.Embedding(nethack.MAX_GLYPH, self.k_dim)\n",
        "\n",
        "        K = embedding_dim  # number of input filters\n",
        "        F = 3  # filter dimensions\n",
        "        S = 1  # stride\n",
        "        P = 1  # padding\n",
        "        M = 16  # number of intermediate filters\n",
        "        Y = 8  # number of output filters\n",
        "        L = num_layers  # number of convnet layers\n",
        "\n",
        "        in_channels = [K] + [M] * (L - 1)\n",
        "        out_channels = [M] * (L - 1) + [Y]\n",
        "\n",
        "        def interleave(xs, ys):\n",
        "            return [val for pair in zip(xs, ys) for val in pair]\n",
        "\n",
        "        conv_extract = [\n",
        "            nn.Conv2d(\n",
        "                in_channels=in_channels[i],\n",
        "                out_channels=out_channels[i],\n",
        "                kernel_size=(F, F),\n",
        "                stride=S,\n",
        "                padding=P,\n",
        "            )\n",
        "            for i in range(L)\n",
        "        ]\n",
        "\n",
        "        self.extract_representation = nn.Sequential(\n",
        "            *interleave(conv_extract, [nn.ELU()] * len(conv_extract))\n",
        "        )\n",
        "\n",
        "        # CNN crop model.\n",
        "        conv_extract_crop = [\n",
        "            nn.Conv2d(\n",
        "                in_channels=in_channels[i],\n",
        "                out_channels=out_channels[i],\n",
        "                kernel_size=(F, F),\n",
        "                stride=S,\n",
        "                padding=P,\n",
        "            )\n",
        "            for i in range(L)\n",
        "        ]\n",
        "\n",
        "        self.extract_crop_representation = nn.Sequential(\n",
        "            *interleave(conv_extract_crop, [nn.ELU()] * len(conv_extract))\n",
        "        )\n",
        "\n",
        "        out_dim = self.k_dim\n",
        "        # CNN over full glyph map\n",
        "        out_dim += self.H * self.W * Y\n",
        "\n",
        "        # CNN crop model.\n",
        "        out_dim += self.crop_dim ** 2 * Y\n",
        "\n",
        "        self.embed_blstats = nn.Sequential(\n",
        "            nn.Linear(self.blstats_size, self.k_dim),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(self.k_dim, self.k_dim),\n",
        "            nn.ReLU(),\n",
        "        )\n",
        "\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(out_dim, self.h_dim),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(self.h_dim, self.h_dim),\n",
        "            nn.ReLU(),\n",
        "        )\n",
        "\n",
        "        if self.use_lstm:\n",
        "            self.core = nn.LSTM(self.h_dim, self.h_dim, num_layers=1)\n",
        "\n",
        "        self.policy = nn.Linear(self.h_dim, self.num_actions)\n",
        "        self.state_value = nn.Linear(self.h_dim, 1)\n",
        "\n",
        "\n",
        "    def _select(self, embed, x):\n",
        "        # Work around slow backward pass of nn.Embedding, see\n",
        "        # https://github.com/pytorch/pytorch/issues/24912\n",
        "        out = embed.weight.index_select(0, x.reshape(-1))\n",
        "        return out.reshape(x.shape + (-1,))\n",
        "\n",
        "    def initial_state(self, batch_size=1):\n",
        "        if not self.use_lstm:\n",
        "            return tuple()\n",
        "        return tuple((\n",
        "            torch.zeros(self.core.num_layers, batch_size, self.core.hidden_size),\n",
        "            torch.zeros(self.core.num_layers, batch_size, self.core.hidden_size)\n",
        "        ))\n",
        "\n",
        "    \n",
        "    def forward(self, state):\n",
        "\n",
        "        glyphs = state[\"glyphs\"]\n",
        "        # -- [T x B x F]\n",
        "        blstats = state[\"blstats\"]\n",
        "        T, B, *_ = glyphs.shape\n",
        "        # -- [B' x H x W]\n",
        "        glyphs = torch.flatten(glyphs, 0, 1)  # Merge time and batch.\n",
        "        # -- [B' x F]\n",
        "        blstats = blstats.view(T * B, -1).float()\n",
        "        # -- [B x H x W]\n",
        "        glyphs = glyphs.long()\n",
        "        # -- [B x 2] x,y coordinates\n",
        "        coordinates = blstats[:, :2]\n",
        "        # TODO ???\n",
        "        # coordinates[:, 0].add_(-1)\n",
        "        # -- [B x F]\n",
        "        # FIXME: hack to use compatible blstats to before\n",
        "        # blstats = blstats[:, [0, 1, 21, 10, 11]]\n",
        "        blstats = blstats.view(T * B, -1).float()\n",
        "        # -- [B x K]\n",
        "        blstats_emb = self.embed_blstats(blstats)\n",
        "        assert blstats_emb.shape[0] == T * B\n",
        "        reps = [blstats_emb] # representations\n",
        "        # -- [B x H' x W']\n",
        "        crop = self.crop(glyphs, coordinates)\n",
        "        # print(\"crop\", crop)\n",
        "        # print(\"at_xy\", glyphs[:, coordinates[:, 1].long(), coordinates[:, 0].long()])\n",
        "        ## self.extract_crop_representation and self.extract_representation forward pass the same\n",
        "        # -- [B x H' x W' x K]\n",
        "        crop_emb = self._select(self.embed, crop)\n",
        "        # CNN crop model.\n",
        "        # -- [B x K x W' x H']\n",
        "        crop_emb = crop_emb.transpose(1, 3)  # -- TODO: slow?\n",
        "        # -- [B x W' x H' x K]\n",
        "        crop_rep = self.extract_crop_representation(crop_emb)\n",
        "        # -- [B x K']\n",
        "        crop_rep = crop_rep.view(T * B, -1)\n",
        "        assert crop_rep.shape[0] == T * B\n",
        "        reps.append(crop_rep)\n",
        "        # -- [B x H x W x K]\n",
        "        glyphs_emb = self._select(self.embed, glyphs)\n",
        "        # glyphs_emb = self.embed(glyphs)\n",
        "        # -- [B x K x W x H]\n",
        "        glyphs_emb = glyphs_emb.transpose(1, 3)  # -- TODO: slow?\n",
        "        # -- [B x W x H x K]\n",
        "        glyphs_rep = self.extract_representation(glyphs_emb)\n",
        "        # -- [B x K']\n",
        "        glyphs_rep = glyphs_rep.view(T * B, -1)\n",
        "        assert glyphs_rep.shape[0] == T * B\n",
        "        # -- [B x K'']\n",
        "        reps.append(glyphs_rep)\n",
        "        st = torch.cat(reps, dim=1)\n",
        "        # -- [B x K]\n",
        "        st = self.fc(st) \n",
        "        # -- [B x A]\n",
        "        policy_logits = self.policy(st)\n",
        "        # -- [B x A]\n",
        "        # print(F.softmax(policy_logits, dim=1))\n",
        "        # if self.training:\n",
        "        #     action = torch.multinomial(F.softmax(policy_logits, dim=1), num_samples=1)\n",
        "        # else:\n",
        "        #     # Don't sample when testing.\n",
        "        #     action = torch.argmax(policy_logits, dim=1)\n",
        "\n",
        "\n",
        "        state_value = self.state_value(st)\n",
        "        prob = F.softmax(policy_logits,dim=-1)\n",
        "        dist = Categorical(prob)\n",
        "        return dist, state_value   "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eXC7ARAom44w"
      },
      "source": [
        "def compute_returns(next_value, rewards, masks, gamma=0.99):\n",
        "    R = next_value\n",
        "    returns = []\n",
        "    for step in reversed(range(len(rewards))):\n",
        "        R = rewards[step] + gamma * R * masks[step]\n",
        "        returns.insert(0, R)\n",
        "    return returns\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80xp44kPNoL7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "78F0UtjRJzXf"
      },
      "source": [
        "# Crop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eMHCmWZ0aG-y"
      },
      "source": [
        "\n",
        "class Crop(nn.Module):\n",
        "    \"\"\"Helper class for NetHackNet below.\"\"\"\n",
        "\n",
        "    def __init__(self, height, width, height_target, width_target):\n",
        "        super(Crop, self).__init__()\n",
        "        self.width = width\n",
        "        self.height = height\n",
        "        self.width_target = width_target\n",
        "        self.height_target = height_target\n",
        "        width_grid = _step_to_range(2 / (self.width - 1), self.width_target)[\n",
        "            None, :\n",
        "        ].expand(self.height_target, -1)\n",
        "        height_grid = _step_to_range(2 / (self.height - 1), height_target)[\n",
        "            :, None\n",
        "        ].expand(-1, self.width_target)\n",
        "\n",
        "        # \"clone\" necessary, https://github.com/pytorch/pytorch/issues/34880\n",
        "        self.register_buffer(\"width_grid\", width_grid.clone())\n",
        "        self.register_buffer(\"height_grid\", height_grid.clone())\n",
        "\n",
        "    def forward(self, inputs, coordinates):\n",
        "        \"\"\"Calculates centered crop around given x,y coordinates.\n",
        "        Args:\n",
        "           inputs [B x H x W]\n",
        "           coordinates [B x 2] x,y coordinates\n",
        "        Returns:\n",
        "           [B x H' x W'] inputs cropped and centered around x,y coordinates.\n",
        "        \"\"\"\n",
        "        assert inputs.shape[1] == self.height, f'{ inputs.shape[1],self.height}'\n",
        "        assert inputs.shape[2] == self.width\n",
        "\n",
        "        inputs = inputs[:, None, :, :].float()\n",
        "\n",
        "        x = coordinates[:, 0]\n",
        "        y = coordinates[:, 1]\n",
        "\n",
        "        x_shift = 2 / (self.width - 1) * (x.float() - self.width // 2)\n",
        "        y_shift = 2 / (self.height - 1) * (y.float() - self.height // 2)\n",
        "\n",
        "        grid = torch.stack(\n",
        "            [\n",
        "                self.width_grid[None, :, :] + x_shift[:, None, None],\n",
        "                self.height_grid[None, :, :] + y_shift[:, None, None],\n",
        "            ],\n",
        "            dim=3,\n",
        "        )\n",
        "\n",
        "        # TODO: only cast to int if original tensor was int\n",
        "        return (\n",
        "            torch.round(F.grid_sample(inputs, grid, align_corners=True))\n",
        "            .squeeze(1)\n",
        "            .long()\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kJveXHsOrR4W"
      },
      "source": [
        "# Run"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9_RAm25qa-vX"
      },
      "source": [
        "def save_checkpoint(\n",
        "    fname,\n",
        "    policy_model,\n",
        "    optimizer,\n",
        "    current_step_number,\n",
        "    scores,\n",
        "    train_loss,\n",
        "    download=False\n",
        "):\n",
        "\n",
        "    checkpoint = {\n",
        "        'current_step_number': current_step_number,\n",
        "        'scores': scores,\n",
        "        'train_loss': train_loss,\n",
        "        'model_state_dict': policy_model.state_dict(),\n",
        "        'optimizer_state_dict': optimizer.state_dict()}\n",
        "\n",
        "    torch.save(checkpoint, path(fname))\n",
        "    if download and Colab: files.download(path) # not working well "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gIyhtMjvHHAj"
      },
      "source": [
        "def dict_to_device(dic):\n",
        "    for key, value in dic.items():\n",
        "        dic[key] = dic[key].to(device)\n",
        "    return dic"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "No79ZATKm41p"
      },
      "source": [
        "\n",
        "def train_ac(env, policy_model,flags, seed, learning_rate,\n",
        "                                             number_episodes,\n",
        "                                             gamma, verbose=True):\n",
        "    # set random seeds (for reproducibility)\n",
        "    max_steps=1e4\n",
        "    num_step_td_update = 5\n",
        "    torch.cuda.manual_seed_all(flags['seed'])\n",
        "    np.random.seed(flags['seed'])\n",
        "    env.seed(flags['seed'])\n",
        "    random.seed(flags['seed'])\n",
        "    optimizer = optim.RMSprop(policy_model.parameters(),lr=learning_rate,eps=0.000001)\n",
        "    scores = [0.0]\n",
        "    scores_mean = []\n",
        "    current_step_number = 0\n",
        "    train_loss = [0]\n",
        "    state = env.reset()\n",
        "    \n",
        "    while current_step_number < max_steps:\n",
        "        state_value = []\n",
        "        rewards = []\n",
        "        log_prob_actions = []\n",
        "        masks = []\n",
        "\n",
        "        for _ in range(num_step_td_update):\n",
        "            \n",
        "            \n",
        "            state_input=format_observations(state)\n",
        "            state_input=dict_to_device(state_input)\n",
        "            dist,value = policy_model(state_input)\n",
        "            action = dist.sample()\n",
        "\n",
        "            log_p = dist.log_prob(action)\n",
        "            state_prime,reward,done,_ = env.step(action)\n",
        "            \n",
        "            scores[-1]+=reward\n",
        "\n",
        "            # clip rewards\n",
        "            \n",
        "\n",
        "            log_prob_actions.append(log_p)\n",
        "            state_value.append(value)\n",
        "            masks.append(1 - done)\n",
        "            rewards.append(reward)\n",
        "            \n",
        "            current_step_number += 1\n",
        "            state = state_prime\n",
        "\n",
        "            if (len(scores) % 20 == 0 and done):\n",
        "                save_checkpoint(\n",
        "                    path('AC_1'+f'-{len(scores)}.pt'),\n",
        "                    policy_model, optimizer, current_step_number,\n",
        "                    scores, train_loss)\n",
        "\n",
        "\n",
        "            if done:\n",
        "                mean_100ep_reward = round(np.mean(scores[-25:-1]), 1)\n",
        "                mean_100ep_loss = round(np.mean((train_loss)[-25:-1]), 1)\n",
        "                print(\n",
        "                    \"steps: {}\".format(current_step_number),\n",
        "                    \"episodes: {}\".format(len(scores)),\n",
        "                    \"ep reward: {}\".format(scores[-1]),\n",
        "                    \"last loss: {}\".format((train_loss[-1])),\n",
        "                    (f\"mean_reward: {mean_100ep_reward:.4f}\"),\n",
        "                    (f\"mean_loss: {mean_100ep_loss:.7f}\")\n",
        "                )\n",
        "                            \n",
        "                if len(scores)%100==0:\n",
        "                    scores_mean.append(round(np.mean(scores[-100:-1]), 1))\n",
        "                state = env.reset()\n",
        "                scores.append(0.0)\n",
        "\n",
        "        state_input=format_observations(state)\n",
        "        state_input=dict_to_device(state_input)\n",
        "        dist,value = policy_model(state_input)\n",
        "        action = dist.sample()\n",
        "\n",
        "        _,next_value = policy_model(state_input)\n",
        "        \n",
        "        returns = compute_returns(next_value, rewards, masks,gamma)\n",
        "        \n",
        "        state_value = torch.cat(state_value)\n",
        "        log_prob_actions = torch.cat(log_prob_actions)\n",
        "        returns   = torch.cat(returns).detach()\n",
        "\n",
        "        change = returns - state_value\n",
        "        p_loss = -torch.mean(log_prob_actions*change.detach())\n",
        "        v_loss = 0.5*torch.mean(torch.pow(change,2))\n",
        "        loss = p_loss + v_loss\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "\n",
        "        torch.nn.utils.clip_grad_norm_(policy_model.parameters(), 40)\n",
        "        optimizer.step()\n",
        "\n",
        "        train_loss.append(loss.item())\n",
        "\n",
        "    PATH = cwd+'/models/Nethackac.pth'\n",
        "    torch.save(policy_model.state_dict(), PATH) \n",
        "    \n",
        "    return policy_model,PATH, scores.copy(),scores_mean.copy()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KCnQnDTzHHAj"
      },
      "source": [
        "def _step_to_range(delta, num_steps):\n",
        "    \"\"\"Range of `num_steps` integers with distance `delta` centered around zero.\"\"\"\n",
        "    return delta * torch.arange(-num_steps // 2, num_steps // 2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CiGi0VhsHHAj"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "II__u4fSrUt3"
      },
      "source": [
        "# Main"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SCIHRCo-m1fC"
      },
      "source": [
        "def main():\n",
        "    # Seed\n",
        "    seeds = [1,2,3,4,5]\n",
        "    \n",
        "    # Initialise environment\n",
        "    env = gym.make(\"NetHackScore-v0\",observation_keys=(\"glyphs\", \"blstats\"))\n",
        "    # hyper-parameters\n",
        "    gamma = 0.99\n",
        "    learning_rate = 0.02\n",
        "    # seed = 214\n",
        "    seed = np.random.choice(seeds)\n",
        "    torch.manual_seed(seed)\n",
        "    env.seed(seed)\n",
        "    number_episodes = 1000\n",
        "    hyper_params = {\n",
        "        \"learning_rate\": 0.00048,  # learning rate for RMSprob\n",
        "        \"discount-factor\": 0.99,  # discount factor\n",
        "        \"num-steps\": int(1e6),  # total number of steps to run the environment for\n",
        "        \"batch-size\": 256,  # number of transitions to optimize at the same time\n",
        "        \"learning-starts\": 10000,  # number of steps before learning starts\n",
        "        \"learning-freq\": 5,  # number of iterations between every optimization step\n",
        "        \"use-double-dqn\": True,  # use double deep Q-learning\n",
        "        \"target-update-freq\": 1000,  # number of iterations between every target network update\n",
        "        \"eps-start\": 1.0,  # e-greedy start threshold\n",
        "        \"eps-end\": 0.01,  # e-greedy end threshold\n",
        "        \"eps-fraction\": .5,  # fraction of num-steps\n",
        "        \"print-freq\":1,\n",
        "        \"save-freq\":25,\n",
        "        \"save-name\":'dqn_1',\n",
        "        \"seed\":seed\n",
        "    }\n",
        "\n",
        "    policy_model = PolicyValueNetwork(env.observation_space,env.action_space.n,False).to(device)\n",
        "    net,path, scores,scores_mean = train_ac(env, policy_model,hyper_params, seed, learning_rate,\n",
        "                                             number_episodes,\n",
        "                                             gamma, verbose=True)\n",
        "    \n",
        "    #Number of times each seed will be run\n",
        "    num_runs = 10\n",
        "\n",
        "    #Run a few episodes on each seed\n",
        "    rewards = []\n",
        "    for seed in seeds:\n",
        "        env.seed(seed)\n",
        "        seed_rewards = []\n",
        "        for i in range(num_runs):\n",
        "            seed_rewards.append(run_episode(env))\n",
        "        rewards.append(np.mean(seed_rewards))\n",
        "\n",
        "    # Close environment and print average reward\n",
        "    env.close()\n",
        "#     print(\"Average Reward: %f\" %(np.mean(rewards)))\n",
        "    \n",
        "    \n",
        "    plt.figure(figsize=(15,15))\n",
        "    plt.plot(rewards)\n",
        "    plt.ylabel(\"reward\")\n",
        "    plt.xlabel(\"episode\")\n",
        "    plt.title(\"episode rewards testing\")\n",
        "    plt.savefig(cwd+'/img/actor_critic_testing_reward_nethackac.png')\n",
        "    \n",
        "    \n",
        "    plt.figure(figsize=(15,15))\n",
        "    plt.plot(scores)\n",
        "    plt.ylabel(\"score\")\n",
        "    plt.xlabel(\"episode\")\n",
        "    plt.savefig(cwd+'/img/actor_critic_traing_reward_nethackac.png')\n",
        "    \n",
        "    plt.figure(figsize=(15,15))\n",
        "    plt.plot(scores_mean)\n",
        "    plt.ylabel(\"score(average over 100 eps)\")\n",
        "    plt.xlabel(\"episode\")\n",
        "    plt.savefig(cwd+'/img/actor_critic_traing_reward_nethackac.png')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X4BgXao4rW0A"
      },
      "source": [
        "# ..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "id": "Tar_EULvnY-O",
        "outputId": "e081c907-d39c-4b78-cdd0-02c7d73f69b1"
      },
      "source": [
        "if __name__ == '__main__':\n",
        "    main()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
            "  out=out, **kwargs)\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  ret = ret.dtype.type(ret / rcount)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "steps: 5000 episodes: 1 ep reward: -49.93999999999863 last loss: 4.3204535357271966e+19 mean_reward: nan mean_loss: 43204535357271965696.0000000\n",
            "steps: 10000 episodes: 2 ep reward: -49.98999999999862 last loss: 2.835441256527351e+19 mean_reward: -49.9000 mean_loss: 28354412565273509888.0000000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-67-c7bc734e5e35>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-62-2b12def1ca21>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     34\u001b[0m     net,path, scores,scores_mean = train_ac(env, policy_model,hyper_params, seed, learning_rate,\n\u001b[1;32m     35\u001b[0m                                              \u001b[0mnumber_episodes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m                                              gamma, verbose=True)\n\u001b[0m\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0;31m#Number of times each seed will be run\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-66-5a32be324df1>\u001b[0m in \u001b[0;36mtrain_ac\u001b[0;34m(env, policy_model, flags, seed, learning_rate, number_episodes, gamma, verbose)\u001b[0m\n\u001b[1;32m     98\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0mPATH\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcwd\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'/models/Nethackac.pth'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m     \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpolicy_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPATH\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mpolicy_model\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mPATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mscores_mean\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(obj, f, pickle_module, pickle_protocol, _use_new_zipfile_serialization)\u001b[0m\n\u001b[1;32m    367\u001b[0m     \u001b[0m_check_dill_version\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpickle_module\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 369\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    370\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0m_use_new_zipfile_serialization\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0m_open_zipfile_writer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mopened_zipfile\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_open_file_like\u001b[0;34m(name_or_buffer, mode)\u001b[0m\n\u001b[1;32m    228\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    229\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_is_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    231\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    232\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m'w'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, mode)\u001b[0m\n\u001b[1;32m    209\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 211\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_open_file\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    212\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/content/models/Nethackac.pth'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "78ZZo0sQLbsK"
      },
      "source": [
        "# .,"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xhBYhiqiMt-2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sYfDR1pNLa1Q"
      },
      "source": [
        "\n",
        "\n",
        "plt.figure(figsize=(15,15))\n",
        "plt.plot(rewards)\n",
        "plt.ylabel(\"reward\")\n",
        "plt.xlabel(\"episode\")\n",
        "plt.title(\"episode rewards testing\")\n",
        "plt.savefig(cwd+'/img/actor_critic_testing_reward_nethackac.png')\n",
        "\n",
        "\n",
        "plt.figure(figsize=(15,15))\n",
        "plt.plot(scores)\n",
        "plt.ylabel(\"score\")\n",
        "plt.xlabel(\"episode\")\n",
        "plt.savefig(cwd+'/img/actor_critic_traing_reward_nethackac.png')\n",
        "\n",
        "plt.figure(figsize=(15,15))\n",
        "plt.plot(scores_mean)\n",
        "plt.ylabel(\"score(average over 100 eps)\")\n",
        "plt.xlabel(\"episode\")\n",
        "plt.savefig(cwd+'/img/actor_critic_traing_reward_nethackac.png')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}