TP_Transformer_Demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TP_Transformer_Demo.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "1d0lNjgu8mzB",
        "colab_type": "code",
        "outputId": "6af57d83-6f9e-49ba-f904-eb72669ec49c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 649
        }
      },
      "source": [
        "!pip install gdown\n",
        "!pip3 install --upgrade torch==1.1.0\n",
        "!pip3 install --upgrade torchtext==0.3.1\n",
        "!pip3 install --upgrade tensorboardX==1.8"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: gdown in /usr/local/lib/python3.6/dist-packages (3.6.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from gdown) (4.28.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from gdown) (2.21.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from gdown) (1.12.0)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->gdown) (2019.9.11)\n",
            "Collecting torch==1.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/60/f685fb2cfb3088736bafbc9bdbb455327bdc8906b606da9c9a81bae1c81e/torch-1.1.0-cp36-cp36m-manylinux1_x86_64.whl (676.9MB)\n",
            "\u001b[K     |████████████████████████████████| 676.9MB 26kB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from torch==1.1.0) (1.17.3)\n",
            "\u001b[31mERROR: torchvision 0.4.1+cu100 has requirement torch==1.3.0, but you'll have torch 1.1.0 which is incompatible.\u001b[0m\n",
            "Installing collected packages: torch\n",
            "  Found existing installation: torch 1.3.0+cu100\n",
            "    Uninstalling torch-1.3.0+cu100:\n",
            "      Successfully uninstalled torch-1.3.0+cu100\n",
            "Successfully installed torch-1.1.0\n",
            "Requirement already up-to-date: torchtext==0.3.1 in /usr/local/lib/python3.6/dist-packages (0.3.1)\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from torchtext==0.3.1) (4.28.1)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from torchtext==0.3.1) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from torchtext==0.3.1) (1.17.3)\n",
            "Requirement already satisfied, skipping upgrade: torch in /usr/local/lib/python3.6/dist-packages (from torchtext==0.3.1) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext==0.3.1) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext==0.3.1) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext==0.3.1) (2019.9.11)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext==0.3.1) (2.8)\n",
            "Collecting tensorboardX==1.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c3/12/dcaf67e1312475b26db9e45e7bb6f32b540671a9ee120b3a72d9e09bc517/tensorboardX-1.8-py2.py3-none-any.whl (216kB)\n",
            "\u001b[K     |████████████████████████████████| 225kB 4.8MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from tensorboardX==1.8) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from tensorboardX==1.8) (1.17.3)\n",
            "Requirement already satisfied, skipping upgrade: protobuf>=3.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorboardX==1.8) (3.10.0)\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.2.0->tensorboardX==1.8) (41.4.0)\n",
            "Installing collected packages: tensorboardX\n",
            "Successfully installed tensorboardX-1.8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e8XCNkxP8pKA",
        "colab_type": "code",
        "outputId": "00f7792c-5947-4412-980a-7c929597249d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "!gdown -O \"demo.tar.gz\" https://drive.google.com/uc?id=14tVaGSAx_Ax6_HyYaSmQDSV1eJWT-0-e"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=14tVaGSAx_Ax6_HyYaSmQDSV1eJWT-0-e\n",
            "To: /content/demo.tar.gz\n",
            "589MB [00:07, 76.1MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "24nZ5VJ-8qSJ",
        "colab_type": "code",
        "outputId": "a0649efd-6ded-469f-b76c-68348b7e139c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 578
        }
      },
      "source": [
        "!tar xvf demo.tar.gz"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "demo/utils/data_loader.py\n",
            "demo/models/tp-transformer.py\n",
            "demo/data/dm_math/all_modules/train.indexes_pt\n",
            "demo/data/dm_math/all_modules/\n",
            "demo/data/dm_math/all_modules/train.vocab\n",
            "demo/data/dm_math/all_modules/extrapolate.xy\n",
            "demo/utils/jit_data_loader.py\n",
            "demo/merge_all_dm_math.py\n",
            "demo/utils/jit_dataset.py\n",
            "demo/models/\n",
            "demo/data/dm_math/all_modules/interpolate.indexes_pt\n",
            "demo/eval.sh\n",
            "demo/models/transformer.py\n",
            "demo/data/dm_math/all_modules/interpolate.xy\n",
            "demo/utils/\n",
            "demo/data/dm_math/all_modules/extrapolate.vocab\n",
            "demo/data/dm_math/vocab.pt\n",
            "demo/utils/trainer.py\n",
            "demo/pretrained/tp-transformer_1.7M.pt\n",
            "demo/train.sh\n",
            "demo/main.py\n",
            "demo/data/dm_math/all_modules/interpolate.vocab\n",
            "demo/data/dm_math/all_modules/FOR DEMO PURPOSES ONLY - ALL 3 SETS ARE INTERPOLATE\n",
            "demo/utils/lib.py\n",
            "demo/\n",
            "demo/data/dm_math/all_modules/extrapolate.indexes_pt\n",
            "demo/data/dm_math/\n",
            "demo/pretrained/\n",
            "demo/data/dm_math/all_modules/train.xy\n",
            "demo/eval_baseline.sh\n",
            "demo/.gitignore\n",
            "demo/data/\n",
            "demo/process_dm_math.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xqp4Pf-48qcI",
        "colab_type": "code",
        "outputId": "44543968-eb09-48cf-a004-ad5860695294",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "cd demo"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/demo\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "65c_wFgx8sDy",
        "colab_type": "code",
        "outputId": "7bed6905-91ef-45c4-d199-397581c9ba69",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 765
        }
      },
      "source": [
        "%run main.py \\\n",
        "--model_name=tp-transformer \\\n",
        "--module_name=all_modules \\\n",
        "--n_layers=6 \\\n",
        "--hidden=512 \\\n",
        "--filter=2048 \\\n",
        "--n_heads=8 \\\n",
        "--load_model=\"pretrained/tp-transformer_1.7M.pt\" \\\n",
        "--batch_size=1024 \\\n",
        "--eval_mode"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Arguments:\n",
            "\tseed=195932581\n",
            "\tmodel_name='tp-transformer'\n",
            "\tmodule_name='all_modules'\n",
            "\tload_model='pretrained/tp-transformer_1.7M.pt'\n",
            "\teval_mode=True\n",
            "\tn_steps=10000\n",
            "\tmax_strikes=1000\n",
            "\tlog_every=50\n",
            "\teval_every=200\n",
            "\tfull_loader=False\n",
            "\tforce_remove=False\n",
            "\tforce_reload=False\n",
            "\tno_train=False\n",
            "\tlog_folder='log/all_modules/tp-transformer/lr=0.0001_bs=1024_h=512_f=2048_nl=6_nh=8_d=0.0_Adam_/195932581'\n",
            "\tlog_suffix=''\n",
            "\toptimizer='Adam'\n",
            "\tlearning_rate=0.0001\n",
            "\tbeta1=0.9\n",
            "\tbeta2=0.995\n",
            "\tbatch_size=1024\n",
            "\tmax_abs_grad_norm=0.1\n",
            "\tgrad_accum_steps=1\n",
            "\tdropout=0.0\n",
            "\thidden=512\n",
            "\tn_layers=6\n",
            "\tn_heads=8\n",
            "\tfilter=2048\n",
            "\td_r=0\n",
            "\tdevice=device(type='cuda')\n",
            "using JIT loader\n",
            "Loading JIT datasets ...\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " Done.\n",
            "  loading index examples from: data/dm_math/all_modules/train.indexes_pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Creating iterators ...\n",
            "Loading JIT datasets ...\n",
            "Creating iterators ...\n",
            "Building model ...\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  built 560,000 examples (0.04 secs)\n",
            "\n",
            "  loading index examples from: data/dm_math/all_modules/interpolate.indexes_pt\n",
            "  built 560,000 examples (0.03 secs)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "done. 49178624 trainable parameters.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HMSTQwHJ8sah",
        "colab_type": "code",
        "outputId": "c69776dc-65ff-4c13-cc00-88a90891092e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "print_random_sample()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "input: ['<sos>Calculate the common denominator of 11/260 and (-349)/12564*(56/26)/(-2).<eos><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>']\n",
            "target:['<sos>2340<eos><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad><pad>']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "039iqqfv8tb5",
        "colab_type": "code",
        "outputId": "968f5b36-f122-4ebf-ef8a-4e45645f0ce5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# copy and paste the input between <sos> and <eos> into the string below to see the TP-Transformer's prediction.\n",
        "terminal.enter(\"Calculate the common denominator of 11/260 and (-349)/12564*(56/26)/(-2).\")"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['<sos>2340<eos>']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "myCG_95fBskl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}