LLM_evaluation_harness_for_Arc_Easy_and_SST.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bD9ALckB-IGp"
      },
      "source": [
        "## Language model evaluation harness starter resource\n",
        "\n",
        "<a target=\"_blank\" href=\"https://colab.research.google.com/github/rajshah4/LLM-Evaluation/blob/main/LLM_evaluation_harness_for_Arc_Easy_and_SST.ipynb\">\n",
        "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/>\n",
        "</a>\n",
        "\n",
        "This showcases how to evaluate your models on the [LM Eval harness](https://github.com/EleutherAI/lm-evaluation-harness) from EleutherAI\n",
        "\n",
        "Source: https://colab.research.google.com/drive/1zmZfdETnQ-AR2BBIK3pFtnP5937J1yaz?usp=sharing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zav0yCBF990_",
        "outputId": "75d6b1f9-0360-404d-e4f8-2aeaf0dfac33"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'lm-evaluation-harness'...\n",
            "remote: Enumerating objects: 22462, done.\u001b[K\n",
            "remote: Counting objects: 100% (5248/5248), done.\u001b[K\n",
            "remote: Compressing objects: 100% (687/687), done.\u001b[K\n",
            "remote: Total 22462 (delta 4760), reused 4854 (delta 4559), pack-reused 17214\u001b[K\n",
            "Receiving objects: 100% (22462/22462), 20.70 MiB | 14.91 MiB/s, done.\n",
            "Resolving deltas: 100% (15465/15465), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/EleutherAI/lm-evaluation-harness/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M8kdWRZsTK70",
        "outputId": "d17ec86d-00d1-4602-d7aa-0c57d30459e7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/lm-evaluation-harness\n"
          ]
        }
      ],
      "source": [
        "%cd lm-evaluation-harness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jYL5OC-ATyNl",
        "outputId": "046fe9e4-5c06-45fd-f19e-aae4adf16c75"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Note: switching to 'e47e01beea79cfe87421e2dac49e64d499c240b4'.\n",
            "\n",
            "You are in 'detached HEAD' state. You can look around, make experimental\n",
            "changes and commit them, and you can discard any commits you make in this\n",
            "state without impacting any branches by switching back to a branch.\n",
            "\n",
            "If you want to create a new branch to retain commits you create, you may\n",
            "do so (now or later) by using -c with the switch command. Example:\n",
            "\n",
            "  git switch -c <new-branch-name>\n",
            "\n",
            "Or undo this operation with:\n",
            "\n",
            "  git switch -\n",
            "\n",
            "Turn off this advice by setting config variable advice.detachedHead to false\n",
            "\n",
            "HEAD is now at e47e01be Merge pull request #435 from EleutherAI/haileyschoelkopf-patch-1\n"
          ]
        }
      ],
      "source": [
        "!git checkout 'e47e01beea79cfe87421e2dac49e64d499c240b4'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BhvJ1jvGUAG6",
        "outputId": "fd973dd8-ef32-436c-d1e3-77b86d9a7317"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.6/519.6 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.0/77.0 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.6/85.6 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.7/227.7 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m53.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.1/111.1 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.6/65.6 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.7/7.7 MB\u001b[0m \u001b[31m106.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m104.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m35.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m258.1/258.1 kB\u001b[0m \u001b[31m31.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m89.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m119.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m34.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for pycountry (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install -q -e ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "r31XwmHQGoy-"
      },
      "outputs": [],
      "source": [
        "import transformers\n",
        "import accelerate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "dyHea5W5HJWS"
      },
      "outputs": [],
      "source": [
        "#!pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wzlixt_MSJEs"
      },
      "source": [
        "https://github.com/google/BIG-bench/issues/934"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G7iBFitW-GMx",
        "outputId": "ed860e62-fdaa-477d-fa0e-8238a40e5fbb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-10-21 03:33:47.513338: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Selected Tasks: ['arc_easy']\n",
            "Using device '0'\n",
            "Downloading (…)lve/main/config.json: 100% 665/665 [00:00<00:00, 2.78MB/s]\n",
            "Downloading model.safetensors: 100% 548M/548M [00:02<00:00, 228MB/s]\n",
            "Downloading (…)neration_config.json: 100% 124/124 [00:00<00:00, 754kB/s]\n",
            "Downloading (…)olve/main/vocab.json: 100% 1.04M/1.04M [00:00<00:00, 3.17MB/s]\n",
            "Downloading (…)olve/main/merges.txt: 100% 456k/456k [00:00<00:00, 1.87MB/s]\n",
            "Downloading (…)/main/tokenizer.json: 100% 1.36M/1.36M [00:00<00:00, 4.19MB/s]\n",
            "Downloading builder script: 100% 5.37k/5.37k [00:00<00:00, 12.1MB/s]\n",
            "Downloading metadata: 100% 4.47k/4.47k [00:00<00:00, 19.1MB/s]\n",
            "Downloading readme: 100% 8.66k/8.66k [00:00<00:00, 20.7MB/s]\n",
            "Downloading data: 100% 681M/681M [00:30<00:00, 22.1MB/s]\n",
            "Generating train split: 100% 2251/2251 [00:00<00:00, 7998.81 examples/s]\n",
            "Generating test split: 100% 2376/2376 [00:00<00:00, 10862.60 examples/s]\n",
            "Generating validation split: 100% 570/570 [00:00<00:00, 11669.10 examples/s]\n",
            "Running loglikelihood requests\n",
            "100% 9496/9496 [02:48<00:00, 56.19it/s]\n",
            "{\n",
            "  \"results\": {\n",
            "    \"arc_easy\": {\n",
            "      \"acc\": 0.43813131313131315,\n",
            "      \"acc_stderr\": 0.010180937100600062,\n",
            "      \"acc_norm\": 0.3947811447811448,\n",
            "      \"acc_norm_stderr\": 0.010030038935883556\n",
            "    }\n",
            "  },\n",
            "  \"versions\": {\n",
            "    \"arc_easy\": 0\n",
            "  },\n",
            "  \"config\": {\n",
            "    \"model\": \"gpt2\",\n",
            "    \"model_args\": \"\",\n",
            "    \"num_fewshot\": 0,\n",
            "    \"batch_size\": null,\n",
            "    \"device\": \"0\",\n",
            "    \"no_cache\": false,\n",
            "    \"limit\": null,\n",
            "    \"bootstrap_iters\": 100000,\n",
            "    \"description_dict\": {}\n",
            "  }\n",
            "}\n",
            "gpt2 (), limit: None, provide_description: False, num_fewshot: 0, batch_size: None\n",
            "|  Task  |Version| Metric |Value |   |Stderr|\n",
            "|--------|------:|--------|-----:|---|-----:|\n",
            "|arc_easy|      0|acc     |0.4381|±  |0.0102|\n",
            "|        |       |acc_norm|0.3948|±  |0.0100|\n",
            "\n",
            "CPU times: user 3.22 s, sys: 365 ms, total: 3.59 s\n",
            "Wall time: 4min 6s\n"
          ]
        }
      ],
      "source": [
        "# 3 minutes to run with Accuracy of 44%\n",
        "%%time\n",
        "!python main.py \\\n",
        "    --model gpt2 \\\n",
        "    --num_fewshot 0 \\\n",
        "    --tasks arc_easy \\\n",
        "    --device 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d91vsJhTY91w",
        "outputId": "1fea4f47-3340-4bdd-ad85-fb1fdb6c837b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-10-21 03:37:52.813056: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Selected Tasks: ['arc_easy']\n",
            "Using device '0'\n",
            "Downloading (…)lve/main/config.json: 100% 1.35k/1.35k [00:00<00:00, 7.19MB/s]\n",
            "Downloading model.safetensors: 100% 5.31G/5.31G [00:22<00:00, 231MB/s]\n",
            "Downloading (…)okenizer_config.json: 100% 200/200 [00:00<00:00, 892kB/s]\n",
            "Downloading (…)olve/main/vocab.json: 100% 798k/798k [00:00<00:00, 2.43MB/s]\n",
            "Downloading (…)olve/main/merges.txt: 100% 456k/456k [00:00<00:00, 1.86MB/s]\n",
            "Downloading (…)cial_tokens_map.json: 100% 90.0/90.0 [00:00<00:00, 477kB/s]\n",
            "Running loglikelihood requests\n",
            "100% 9496/9496 [08:25<00:00, 18.80it/s]\n",
            "{\n",
            "  \"results\": {\n",
            "    \"arc_easy\": {\n",
            "      \"acc\": 0.5618686868686869,\n",
            "      \"acc_stderr\": 0.010180937100600074,\n",
            "      \"acc_norm\": 0.502104377104377,\n",
            "      \"acc_norm_stderr\": 0.010259692651537032\n",
            "    }\n",
            "  },\n",
            "  \"versions\": {\n",
            "    \"arc_easy\": 0\n",
            "  },\n",
            "  \"config\": {\n",
            "    \"model\": \"gpt2\",\n",
            "    \"model_args\": \"pretrained=EleutherAI/gpt-neo-1.3B\",\n",
            "    \"num_fewshot\": 0,\n",
            "    \"batch_size\": null,\n",
            "    \"device\": \"0\",\n",
            "    \"no_cache\": false,\n",
            "    \"limit\": null,\n",
            "    \"bootstrap_iters\": 100000,\n",
            "    \"description_dict\": {}\n",
            "  }\n",
            "}\n",
            "gpt2 (pretrained=EleutherAI/gpt-neo-1.3B), limit: None, provide_description: False, num_fewshot: 0, batch_size: None\n",
            "|  Task  |Version| Metric |Value |   |Stderr|\n",
            "|--------|------:|--------|-----:|---|-----:|\n",
            "|arc_easy|      0|acc     |0.5619|±  |0.0102|\n",
            "|        |       |acc_norm|0.5021|±  |0.0103|\n",
            "\n",
            "CPU times: user 7.27 s, sys: 774 ms, total: 8.04 s\n",
            "Wall time: 9min 49s\n"
          ]
        }
      ],
      "source": [
        "#10 minutes to run with Accuracy of 56%\n",
        "%%time\n",
        "!python main.py \\\n",
        "    --model gpt2 \\\n",
        "    --model_args pretrained=EleutherAI/gpt-neo-1.3B \\\n",
        "    --num_fewshot 0 \\\n",
        "    --tasks arc_easy \\\n",
        "    --device 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IwSnwQsaZ7r1",
        "outputId": "4c22f046-48d3-4e6b-f9e0-62a880303b7f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-10-21 03:47:48.449712: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Selected Tasks: ['arc_easy']\n",
            "Using device '0'\n",
            "Downloading (…)lve/main/config.json: 100% 1.46k/1.46k [00:00<00:00, 7.78MB/s]\n",
            "Downloading model.safetensors: 100% 10.7G/10.7G [01:10<00:00, 152MB/s] \n",
            "^C\n",
            "CPU times: user 1.09 s, sys: 250 ms, total: 1.34 s\n",
            "Wall time: 2min 4s\n"
          ]
        }
      ],
      "source": [
        "#18 #10 minutes to run with Accuracy of 61%\n",
        "%%time\n",
        "!python main.py \\\n",
        "    --model gpt2 \\\n",
        "    --model_args pretrained=EleutherAI/gpt-neo-2.7B \\\n",
        "    --num_fewshot 0 \\\n",
        "    --tasks arc_easy \\\n",
        "    --device 0"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Its [best practice](https://stackoverflow.com/questions/44960472/storing-passwords-in-jupyter-notebooks) to use getpass in these types of notebooks. Your API key should be something like `sk-...`"
      ],
      "metadata": {
        "id": "xYUTbWNtVl9u"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "asj6HXacKfc_",
        "outputId": "f10438c0-b1c2-4cb0-e914-99ee56dc25da",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your OPENAI API Key··········\n"
          ]
        }
      ],
      "source": [
        "import getpass\n",
        "import os\n",
        "open_ai_key = getpass.getpass('Enter your OPENAI API Key')\n",
        "\n",
        "os.environ['OPENAI_API_SECRET_KEY'] = open_ai_key"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SzP-jZbGJfv3",
        "outputId": "dbc06968-8b1a-40a4-e275-4c39ec25ab55"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-10-21 03:51:06.509333: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Selected Tasks: ['sst']\n",
            "Downloading builder script: 100% 28.8k/28.8k [00:00<00:00, 36.3MB/s]\n",
            "Downloading metadata: 100% 28.7k/28.7k [00:00<00:00, 35.4MB/s]\n",
            "Downloading readme: 100% 27.9k/27.9k [00:00<00:00, 49.1MB/s]\n",
            "Downloading data: 100% 7.44M/7.44M [00:00<00:00, 81.4MB/s]\n",
            "Generating train split: 100% 67349/67349 [00:02<00:00, 26948.60 examples/s]\n",
            "Generating validation split: 100% 872/872 [00:00<00:00, 27797.11 examples/s]\n",
            "Generating test split: 100% 1821/1821 [00:00<00:00, 24548.83 examples/s]\n",
            "Running loglikelihood requests\n",
            "100% 88/88 [01:34<00:00,  1.07s/it]\n",
            "{\n",
            "  \"results\": {\n",
            "    \"sst\": {\n",
            "      \"acc\": 0.8600917431192661,\n",
            "      \"acc_stderr\": 0.011753981006588683\n",
            "    }\n",
            "  },\n",
            "  \"versions\": {\n",
            "    \"sst\": 0\n",
            "  },\n",
            "  \"config\": {\n",
            "    \"model\": \"gpt3\",\n",
            "    \"model_args\": \"engine=davinci\",\n",
            "    \"num_fewshot\": 2,\n",
            "    \"batch_size\": null,\n",
            "    \"device\": null,\n",
            "    \"no_cache\": false,\n",
            "    \"limit\": null,\n",
            "    \"bootstrap_iters\": 100000,\n",
            "    \"description_dict\": {}\n",
            "  }\n",
            "}\n",
            "gpt3 (engine=davinci), limit: None, provide_description: False, num_fewshot: 2, batch_size: None\n",
            "|Task|Version|Metric|Value |   |Stderr|\n",
            "|----|------:|------|-----:|---|-----:|\n",
            "|sst |      0|acc   |0.8601|±  |0.0118|\n",
            "\n",
            "CPU times: user 821 ms, sys: 121 ms, total: 942 ms\n",
            "Wall time: 2min 3s\n"
          ]
        }
      ],
      "source": [
        "#2 minutes to run with 88 requests and Accuracy of 86%\n",
        "%%time\n",
        "!python main.py \\\n",
        "    --model gpt3 \\\n",
        "    --model_args engine=davinci \\\n",
        "    --num_fewshot 2 \\\n",
        "    --tasks sst"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5zHkNETEAhFc"
      },
      "source": [
        "## Making a new task for the harness\n",
        "\n",
        "This part documents how to create a new task for the language model evaluation harness and is based on [this document](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/docs/task_guide.md)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KOxK9NsVAo6G",
        "outputId": "0d13188a-88ac-448e-b18d-a12ca0d9ad84"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'lm-evaluation-harness-new-task'...\n",
            "remote: Enumerating objects: 7910, done.\u001b[K\n",
            "remote: Counting objects: 100% (766/766), done.\u001b[K\n",
            "remote: Compressing objects: 100% (65/65), done.\u001b[K\n",
            "remote: Total 7910 (delta 730), reused 701 (delta 701), pack-reused 7144\u001b[K\n",
            "Receiving objects: 100% (7910/7910), 9.49 MiB | 21.74 MiB/s, done.\n",
            "Resolving deltas: 100% (5080/5080), done.\n",
            "[Errno 2] No such file or directory: 'lm-evaluation-harness-new-task'\n",
            "/content/lm-evaluation-harness\n",
            "Switched to a new branch 'cool-patrol'\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m203.7/203.7 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.6/62.6 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.9/98.9 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.7/3.7 MB\u001b[0m \u001b[31m28.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.5/227.5 kB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.9/468.9 kB\u001b[0m \u001b[31m31.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "# After forking...\n",
        "!cd .. && git clone https://github.com/esbenkc/lm-evaluation-harness.git lm-evaluation-harness-new-task\n",
        "%cd lm-evaluation-harness-new-task\n",
        "!git checkout -b \"cool-patrol\"\n",
        "!pip install -q -e \".[dev]\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "D3PsZoDEB_QD"
      },
      "outputs": [],
      "source": [
        "# See https://github.com/EleutherAI/lm-evaluation-harness/blob/master/docs/task_guide.md#creating-your-task-file\n",
        "\n",
        "!cp templates/new_task.py lm_eval/tasks/cool-patrol.py"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "fkIbhraPOIZK"
      },
      "execution_count": 13,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}