Mental_Health_Testing_Dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyP0yFCO/EFVSitJLtYBWhuH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/prasadsawant7/OpenAI_API_Fine_Tuning/blob/main/Mental_Health_Testing_Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6wbQheOyPBoS"
      },
      "outputs": [],
      "source": [
        "! pip install pandas numpy requests beautifulsoup4 openai"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import requests as req\n",
        "from bs4 import BeautifulSoup\n",
        "import openai\n",
        "import itertools\n",
        "import json"
      ],
      "metadata": {
        "id": "cnxRkndZlp93"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Immune System Dataset for testing purpose run this cell automatically prompt:completion .json file will be generated\n",
        "\n",
        "r = req.get('https://rajpurkar.github.io/SQuAD-explorer/explore/1.1/dev/Immune_system.html')\n",
        "soup = BeautifulSoup(r.content, 'html.parser')\n",
        "questions = []\n",
        "answers = []\n",
        "paras = soup.find_all('div', class_='para-wrap')\n",
        "\n",
        "for para in paras:\n",
        "  row = para.find('div', class_='row')\n",
        "  cols = row.find_all('div', class_='col-md-6')\n",
        "  que = set()\n",
        "  for i, col in enumerate(cols):\n",
        "    if i == 0:\n",
        "      pre = col.find('pre')\n",
        "      answers.append(pre.text)\n",
        "    elif i == 1:\n",
        "      strongs = col.find_all('strong', class_='question')\n",
        "      for strong in strongs:\n",
        "        que.add(strong.text)\n",
        "  questions.append(list(que))\n",
        "\n",
        "len(list(itertools.chain.from_iterable(questions)))\n",
        "dataset = list()\n",
        "for i in range(len(questions)):\n",
        "  data = dict()\n",
        "  data.update({\"Questions\": questions[i]})\n",
        "  data.update({\"Answer\": answers[i]})\n",
        "  dataset.append(data)\n",
        "\n",
        "json_data = json.dumps(dataset)\n",
        "\n",
        "with open(\"immune_system_Q&A.json\", \"w\") as file:\n",
        "  file.write(json_data)\n",
        "\n",
        "openai.api_key = \"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\"\n",
        "jsonl = ''\n",
        "\n",
        "for data in dataset:\n",
        "  paragraph = data[\"Answer\"]\n",
        "  for question in data[\"Questions\"]:\n",
        "    prompt = f\"Question: {question}\\nAnswer:\"\n",
        "    response = openai.Completion.create(\n",
        "        engine='text-davinci-003',\n",
        "        prompt=prompt,\n",
        "        max_tokens=100,\n",
        "        n=1,\n",
        "        stop=None,\n",
        "        temperature=0.8,\n",
        "        top_p=1.0,\n",
        "        frequency_penalty=0.0,\n",
        "        presence_penalty=0.0\n",
        "    )\n",
        "    answer = response.choices[0].text.strip()\n",
        "    jsonl += '{\"prompt\":\"'+ question + ' ->\",\"completion\": \"' + answer + '\"}\\n'\n",
        "    with open(\"immune_system_prompt_completion.json\", \"a\") as file:\n",
        "      file.write(jsonl)"
      ],
      "metadata": {
        "id": "Qc6eZ6RiPH7L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Dataset from: https://github.com/nbertagnolli/counsel-chat\n",
        "\n",
        "df = pd.read_csv(\"./20200325_counsel_chat.csv\", index_col=0)\n",
        "df.drop([\"questionLink\", \"therapistInfo\", \"therapistURL\", \"views\", \"split\"], axis=1, inplace=True)\n",
        "df.rename(columns={\"questionID\": \"id\", \"questionTitle\": \"prompt\", \"answerText\": \"completion\"}, inplace=True)\n",
        "\n",
        "dataset1 = []\n",
        "dataset2 = []\n",
        "\n",
        "def create_dataset() -> list:\n",
        "  dataset = []\n",
        "\n",
        "  for i in range(max(df[\"id\"]) + 1):\n",
        "    upvotes = []\n",
        "\n",
        "    for j in range(len(df)):\n",
        "      if(df[\"id\"][j] == i): upvotes.append({\"index\": j, \"upvotes\": df[\"upvotes\"][j]})\n",
        "\n",
        "    rev_sorted_upvotes = sorted(upvotes, key=lambda x: x['upvotes'], reverse=True)\n",
        "\n",
        "    if(len(rev_sorted_upvotes) == 0):\n",
        "      pass\n",
        "    elif(len(rev_sorted_upvotes) == 1):\n",
        "      dataset.append({\"prompt\": df[\"prompt\"].loc[rev_sorted_upvotes[0][\"index\"]], \"completion\": df[\"completion\"].loc[rev_sorted_upvotes[0][\"index\"]]})\n",
        "      dataset.append({\"prompt\": df[\"questionText\"].loc[rev_sorted_upvotes[0][\"index\"]], \"completion\": df[\"completion\"].loc[rev_sorted_upvotes[0][\"index\"]]})\n",
        "    elif(len(rev_sorted_upvotes) >= 2):\n",
        "      dataset.append({\"prompt\": df[\"prompt\"].loc[rev_sorted_upvotes[0][\"index\"]], \"completion\": df[\"completion\"].loc[rev_sorted_upvotes[0][\"index\"]]})\n",
        "      dataset.append({\"prompt\": df[\"questionText\"].loc[rev_sorted_upvotes[0][\"index\"]], \"completion\": df[\"completion\"].loc[rev_sorted_upvotes[1][\"index\"]]})\n",
        "\n",
        "  return dataset\n",
        "\n",
        "dataset1.extend(create_dataset())\n",
        "\n",
        "# Data Collected from this website: https://drnehamehta.com/questions-to-ask-a-psychologist/\n",
        "\n",
        "r = req.get(\"https://drnehamehta.com/questions-to-ask-a-psychologist/\")\n",
        "openai.api_key = \"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\"\n",
        "\n",
        "soup = BeautifulSoup(r.content, \"html.parser\")\n",
        "\n",
        "em = soup.find_all(\"em\")\n",
        "\n",
        "que_list = []\n",
        "for em in em:\n",
        "  questions = em.text.split(\"?\")\n",
        "  for question in questions:\n",
        "    updt_que = f\"{question[3:].strip()}?\"\n",
        "    if updt_que == \"?\" or updt_que == \"\": pass\n",
        "    else: que_list.append(updt_que)\n",
        "\n",
        "def get_completion(prompt: str) -> dict:\n",
        "  response = openai.Completion.create(\n",
        "      engine=\"text-davinci-003\",\n",
        "      prompt=prompt,\n",
        "      max_tokens=100,\n",
        "      n=1,\n",
        "      stop=None,\n",
        "      temperature=0.5,\n",
        "      top_p=1.0,\n",
        "      frequency_penalty=0.0,\n",
        "      presence_penalty=0.0\n",
        "  )\n",
        "  completion = response.choices[0].text.strip()\n",
        "  return {\"prompt\": prompt, \"completion\": completion}\n",
        "\n",
        "for question in que_list:\n",
        "  dataset2.append(get_completion(question))\n",
        "\n",
        "dataset1.extend(dataset2)\n",
        "\n",
        "data = pd.DataFrame(dataset1)\n",
        "data"
      ],
      "metadata": {
        "id": "BY93PLCmISgu",
        "outputId": "a97541dd-5a2e-448e-f3f4-4428221bc2b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 423
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                 prompt  \\\n",
              "0     Can I change my feeling of being worthless to ...   \n",
              "1     I'm going through some things with my feelings...   \n",
              "2     Why does it feel as if self-harm is an addiction?   \n",
              "3     I self-harm, and I stop for awhile. Then when ...   \n",
              "4     How can I help my husband after a suicide atte...   \n",
              "...                                                 ...   \n",
              "1674  How can I encourage communication skills in my...   \n",
              "1675  Are there ways to help my child get more sleep...   \n",
              "1676  What are my child’s strengths and how can they...   \n",
              "1677                           How can it be diagnosed?   \n",
              "1678  What kind of medicine will be prescribed by th...   \n",
              "\n",
              "                                             completion  \n",
              "0     If everyone thinks you're worthless, then mayb...  \n",
              "1     Hello, and thank you for your question and see...  \n",
              "2     In a way, self-harm can present somewhat like ...  \n",
              "3     Let me start by saying there are never too man...  \n",
              "4     Hello, and thank you for your question. This i...  \n",
              "...                                                 ...  \n",
              "1674  1. Read with your child. Reading stories is a ...  \n",
              "1675  1. Establish a consistent bedtime and wake-up ...  \n",
              "1676  There is no single answer to this question, as...  \n",
              "1677  Diagnosis of a hernia typically begins with a ...  \n",
              "1678  The type of medicine prescribed by a doctor wi...  \n",
              "\n",
              "[1679 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "\n",
              "  <div id=\"df-2f5e6fc7-d7df-4e28-8ba0-f412376ed335\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>prompt</th>\n",
              "      <th>completion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Can I change my feeling of being worthless to ...</td>\n",
              "      <td>If everyone thinks you're worthless, then mayb...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I'm going through some things with my feelings...</td>\n",
              "      <td>Hello, and thank you for your question and see...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Why does it feel as if self-harm is an addiction?</td>\n",
              "      <td>In a way, self-harm can present somewhat like ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I self-harm, and I stop for awhile. Then when ...</td>\n",
              "      <td>Let me start by saying there are never too man...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>How can I help my husband after a suicide atte...</td>\n",
              "      <td>Hello, and thank you for your question. This i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1674</th>\n",
              "      <td>How can I encourage communication skills in my...</td>\n",
              "      <td>1. Read with your child. Reading stories is a ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1675</th>\n",
              "      <td>Are there ways to help my child get more sleep...</td>\n",
              "      <td>1. Establish a consistent bedtime and wake-up ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1676</th>\n",
              "      <td>What are my child’s strengths and how can they...</td>\n",
              "      <td>There is no single answer to this question, as...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1677</th>\n",
              "      <td>How can it be diagnosed?</td>\n",
              "      <td>Diagnosis of a hernia typically begins with a ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1678</th>\n",
              "      <td>What kind of medicine will be prescribed by th...</td>\n",
              "      <td>The type of medicine prescribed by a doctor wi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1679 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2f5e6fc7-d7df-4e28-8ba0-f412376ed335')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "\n",
              "\n",
              "\n",
              "    <div id=\"df-ad1111e3-517a-4ccc-b65b-bbe59ca46321\">\n",
              "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ad1111e3-517a-4ccc-b65b-bbe59ca46321')\"\n",
              "              title=\"Suggest charts.\"\n",
              "              style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "      </button>\n",
              "    </div>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "    background-color: #E8F0FE;\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: #1967D2;\n",
              "    height: 32px;\n",
              "    padding: 0 0 0 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: #E2EBFA;\n",
              "    box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: #174EA6;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "    background-color: #3B4455;\n",
              "    fill: #D2E3FC;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart:hover {\n",
              "    background-color: #434B5C;\n",
              "    box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "    filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "    fill: #FFFFFF;\n",
              "  }\n",
              "</style>\n",
              "\n",
              "    <script>\n",
              "      async function quickchart(key) {\n",
              "        const containerElement = document.querySelector('#' + key);\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      }\n",
              "    </script>\n",
              "\n",
              "      <script>\n",
              "\n",
              "function displayQuickchartButton(domScope) {\n",
              "  let quickchartButtonEl =\n",
              "    domScope.querySelector('#df-ad1111e3-517a-4ccc-b65b-bbe59ca46321 button.colab-df-quickchart');\n",
              "  quickchartButtonEl.style.display =\n",
              "    google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "}\n",
              "\n",
              "        displayQuickchartButton(document);\n",
              "      </script>\n",
              "      <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2f5e6fc7-d7df-4e28-8ba0-f412376ed335 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2f5e6fc7-d7df-4e28-8ba0-f412376ed335');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.to_csv(\"mental_health_p_c.csv\", index=False)"
      ],
      "metadata": {
        "id": "3_42wt2YvWit"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}