Improvised ML Model Optimization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Weighted Clustering.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMDo7alfwRfiyRMbylN82Az",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/christophersingh/Mental-Health-Classification-Research/blob/main/Improvised%20ML%20Model%20Optimization.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gpwf6o2oXgpw"
      },
      "source": [
        "Import statistically significant data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "DzRhK1faWltY",
        "outputId": "791cbf53-d7a2-4e0c-9ded-eb966c6be026"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np \n",
        "\n",
        "df = pd.read_csv('./statistically_significant_data.csv')\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_period</th>\n",
              "      <th>value</th>\n",
              "      <th>CASE_RATE</th>\n",
              "      <th>HOSPITALIZED_RATE</th>\n",
              "      <th>sentiment__negative</th>\n",
              "      <th>sentiment__positive</th>\n",
              "      <th>indicator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>171.46</td>\n",
              "      <td>13.55</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>1554.45</td>\n",
              "      <td>153.69</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2529.03</td>\n",
              "      <td>630.34</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2552.64</td>\n",
              "      <td>1192.50</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2976.74</td>\n",
              "      <td>1830.07</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>20207.72</td>\n",
              "      <td>1160.60</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>15941.19</td>\n",
              "      <td>2540.10</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7067.11</td>\n",
              "      <td>2418.35</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7477.43</td>\n",
              "      <td>3995.12</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>6302.86</td>\n",
              "      <td>958.30</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 7 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       time_period  value  ...  sentiment__positive  indicator\n",
              "0              1.0   28.7  ...                 50.0        1.0\n",
              "1              1.0   28.7  ...                 50.0        1.0\n",
              "2              1.0   28.7  ...                 50.0        1.0\n",
              "3              1.0   28.7  ...                 50.0        1.0\n",
              "4              1.0   28.7  ...                 50.0        1.0\n",
              "...            ...    ...  ...                  ...        ...\n",
              "35779         23.0   35.8  ...                 49.0        2.0\n",
              "35780         23.0   35.8  ...                 49.0        2.0\n",
              "35781         23.0   35.8  ...                 49.0        2.0\n",
              "35782         23.0   35.8  ...                 49.0        2.0\n",
              "35783         23.0   35.8  ...                 49.0        2.0\n",
              "\n",
              "[35784 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GYyffsN-XlKA"
      },
      "source": [
        "Prepare Pipelines and Mixture Components"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LJ83-_t_bEJw"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from numpy import random\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.mixture import GaussianMixture\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.pipeline import make_pipeline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "65zxAOxZXpAk"
      },
      "source": [
        "Normalize Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JGbdS-Jqd_UD"
      },
      "source": [
        "X = df.values\n",
        "X = StandardScaler().fit_transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kOJ95rQ_bE-m"
      },
      "source": [
        "scaler = StandardScaler()\n",
        "model = KMeans(n_clusters=3, max_iter=5000, n_init=10, random_state=0) \n",
        "pipeline = make_pipeline(model)\n",
        "pipeline.fit(X)\n",
        "res = pipeline.predict(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWVXGVqkbdz5"
      },
      "source": [
        "model = GaussianMixture(n_components=3, max_iter=10000, n_init=100, random_state=0) \n",
        "res2 = model.fit_predict(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "1bqlQEeKby29",
        "outputId": "a92d5989-778b-401a-dda7-21f2308ffc57"
      },
      "source": [
        "df['K-Means Cluster'] = res\n",
        "df['GMM Result'] = res2\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_period</th>\n",
              "      <th>value</th>\n",
              "      <th>CASE_RATE</th>\n",
              "      <th>HOSPITALIZED_RATE</th>\n",
              "      <th>sentiment__negative</th>\n",
              "      <th>sentiment__positive</th>\n",
              "      <th>indicator</th>\n",
              "      <th>K-Means Cluster</th>\n",
              "      <th>GMM Result</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>171.46</td>\n",
              "      <td>13.55</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>1554.45</td>\n",
              "      <td>153.69</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2529.03</td>\n",
              "      <td>630.34</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2552.64</td>\n",
              "      <td>1192.50</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2976.74</td>\n",
              "      <td>1830.07</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>20207.72</td>\n",
              "      <td>1160.60</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>15941.19</td>\n",
              "      <td>2540.10</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7067.11</td>\n",
              "      <td>2418.35</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7477.43</td>\n",
              "      <td>3995.12</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>6302.86</td>\n",
              "      <td>958.30</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       time_period  value  CASE_RATE  ...  indicator  K-Means Cluster  GMM Result\n",
              "0              1.0   28.7     171.46  ...        1.0                1           1\n",
              "1              1.0   28.7    1554.45  ...        1.0                1           1\n",
              "2              1.0   28.7    2529.03  ...        1.0                1           1\n",
              "3              1.0   28.7    2552.64  ...        1.0                1           1\n",
              "4              1.0   28.7    2976.74  ...        1.0                1           1\n",
              "...            ...    ...        ...  ...        ...              ...         ...\n",
              "35779         23.0   35.8   20207.72  ...        2.0                0           2\n",
              "35780         23.0   35.8   15941.19  ...        2.0                0           2\n",
              "35781         23.0   35.8    7067.11  ...        2.0                0           2\n",
              "35782         23.0   35.8    7477.43  ...        2.0                0           2\n",
              "35783         23.0   35.8    6302.86  ...        2.0                2           2\n",
              "\n",
              "[35784 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BWL4PLqNXEAh"
      },
      "source": [
        "**Weighted k-means**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "16f1FpxoXAKc"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns; \n",
        "sns.set()  # for plot styling\n",
        "from sklearn.cluster import KMeans\n",
        "from pandas import DataFrame\n",
        "import random\n",
        "from sklearn.datasets.samples_generator import make_blobs\n",
        "from scipy.stats import skewnorm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E9I9fOA5XlzK"
      },
      "source": [
        "kmeans = KMeans(n_clusters=3, random_state=0, max_iter=5000)\n",
        "X = np.array(df.drop(['indicator'], 1).astype(float))\n",
        "Y = np.array(df['indicator'].astype(float))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pe6h0_hxt2_r"
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn import preprocessing\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "min_max_scaler = preprocessing.MinMaxScaler()\n",
        "x_scaled = StandardScaler().fit_transform(X)\n",
        "X = pd.DataFrame(x_scaled)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hYdy5QldXvFM"
      },
      "source": [
        "wt_kmeansclus = kmeans.fit(X)\n",
        "predicted_kmeans = kmeans.predict(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TT-CLIx3auB2"
      },
      "source": [
        "Show weighted centers of Features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SW859g5QMxea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 137
        },
        "outputId": "8b5fd15f-e323-407d-ed7e-9ee826981854"
      },
      "source": [
        "centersdf"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Weight time_period</th>\n",
              "      <th>Weight value</th>\n",
              "      <th>Weight case_rate</th>\n",
              "      <th>Weight hospitalized_rate</th>\n",
              "      <th>Weight sentiment__negative</th>\n",
              "      <th>Weight sentiment__positive</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>cluster_assignment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>-0.170563</td>\n",
              "      <td>0.983460</td>\n",
              "      <td>-0.239989</td>\n",
              "      <td>-0.115356</td>\n",
              "      <td>-0.169931</td>\n",
              "      <td>0.169931</td>\n",
              "      <td>1.307777</td>\n",
              "      <td>1.216099</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.131110</td>\n",
              "      <td>0.316017</td>\n",
              "      <td>1.587126</td>\n",
              "      <td>0.782792</td>\n",
              "      <td>1.099392</td>\n",
              "      <td>-1.099392</td>\n",
              "      <td>-1.468534</td>\n",
              "      <td>0.247146</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "      <td>-0.285642</td>\n",
              "      <td>-0.405569</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Weight time_period  Weight value  ...         7  cluster_assignment\n",
              "0           -0.170563      0.983460  ...  1.216099                   0\n",
              "1            1.131110      0.316017  ...  0.247146                   1\n",
              "2           -0.190397     -1.178939  ... -0.405569                   2\n",
              "\n",
              "[3 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zT-I1uZJapjf"
      },
      "source": [
        "Get weight by column and by observation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 234
        },
        "id": "jzyWfsAnYxFs",
        "outputId": "cc42b8ff-9d12-41f1-b7fe-d6e799d82ad7"
      },
      "source": [
        "df['cluster_assignment'] = predicted_kmeans\n",
        "centers = wt_kmeansclus.cluster_centers_\n",
        "centersdf=pd.DataFrame(centers)\n",
        "centersdf['cluster_assignment'] = centersdf.index\n",
        "centersdf=centersdf.rename(columns = {0:'Weight time_period',1:'Weight value', 2:'Weight case_rate', 3:'Weight hospitalized_rate', 4:'Weight sentiment__negative', 5:'Weight sentiment__positive'})\n",
        "df=df.merge(centersdf,on='cluster_assignment',how='left')\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_period</th>\n",
              "      <th>value</th>\n",
              "      <th>CASE_RATE</th>\n",
              "      <th>HOSPITALIZED_RATE</th>\n",
              "      <th>sentiment__negative</th>\n",
              "      <th>sentiment__positive</th>\n",
              "      <th>indicator</th>\n",
              "      <th>K-Means Cluster</th>\n",
              "      <th>GMM Result</th>\n",
              "      <th>cluster_assignment</th>\n",
              "      <th>Weight time_period</th>\n",
              "      <th>Weight value</th>\n",
              "      <th>Weight case_rate</th>\n",
              "      <th>Weight hospitalized_rate</th>\n",
              "      <th>Weight sentiment__negative</th>\n",
              "      <th>Weight sentiment__positive</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>171.46</td>\n",
              "      <td>13.55</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "      <td>-0.285642</td>\n",
              "      <td>-0.405569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>1554.45</td>\n",
              "      <td>153.69</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "      <td>-0.285642</td>\n",
              "      <td>-0.405569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2529.03</td>\n",
              "      <td>630.34</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "      <td>-0.285642</td>\n",
              "      <td>-0.405569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2552.64</td>\n",
              "      <td>1192.50</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "      <td>-0.285642</td>\n",
              "      <td>-0.405569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2976.74</td>\n",
              "      <td>1830.07</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "      <td>-0.285642</td>\n",
              "      <td>-0.405569</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   time_period  value  ...         6         7\n",
              "0          1.0   28.7  ... -0.285642 -0.405569\n",
              "1          1.0   28.7  ... -0.285642 -0.405569\n",
              "2          1.0   28.7  ... -0.285642 -0.405569\n",
              "3          1.0   28.7  ... -0.285642 -0.405569\n",
              "4          1.0   28.7  ... -0.285642 -0.405569\n",
              "\n",
              "[5 rows x 18 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6i-XL6o96Zhr"
      },
      "source": [
        "**Get Weights**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "id": "gvK0uNOF9uWj",
        "outputId": "b1f173d7-18cc-4fab-e100-23f2fcda214d"
      },
      "source": [
        "del df['cluster_assignment']\n",
        "del df[6]\n",
        "del df[7]\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_period</th>\n",
              "      <th>value</th>\n",
              "      <th>CASE_RATE</th>\n",
              "      <th>HOSPITALIZED_RATE</th>\n",
              "      <th>sentiment__negative</th>\n",
              "      <th>sentiment__positive</th>\n",
              "      <th>indicator</th>\n",
              "      <th>K-Means Cluster</th>\n",
              "      <th>GMM Result</th>\n",
              "      <th>Weight time_period</th>\n",
              "      <th>Weight value</th>\n",
              "      <th>Weight case_rate</th>\n",
              "      <th>Weight hospitalized_rate</th>\n",
              "      <th>Weight sentiment__negative</th>\n",
              "      <th>Weight sentiment__positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>171.46</td>\n",
              "      <td>13.55</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>1554.45</td>\n",
              "      <td>153.69</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2529.03</td>\n",
              "      <td>630.34</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2552.64</td>\n",
              "      <td>1192.50</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2976.74</td>\n",
              "      <td>1830.07</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>20207.72</td>\n",
              "      <td>1160.60</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1.131110</td>\n",
              "      <td>0.316017</td>\n",
              "      <td>1.587126</td>\n",
              "      <td>0.782792</td>\n",
              "      <td>1.099392</td>\n",
              "      <td>-1.099392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>15941.19</td>\n",
              "      <td>2540.10</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1.131110</td>\n",
              "      <td>0.316017</td>\n",
              "      <td>1.587126</td>\n",
              "      <td>0.782792</td>\n",
              "      <td>1.099392</td>\n",
              "      <td>-1.099392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7067.11</td>\n",
              "      <td>2418.35</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1.131110</td>\n",
              "      <td>0.316017</td>\n",
              "      <td>1.587126</td>\n",
              "      <td>0.782792</td>\n",
              "      <td>1.099392</td>\n",
              "      <td>-1.099392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7477.43</td>\n",
              "      <td>3995.12</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1.131110</td>\n",
              "      <td>0.316017</td>\n",
              "      <td>1.587126</td>\n",
              "      <td>0.782792</td>\n",
              "      <td>1.099392</td>\n",
              "      <td>-1.099392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>6302.86</td>\n",
              "      <td>958.30</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>-0.170563</td>\n",
              "      <td>0.983460</td>\n",
              "      <td>-0.239989</td>\n",
              "      <td>-0.115356</td>\n",
              "      <td>-0.169931</td>\n",
              "      <td>0.169931</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 15 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       time_period  ...  Weight sentiment__positive\n",
              "0              1.0  ...                    0.176591\n",
              "1              1.0  ...                    0.176591\n",
              "2              1.0  ...                    0.176591\n",
              "3              1.0  ...                    0.176591\n",
              "4              1.0  ...                    0.176591\n",
              "...            ...  ...                         ...\n",
              "35779         23.0  ...                   -1.099392\n",
              "35780         23.0  ...                   -1.099392\n",
              "35781         23.0  ...                   -1.099392\n",
              "35782         23.0  ...                   -1.099392\n",
              "35783         23.0  ...                    0.169931\n",
              "\n",
              "[35784 rows x 15 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 324
        },
        "id": "t-L4ey9ylMy0",
        "outputId": "0d7f4648-0a70-42d8-ce97-5bae934cd37d"
      },
      "source": [
        "df.describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_period</th>\n",
              "      <th>value</th>\n",
              "      <th>CASE_RATE</th>\n",
              "      <th>HOSPITALIZED_RATE</th>\n",
              "      <th>sentiment__negative</th>\n",
              "      <th>sentiment__positive</th>\n",
              "      <th>indicator</th>\n",
              "      <th>K-Means Cluster</th>\n",
              "      <th>GMM Result</th>\n",
              "      <th>Weight time_period</th>\n",
              "      <th>Weight value</th>\n",
              "      <th>Weight case_rate</th>\n",
              "      <th>Weight hospitalized_rate</th>\n",
              "      <th>Weight sentiment__negative</th>\n",
              "      <th>Weight sentiment__positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.00000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>14.045775</td>\n",
              "      <td>31.663146</td>\n",
              "      <td>4320.33733</td>\n",
              "      <td>1208.542817</td>\n",
              "      <td>500.704225</td>\n",
              "      <td>49.295775</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.057903</td>\n",
              "      <td>1.178404</td>\n",
              "      <td>0.057451</td>\n",
              "      <td>-0.263939</td>\n",
              "      <td>0.081119</td>\n",
              "      <td>0.037707</td>\n",
              "      <td>0.059015</td>\n",
              "      <td>-0.059015</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>6.431857</td>\n",
              "      <td>5.701017</td>\n",
              "      <td>3303.88542</td>\n",
              "      <td>998.747750</td>\n",
              "      <td>8.953674</td>\n",
              "      <td>8.953674</td>\n",
              "      <td>0.816508</td>\n",
              "      <td>0.720390</td>\n",
              "      <td>0.675609</td>\n",
              "      <td>0.508385</td>\n",
              "      <td>0.979133</td>\n",
              "      <td>0.713090</td>\n",
              "      <td>0.352880</td>\n",
              "      <td>0.492563</td>\n",
              "      <td>0.492563</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>22.000000</td>\n",
              "      <td>171.46000</td>\n",
              "      <td>13.550000</td>\n",
              "      <td>469.000000</td>\n",
              "      <td>30.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>-1.099392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>10.000000</td>\n",
              "      <td>25.100000</td>\n",
              "      <td>2593.36250</td>\n",
              "      <td>280.327500</td>\n",
              "      <td>495.000000</td>\n",
              "      <td>43.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.169931</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>14.500000</td>\n",
              "      <td>33.000000</td>\n",
              "      <td>3797.39500</td>\n",
              "      <td>905.785000</td>\n",
              "      <td>501.000000</td>\n",
              "      <td>49.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>20.000000</td>\n",
              "      <td>36.100000</td>\n",
              "      <td>4745.64750</td>\n",
              "      <td>1886.595000</td>\n",
              "      <td>507.000000</td>\n",
              "      <td>55.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>-0.170563</td>\n",
              "      <td>0.983460</td>\n",
              "      <td>-0.239989</td>\n",
              "      <td>-0.115356</td>\n",
              "      <td>-0.169931</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>23.000000</td>\n",
              "      <td>41.700000</td>\n",
              "      <td>20207.72000</td>\n",
              "      <td>3995.120000</td>\n",
              "      <td>520.000000</td>\n",
              "      <td>81.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>2.000000</td>\n",
              "      <td>1.131110</td>\n",
              "      <td>0.983460</td>\n",
              "      <td>1.587126</td>\n",
              "      <td>0.782792</td>\n",
              "      <td>1.099392</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        time_period  ...  Weight sentiment__positive\n",
              "count  35784.000000  ...                35784.000000\n",
              "mean      14.045775  ...                   -0.059015\n",
              "std        6.431857  ...                    0.492563\n",
              "min        1.000000  ...                   -1.099392\n",
              "25%       10.000000  ...                    0.169931\n",
              "50%       14.500000  ...                    0.176591\n",
              "75%       20.000000  ...                    0.176591\n",
              "max       23.000000  ...                    0.176591\n",
              "\n",
              "[8 rows x 15 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "id": "1RL7i68plZyY",
        "outputId": "de17691a-3aea-40d5-cde1-876b8acc3370"
      },
      "source": [
        "weights = df[['Weight time_period', 'Weight value', 'Weight case_rate', 'Weight hospitalized_rate', 'Weight sentiment__negative', 'Weight sentiment__positive']].describe()\n",
        "weights"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Weight time_period</th>\n",
              "      <th>Weight value</th>\n",
              "      <th>Weight case_rate</th>\n",
              "      <th>Weight hospitalized_rate</th>\n",
              "      <th>Weight sentiment__negative</th>\n",
              "      <th>Weight sentiment__positive</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "      <td>35784.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.057451</td>\n",
              "      <td>-0.263939</td>\n",
              "      <td>0.081119</td>\n",
              "      <td>0.037707</td>\n",
              "      <td>0.059015</td>\n",
              "      <td>-0.059015</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.508385</td>\n",
              "      <td>0.979133</td>\n",
              "      <td>0.713090</td>\n",
              "      <td>0.352880</td>\n",
              "      <td>0.492563</td>\n",
              "      <td>0.492563</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>-1.099392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.169931</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>-0.190397</td>\n",
              "      <td>-1.178939</td>\n",
              "      <td>-0.265806</td>\n",
              "      <td>-0.137239</td>\n",
              "      <td>-0.176591</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>-0.170563</td>\n",
              "      <td>0.983460</td>\n",
              "      <td>-0.239989</td>\n",
              "      <td>-0.115356</td>\n",
              "      <td>-0.169931</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.131110</td>\n",
              "      <td>0.983460</td>\n",
              "      <td>1.587126</td>\n",
              "      <td>0.782792</td>\n",
              "      <td>1.099392</td>\n",
              "      <td>0.176591</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Weight time_period  ...  Weight sentiment__positive\n",
              "count        35784.000000  ...                35784.000000\n",
              "mean             0.057451  ...                   -0.059015\n",
              "std              0.508385  ...                    0.492563\n",
              "min             -0.190397  ...                   -1.099392\n",
              "25%             -0.190397  ...                    0.169931\n",
              "50%             -0.190397  ...                    0.176591\n",
              "75%             -0.170563  ...                    0.176591\n",
              "max              1.131110  ...                    0.176591\n",
              "\n",
              "[8 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RFzv3zG9X3lj"
      },
      "source": [
        "Take standard deviation to obtain feature weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L6UmFQyglygx"
      },
      "source": [
        "helper = weights.loc['std']\n",
        "helper"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ks8pNfYKMmwL"
      },
      "source": [
        "time_period_weight = helper['Weight time_period']\n",
        "value_weight = helper['Weight value']\n",
        "case_rate_weight = helper['Weight case_rate']\n",
        "hospitalized_rate_weight = helper['Weight hospitalized_rate']\n",
        "sentiment_negative_weight = helper['Weight sentiment__negative']\n",
        "sentiment_positive_weight = helper['Weight sentiment__positive']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "UqN388PjNGk7",
        "outputId": "5feda665-c4d9-4230-a83d-c6e53caf6ae4"
      },
      "source": [
        "df.drop(['K-Means Cluster', 'GMM Result', 'Weight time_period', 'Weight value', 'Weight case_rate', 'Weight hospitalized_rate', 'Weight sentiment__negative', 'Weight sentiment__positive'], axis=1, inplace=True)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_period</th>\n",
              "      <th>value</th>\n",
              "      <th>CASE_RATE</th>\n",
              "      <th>HOSPITALIZED_RATE</th>\n",
              "      <th>sentiment__negative</th>\n",
              "      <th>sentiment__positive</th>\n",
              "      <th>indicator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>171.46</td>\n",
              "      <td>13.55</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>1554.45</td>\n",
              "      <td>153.69</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2529.03</td>\n",
              "      <td>630.34</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2552.64</td>\n",
              "      <td>1192.50</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1.0</td>\n",
              "      <td>28.7</td>\n",
              "      <td>2976.74</td>\n",
              "      <td>1830.07</td>\n",
              "      <td>500.0</td>\n",
              "      <td>50.0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>20207.72</td>\n",
              "      <td>1160.60</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>15941.19</td>\n",
              "      <td>2540.10</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7067.11</td>\n",
              "      <td>2418.35</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>7477.43</td>\n",
              "      <td>3995.12</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>23.0</td>\n",
              "      <td>35.8</td>\n",
              "      <td>6302.86</td>\n",
              "      <td>958.30</td>\n",
              "      <td>501.0</td>\n",
              "      <td>49.0</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 7 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       time_period  value  ...  sentiment__positive  indicator\n",
              "0              1.0   28.7  ...                 50.0        1.0\n",
              "1              1.0   28.7  ...                 50.0        1.0\n",
              "2              1.0   28.7  ...                 50.0        1.0\n",
              "3              1.0   28.7  ...                 50.0        1.0\n",
              "4              1.0   28.7  ...                 50.0        1.0\n",
              "...            ...    ...  ...                  ...        ...\n",
              "35779         23.0   35.8  ...                 49.0        2.0\n",
              "35780         23.0   35.8  ...                 49.0        2.0\n",
              "35781         23.0   35.8  ...                 49.0        2.0\n",
              "35782         23.0   35.8  ...                 49.0        2.0\n",
              "35783         23.0   35.8  ...                 49.0        2.0\n",
              "\n",
              "[35784 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "yMRrSLGvNu1O",
        "outputId": "91334c74-ab17-4297-b3c0-de21fa299ef8"
      },
      "source": [
        "indicator = list(df['indicator'])\n",
        "df.drop(['indicator'], axis=1, inplace=True)\n",
        "import pandas as pd\n",
        "from sklearn import preprocessing\n",
        "\n",
        "x = df.values #returns a numpy array\n",
        "min_max_scaler = preprocessing.MinMaxScaler()\n",
        "x_scaled = min_max_scaler.fit_transform(x)\n",
        "df = pd.DataFrame(x_scaled)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.069024</td>\n",
              "      <td>0.035197</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.117665</td>\n",
              "      <td>0.154911</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.118844</td>\n",
              "      <td>0.296102</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.140010</td>\n",
              "      <td>0.456232</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.288090</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.787060</td>\n",
              "      <td>0.634561</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.344159</td>\n",
              "      <td>0.603983</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.364637</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.306015</td>\n",
              "      <td>0.237281</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 6 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         0         1         2         3         4         5\n",
              "0      0.0  0.340102  0.000000  0.000000  0.607843  0.392157\n",
              "1      0.0  0.340102  0.069024  0.035197  0.607843  0.392157\n",
              "2      0.0  0.340102  0.117665  0.154911  0.607843  0.392157\n",
              "3      0.0  0.340102  0.118844  0.296102  0.607843  0.392157\n",
              "4      0.0  0.340102  0.140010  0.456232  0.607843  0.392157\n",
              "...    ...       ...       ...       ...       ...       ...\n",
              "35779  1.0  0.700508  1.000000  0.288090  0.627451  0.372549\n",
              "35780  1.0  0.700508  0.787060  0.634561  0.627451  0.372549\n",
              "35781  1.0  0.700508  0.344159  0.603983  0.627451  0.372549\n",
              "35782  1.0  0.700508  0.364637  1.000000  0.627451  0.372549\n",
              "35783  1.0  0.700508  0.306015  0.237281  0.627451  0.372549\n",
              "\n",
              "[35784 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C-h1-IDKa3rC"
      },
      "source": [
        "Obatain adjusted feature weight"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_GoJcy3rPiB-"
      },
      "source": [
        "feature_weights = [time_period_weight, value_weight, case_rate_weight, hospitalized_rate_weight, sentiment_negative_weight, sentiment_positive_weight]\n",
        "\n",
        "def cluster(f1, f2, f3, f4, f5, f6):\n",
        "  adjusted_weight_f1 = f1 * feature_weights[0]\n",
        "  adjusted_weight_f2 = f2 * feature_weights[1]\n",
        "  adjusted_weight_f3 = f3 * feature_weights[2]\n",
        "  adjusted_weight_f4 = f4 * feature_weights[3]\n",
        "  adjusted_weight_f5 = f5 * feature_weights[4]\n",
        "  adjusted_weight_f6 = f6 * feature_weights[5]\n",
        "  total = adjusted_weight_f1 + adjusted_weight_f2 + adjusted_weight_f3 + adjusted_weight_f4 + adjusted_weight_f5 + adjusted_weight_f6\n",
        "  return np.log(total)\n",
        "\n",
        "clusterResult = []\n",
        "for index, row in df.iterrows():\n",
        "    getCluster = cluster(row[0], row[1], row[2], row[3], row[4], row[5])\n",
        "    clusterResult.append(getCluster)\n",
        "\n",
        "df['feature_weight'] = clusterResult"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "S-zUBFNUSey5",
        "outputId": "3ab4900d-2152-44b2-cca5-bf3f8be687d1"
      },
      "source": [
        "df['indicator'] = indicator\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>feature_weight</th>\n",
              "      <th>indicator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>-0.191684</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.069024</td>\n",
              "      <td>0.035197</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>-0.119675</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.117665</td>\n",
              "      <td>0.154911</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>-0.036520</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.118844</td>\n",
              "      <td>0.296102</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>0.014694</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.140010</td>\n",
              "      <td>0.456232</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>0.082872</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.288090</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.916926</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.787060</td>\n",
              "      <td>0.634561</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.905030</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.344159</td>\n",
              "      <td>0.603983</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.763320</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.364637</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.832795</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.306015</td>\n",
              "      <td>0.237281</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.687525</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 8 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         0         1         2  ...         5  feature_weight  indicator\n",
              "0      0.0  0.340102  0.000000  ...  0.392157       -0.191684        1.0\n",
              "1      0.0  0.340102  0.069024  ...  0.392157       -0.119675        1.0\n",
              "2      0.0  0.340102  0.117665  ...  0.392157       -0.036520        1.0\n",
              "3      0.0  0.340102  0.118844  ...  0.392157        0.014694        1.0\n",
              "4      0.0  0.340102  0.140010  ...  0.392157        0.082872        1.0\n",
              "...    ...       ...       ...  ...       ...             ...        ...\n",
              "35779  1.0  0.700508  1.000000  ...  0.372549        0.916926        2.0\n",
              "35780  1.0  0.700508  0.787060  ...  0.372549        0.905030        2.0\n",
              "35781  1.0  0.700508  0.344159  ...  0.372549        0.763320        2.0\n",
              "35782  1.0  0.700508  0.364637  ...  0.372549        0.832795        2.0\n",
              "35783  1.0  0.700508  0.306015  ...  0.372549        0.687525        2.0\n",
              "\n",
              "[35784 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 287
        },
        "id": "5ZMOQf0_qhZt",
        "outputId": "210e4f78-e4d4-4b9a-dcb2-b7509cf53f01"
      },
      "source": [
        "df[df['indicator']==0].describe()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>feature_weight</th>\n",
              "      <th>indicator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>11928.000000</td>\n",
              "      <td>11928.000000</td>\n",
              "      <td>11928.000000</td>\n",
              "      <td>11928.000000</td>\n",
              "      <td>11928.000000</td>\n",
              "      <td>11928.000000</td>\n",
              "      <td>11928.000000</td>\n",
              "      <td>11928.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.592990</td>\n",
              "      <td>0.539179</td>\n",
              "      <td>0.207068</td>\n",
              "      <td>0.300131</td>\n",
              "      <td>0.621651</td>\n",
              "      <td>0.378349</td>\n",
              "      <td>0.431477</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.292365</td>\n",
              "      <td>0.129753</td>\n",
              "      <td>0.164900</td>\n",
              "      <td>0.250850</td>\n",
              "      <td>0.175567</td>\n",
              "      <td>0.175567</td>\n",
              "      <td>0.217639</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.243655</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>-0.246252</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.409091</td>\n",
              "      <td>0.436548</td>\n",
              "      <td>0.120876</td>\n",
              "      <td>0.067003</td>\n",
              "      <td>0.509804</td>\n",
              "      <td>0.254902</td>\n",
              "      <td>0.281836</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.613636</td>\n",
              "      <td>0.558376</td>\n",
              "      <td>0.180969</td>\n",
              "      <td>0.224091</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.443400</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.863636</td>\n",
              "      <td>0.624365</td>\n",
              "      <td>0.228295</td>\n",
              "      <td>0.470429</td>\n",
              "      <td>0.745098</td>\n",
              "      <td>0.490196</td>\n",
              "      <td>0.587654</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.741117</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.844852</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                  0             1  ...  feature_weight  indicator\n",
              "count  11928.000000  11928.000000  ...    11928.000000    11928.0\n",
              "mean       0.592990      0.539179  ...        0.431477        0.0\n",
              "std        0.292365      0.129753  ...        0.217639        0.0\n",
              "min        0.000000      0.243655  ...       -0.246252        0.0\n",
              "25%        0.409091      0.436548  ...        0.281836        0.0\n",
              "50%        0.613636      0.558376  ...        0.443400        0.0\n",
              "75%        0.863636      0.624365  ...        0.587654        0.0\n",
              "max        1.000000      0.741117  ...        0.844852        0.0\n",
              "\n",
              "[8 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Ala7pvzWxCM",
        "outputId": "2ea5e80f-8404-4cbc-fb1e-fd5b686fb45a"
      },
      "source": [
        "indicator0, indicator1, indicator2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.28183599677213284, -0.02747191424547771, 0.4296134921993718)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "omA6oUxKJ8F5"
      },
      "source": [
        "Find distance between all features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "ErFqy3joJ-WS",
        "outputId": "5a9e558e-f5fa-42fe-fb46-e26ca05f5602"
      },
      "source": [
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>feature_weight</th>\n",
              "      <th>indicator</th>\n",
              "      <th>clusters</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>-0.191684</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.069024</td>\n",
              "      <td>0.035197</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>-0.119675</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.117665</td>\n",
              "      <td>0.154911</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>-0.036520</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.118844</td>\n",
              "      <td>0.296102</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>0.014694</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.140010</td>\n",
              "      <td>0.456232</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>0.082872</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.288090</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.916926</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.787060</td>\n",
              "      <td>0.634561</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.905030</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.344159</td>\n",
              "      <td>0.603983</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.763320</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.364637</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.832795</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.306015</td>\n",
              "      <td>0.237281</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>0.687525</td>\n",
              "      <td>2.0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         0         1         2  ...  feature_weight  indicator  clusters\n",
              "0      0.0  0.340102  0.000000  ...       -0.191684        1.0         1\n",
              "1      0.0  0.340102  0.069024  ...       -0.119675        1.0         1\n",
              "2      0.0  0.340102  0.117665  ...       -0.036520        1.0         1\n",
              "3      0.0  0.340102  0.118844  ...        0.014694        1.0         1\n",
              "4      0.0  0.340102  0.140010  ...        0.082872        1.0         1\n",
              "...    ...       ...       ...  ...             ...        ...       ...\n",
              "35779  1.0  0.700508  1.000000  ...        0.916926        2.0         2\n",
              "35780  1.0  0.700508  0.787060  ...        0.905030        2.0         2\n",
              "35781  1.0  0.700508  0.344159  ...        0.763320        2.0         2\n",
              "35782  1.0  0.700508  0.364637  ...        0.832795        2.0         2\n",
              "35783  1.0  0.700508  0.306015  ...        0.687525        2.0         2\n",
              "\n",
              "[35784 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "DQ3vnBIXKA4z",
        "outputId": "98bee671-14cd-453c-c9b4-c83e82f4da16"
      },
      "source": [
        "df.drop(['feature_weight', 'clusters'], axis=1, inplace=True)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>indicator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.069024</td>\n",
              "      <td>0.035197</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.117665</td>\n",
              "      <td>0.154911</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.118844</td>\n",
              "      <td>0.296102</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.340102</td>\n",
              "      <td>0.140010</td>\n",
              "      <td>0.456232</td>\n",
              "      <td>0.607843</td>\n",
              "      <td>0.392157</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.288090</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.787060</td>\n",
              "      <td>0.634561</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.344159</td>\n",
              "      <td>0.603983</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.364637</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>1.0</td>\n",
              "      <td>0.700508</td>\n",
              "      <td>0.306015</td>\n",
              "      <td>0.237281</td>\n",
              "      <td>0.627451</td>\n",
              "      <td>0.372549</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 7 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         0         1         2         3         4         5  indicator\n",
              "0      0.0  0.340102  0.000000  0.000000  0.607843  0.392157        1.0\n",
              "1      0.0  0.340102  0.069024  0.035197  0.607843  0.392157        1.0\n",
              "2      0.0  0.340102  0.117665  0.154911  0.607843  0.392157        1.0\n",
              "3      0.0  0.340102  0.118844  0.296102  0.607843  0.392157        1.0\n",
              "4      0.0  0.340102  0.140010  0.456232  0.607843  0.392157        1.0\n",
              "...    ...       ...       ...       ...       ...       ...        ...\n",
              "35779  1.0  0.700508  1.000000  0.288090  0.627451  0.372549        2.0\n",
              "35780  1.0  0.700508  0.787060  0.634561  0.627451  0.372549        2.0\n",
              "35781  1.0  0.700508  0.344159  0.603983  0.627451  0.372549        2.0\n",
              "35782  1.0  0.700508  0.364637  1.000000  0.627451  0.372549        2.0\n",
              "35783  1.0  0.700508  0.306015  0.237281  0.627451  0.372549        2.0\n",
              "\n",
              "[35784 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TMQHq_lTECRI"
      },
      "source": [
        "feature_weights = [time_period_weight, value_weight, case_rate_weight, hospitalized_rate_weight, sentiment_negative_weight, sentiment_positive_weight]\n",
        "\n",
        "def cluster(f1, f2, f3, f4, f5, f6):\n",
        "  adjusted_weight_f1 = f1 * (feature_weights[0]) * math.phi\n",
        "  adjusted_weight_f2 = f2 * (feature_weights[1]) * math.phi\n",
        "  adjusted_weight_f3 = f3 * (feature_weights[2]) * math.phi\n",
        "  adjusted_weight_f4 = f4 * (feature_weights[3]) * math.phi\n",
        "  adjusted_weight_f5 = f5 * (feature_weights[4]) * math.phi\n",
        "  adjusted_weight_f6 = f6 * (feature_weights[5]) * math.phi\n",
        "  total = adjusted_weight_f1 + adjusted_weight_f2 + adjusted_weight_f3 + adjusted_weight_f4 + adjusted_weight_f5 + adjusted_weight_f6\n",
        "  return (np.log(total))\n",
        "\n",
        "clusterResult = []\n",
        "for index, row in df.iterrows():\n",
        "    getCluster = cluster(row[0], row[1], row[2], row[3], row[4], row[5])\n",
        "    clusterResult.append(getCluster)\n",
        "\n",
        "df['feature_weight'] = clusterResult"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PMfqCNMqZQ7j"
      },
      "source": [
        "Summation Operator"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iYpl8saeaYjP"
      },
      "source": [
        "Iterate through dataframe to convert all observations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "vKNLI9EmX-aW",
        "outputId": "fae1cf1a-cb06-49ab-cdbd-4d70949d5e99"
      },
      "source": [
        "feature_weights = [time_period_weight, value_weight, case_rate_weight, hospitalized_rate_weight, sentiment_negative_weight, sentiment_positive_weight]\n",
        "\n",
        "def cluster(f1, f2, f3, f4, f5, f6):\n",
        "  adjusted_weight_f1 = f1 * (feature_weights[0]) * math.phi\n",
        "  adjusted_weight_f2 = f2 * (feature_weights[1]) * math.phi\n",
        "  adjusted_weight_f3 = f3 * (feature_weights[2]) * math.phi\n",
        "  adjusted_weight_f4 = f4 * (feature_weights[3]) * math.phi\n",
        "  adjusted_weight_f5 = f5 * (feature_weights[4]) * math.phi\n",
        "  adjusted_weight_f6 = f6 * (feature_weights[5]) * math.phi \n",
        "  #total = adjusted_weight_f1 + adjusted_weight_f2 + adjusted_weight_f3 + adjusted_weight_f4 + adjusted_weight_f5 + adjusted_weight_f6\n",
        "  return adjusted_weight_f1, adjusted_weight_f2, adjusted_weight_f3, adjusted_weight_f4, adjusted_weight_f5, adjusted_weight_f6\n",
        "\n",
        "clusterResult = []\n",
        "weightF1, weightF2, weightF3, weightF4, weightF5,weightF6 = [],[],[],[],[],[]\n",
        "for index, row in df.iterrows():\n",
        "    f1, f2, f3, f4, f5, f6 = cluster(row[0], row[1], row[2], row[3], row[4], row[5])\n",
        "    weightF1.append(f1)\n",
        "    weightF2.append(f2)\n",
        "    weightF3.append(f3)\n",
        "    weightF4.append(f4)\n",
        "    weightF5.append(f5)\n",
        "    weightF6.append(f6)\n",
        "    \n",
        "new_df = pd.DataFrame(list(zip(weightF1, weightF2, weightF3, weightF4, weightF5, weightF6, df['indicator'].values)),columns =['time_period', 'value', 'case_rate','hospitalized_rate','negative_tweets','positive_tweets', 'indicator'])\n",
        "#df['feature_weight'] = clusterResult\n",
        "new_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>time_period</th>\n",
              "      <th>value</th>\n",
              "      <th>case_rate</th>\n",
              "      <th>hospitalized_rate</th>\n",
              "      <th>negative_tweets</th>\n",
              "      <th>positive_tweets</th>\n",
              "      <th>indicator</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.333005</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.299401</td>\n",
              "      <td>0.193162</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.333005</td>\n",
              "      <td>0.049221</td>\n",
              "      <td>0.012420</td>\n",
              "      <td>0.299401</td>\n",
              "      <td>0.193162</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.333005</td>\n",
              "      <td>0.083906</td>\n",
              "      <td>0.054665</td>\n",
              "      <td>0.299401</td>\n",
              "      <td>0.193162</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.333005</td>\n",
              "      <td>0.084746</td>\n",
              "      <td>0.104488</td>\n",
              "      <td>0.299401</td>\n",
              "      <td>0.193162</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.333005</td>\n",
              "      <td>0.099840</td>\n",
              "      <td>0.160995</td>\n",
              "      <td>0.299401</td>\n",
              "      <td>0.193162</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35779</th>\n",
              "      <td>0.508385</td>\n",
              "      <td>0.685890</td>\n",
              "      <td>0.713090</td>\n",
              "      <td>0.101661</td>\n",
              "      <td>0.309059</td>\n",
              "      <td>0.183504</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35780</th>\n",
              "      <td>0.508385</td>\n",
              "      <td>0.685890</td>\n",
              "      <td>0.561245</td>\n",
              "      <td>0.223924</td>\n",
              "      <td>0.309059</td>\n",
              "      <td>0.183504</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35781</th>\n",
              "      <td>0.508385</td>\n",
              "      <td>0.685890</td>\n",
              "      <td>0.245416</td>\n",
              "      <td>0.213134</td>\n",
              "      <td>0.309059</td>\n",
              "      <td>0.183504</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35782</th>\n",
              "      <td>0.508385</td>\n",
              "      <td>0.685890</td>\n",
              "      <td>0.260019</td>\n",
              "      <td>0.352880</td>\n",
              "      <td>0.309059</td>\n",
              "      <td>0.183504</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35783</th>\n",
              "      <td>0.508385</td>\n",
              "      <td>0.685890</td>\n",
              "      <td>0.218216</td>\n",
              "      <td>0.083732</td>\n",
              "      <td>0.309059</td>\n",
              "      <td>0.183504</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>35784 rows × 7 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       time_period     value  ...  positive_tweets  indicator\n",
              "0         0.000000  0.333005  ...         0.193162        1.0\n",
              "1         0.000000  0.333005  ...         0.193162        1.0\n",
              "2         0.000000  0.333005  ...         0.193162        1.0\n",
              "3         0.000000  0.333005  ...         0.193162        1.0\n",
              "4         0.000000  0.333005  ...         0.193162        1.0\n",
              "...            ...       ...  ...              ...        ...\n",
              "35779     0.508385  0.685890  ...         0.183504        2.0\n",
              "35780     0.508385  0.685890  ...         0.183504        2.0\n",
              "35781     0.508385  0.685890  ...         0.183504        2.0\n",
              "35782     0.508385  0.685890  ...         0.183504        2.0\n",
              "35783     0.508385  0.685890  ...         0.183504        2.0\n",
              "\n",
              "[35784 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4w4jDqL9ZujD"
      },
      "source": [
        "Partition using derived metrics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26cLe3PVaY2N"
      },
      "source": [
        "value = new_df['value'].values\n",
        "clusters = []\n",
        "#Min/max +/- standard deviation to get the thresholds\n",
        "std_err = value.std()\n",
        "low = value.mean() - std_err\n",
        "high = value.mean() + std_err\n",
        "for x in value:\n",
        "  if (0 <= x <= low):\n",
        "    clusters.append(1)\n",
        "  elif (low < x <= high):\n",
        "    clusters.append(0)\n",
        "  else:\n",
        "    clusters.append(2)\n",
        "\n",
        "new_df['clusters'] = clusters"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q_u-km3Yfn95",
        "outputId": "85083774-2d91-4210-8739-73cdefb1f5a4"
      },
      "source": [
        "print(\"Accuracy:\",accuracy_score(new_df['indicator'] , new_df['clusters']))\n",
        "print(classification_report(new_df['indicator'] , new_df['clusters']))\n",
        "print(confusion_matrix(new_df['indicator'] , new_df['clusters']))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 0.8615023474178404\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.85      0.70      0.77     11928\n",
            "         1.0       0.93      0.95      0.94     11928\n",
            "         2.0       0.81      0.93      0.86     11928\n",
            "\n",
            "    accuracy                           0.86     35784\n",
            "   macro avg       0.86      0.86      0.86     35784\n",
            "weighted avg       0.86      0.86      0.86     35784\n",
            "\n",
            "[[ 8400   882  2646]\n",
            " [  588 11340     0]\n",
            " [  840     0 11088]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0PVWIgtuaJoK"
      },
      "source": [
        "Get Cluster Entropy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "021OZ72I59Ss",
        "outputId": "bc9f7ae9-b22c-4d6e-d704-ab1f90bdf371"
      },
      "source": [
        "import math\n",
        "def get_entropys(df_clustered, label_col_name='indicator', cluster_col_name='cluster', cluster_values=range(1,3,1)):\n",
        "    p = [[round(df_clustered[(df_clustered[label_col_name]==label) & (df_clustered[cluster_col_name]==cluster)].shape[0]/\n",
        "                df_clustered[df_clustered[cluster_col_name]==cluster].shape[0],3) for label in cluster_values] for cluster in cluster_values]\n",
        "    p_log_p = [[p_i*math.log(p_i) for p_i in p_i_s if p_i!=0] for p_i_s in p]\n",
        "    e = [-round(sum(p_log_p_i),3) for p_log_p_i in p_log_p]\n",
        "    return e\n",
        "  \n",
        "entropys_5 = get_entropys(new_df, label_col_name='indicator', cluster_col_name='clusters', cluster_values=range(1,3,1))\n",
        "print(\"Clusters' Entropy\")\n",
        "print(\"Entropy: {0:.3f}\".format(min(entropys_5)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Clusters' Entropy\n",
            "Entropy: 0.259\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w0jBfrwsaMKY"
      },
      "source": [
        "Get Cluster Purity"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BSoJipMf3LtW",
        "outputId": "c3862b71-454b-4d4e-835e-880b10e0cb29"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "\n",
        "def purity_score(y_true, y_pred):\n",
        "    # compute contingency matrix (also called confusion matrix)\n",
        "    contingency_matrix = metrics.cluster.contingency_matrix(y_true, y_pred)\n",
        "    # return purity\n",
        "    return np.sum(np.amax(contingency_matrix, axis=0)) / np.sum(contingency_matrix) \n",
        "\n",
        "purity = purity_score(new_df['indicator'], new_df['clusters'])\n",
        "print(\"The purity is:\",purity)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The purity is: 0.8615023474178404\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bxjtt6Rq3hXO",
        "outputId": "11033b32-f03f-48c4-fc04-c06ddc00a13d"
      },
      "source": [
        "from sklearn.metrics.cluster import v_measure_score\n",
        "v_measure = v_measure_score(new_df['indicator'], new_df['clusters'])\n",
        "print(\"The v_measure is:\",v_measure)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The v_measure is: 0.6226071463542034\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rLSDlqA66IOc",
        "outputId": "0a3454cd-f734-4d9f-ad0d-2d26c20ccd02"
      },
      "source": [
        "from sklearn.metrics.cluster import adjusted_rand_score\n",
        "randScore = adjusted_rand_score(new_df['indicator'], new_df['clusters'])\n",
        "print(\"The rand_score is:\",randScore)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The rand_score is: 0.6509110504747214\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fauRhyv8Tt1L",
        "outputId": "cf710753-2cd6-45ae-c251-e6f3168662c5"
      },
      "source": [
        "from sklearn.metrics import silhouette_score \n",
        "from sklearn.metrics import calinski_harabasz_score\n",
        "from sklearn.metrics import davies_bouldin_score\n",
        "\n",
        "score_kemans_s = silhouette_score(new_df, clusters, metric='euclidean')\n",
        "score_kemans_c = calinski_harabasz_score(new_df, clusters)\n",
        "score_kemans_d = davies_bouldin_score(new_df, clusters)\n",
        "print('Silhouette Score: %.4f' % score_kemans_s)\n",
        "print('Calinski Harabasz Score: %.4f' % score_kemans_c)\n",
        "print('Davies Bouldin Score: %.4f' % score_kemans_d)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Silhouette Score: 0.7032\n",
            "Calinski Harabasz Score: 184390.7946\n",
            "Davies Bouldin Score: 0.4429\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}