CV19_Synthesis_Latest.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CV19_Synthesis_Latest.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ammarisme/covid-19/blob/master/CV19_Synthesis_Latest.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yQ-Ce_oZAeFm",
        "colab_type": "code",
        "outputId": "015045eb-8bfe-42b6-b14f-1b8c7dc76809",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns; sns.set()\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-o4WRgUnCC10",
        "colab_type": "code",
        "outputId": "226bdc27-b71b-41a6-ebc7-75265e1f9898",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "PATH = \"/content/drive/My Drive/covid\"\n",
        "INPUT_DIR = '/input_mix' #mean, std\n",
        "dataset_class = \"training\"\n",
        "source_file =PATH+ INPUT_DIR+'/validation/cleaned/cleaned_worldwide_cases_27_04.csv'\n",
        "TARGET_DIR = PATH+  INPUT_DIR+'/'+dataset_class+'/cleaned/'"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h1iuR4vUd4FX",
        "colab_type": "code",
        "outputId": "c6edf1a8-26a6-49ec-a099-1a97c1e0f748",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "original_data = pd.read_csv(source_file)\n",
        "original_data.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>cases</th>\n",
              "      <th>deaths</th>\n",
              "      <th>popData2018</th>\n",
              "      <th>countriesAndTerritories</th>\n",
              "      <th>geoId</th>\n",
              "      <th>countryterritoryCode</th>\n",
              "      <th>continentExp</th>\n",
              "      <th>_country_code</th>\n",
              "      <th>synthesis_seq</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>14439018</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>14439018</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>14439018</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>14439018</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>14439018</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   cases  deaths  popData2018  ...  continentExp  _country_code  synthesis_seq\n",
              "0      1       0     14439018  ...             1              1              0\n",
              "1      1       0     14439018  ...             1              1              0\n",
              "2      0       0     14439018  ...             1              1              0\n",
              "3      0       1     14439018  ...             1              1              0\n",
              "4      0       0     14439018  ...             1              1              0\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LZDce0cTTAzp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_synthetic_data(sample_dataset, window_mean, window_std, sythesize_ratio = 2, forced_reverse = False):\n",
        "    synthetic_data = pd.DataFrame(columns=sample_dataset.columns)\n",
        "    synthetic_data.insert(len(sample_dataset.columns), \"synthesis_seq\", [], True) \n",
        "\n",
        "    \n",
        "    for k in range(sythesize_ratio):\n",
        "      if len(synthetic_data) >= len(sample_dataset) * sythesize_ratio:\n",
        "        break;\n",
        "      #this loop generates a set that resembles the entire dataset\n",
        "      for country in sample_dataset['countryterritoryCode'].unique():\n",
        "        country_synthetic = pd.DataFrame(columns=synthetic_data.columns)\n",
        "        input_sequence_len = int(np.random.normal(window_mean, window_std))  #needs to be randomized\n",
        "\n",
        "        #population data change\n",
        "        country_data_i = sample_dataset[sample_dataset['countryterritoryCode'] == country]\n",
        "        if len(country_data_i) < input_sequence_len :\n",
        "          continue\n",
        "        feature_length = 2 #number of features to be randomized\n",
        "        country_data_array = np.array([country_data_i['cases'].to_numpy(), country_data_i['deaths'].to_numpy()])\n",
        "        country_data_array = country_data_array.reshape(feature_length,len(country_data_i))\n",
        "        x = country_data_array[:feature_length].T\n",
        "        \n",
        "        reversed = np.random.normal(0,1)>0\n",
        "        if reversed:\n",
        "          x = x[::-1]\n",
        "            \n",
        "        sets =0\n",
        "        x_list = []\n",
        "        dict_x = dict()\n",
        "        for i in range(input_sequence_len):\n",
        "          array_len = ((len(x) -i) - ((len(x)-i)%input_sequence_len))+i\n",
        "          if array_len <= 0:\n",
        "            continue\n",
        "          sets = int( array_len/ input_sequence_len)\n",
        "          if sets <= 0:\n",
        "            continue\n",
        "          #print('input seq : ', i , ' ', array_len , ' ',array_len-i , ' number of sets : ', sets)\n",
        "          x_temp = x[i:array_len].T.reshape(sets,feature_length,input_sequence_len)\n",
        "          uniq_keys = np.array([i+(input_sequence_len*k) for k in range(sets)])\n",
        "          x_temp = x_temp.reshape(feature_length,sets,input_sequence_len)\n",
        "          arrays_split = np.hsplit(x_temp,sets)\n",
        "          dict_x.update(dict(zip(uniq_keys, arrays_split)))\n",
        "\n",
        "        temp_x_list  = [dict_x[i].T for i in sorted(dict_x.keys())]        \n",
        "        temp_x_list = np.array(temp_x_list).squeeze()\n",
        "        feature_means = np.mean(temp_x_list, axis=1)\n",
        "        feature_std = np.std(temp_x_list, axis=1)\n",
        "        random_normal_feature_values = np.random.normal(feature_means, feature_std).T\n",
        "        random_normal_feature_values = np.round(random_normal_feature_values,0)\n",
        "        random_normal_feature_values[random_normal_feature_values < 0] = 0\n",
        "\n",
        "        if reversed:\n",
        "          random_normal_feature_values = random_normal_feature_values.T[::-1]\n",
        "          random_normal_feature_values = random_normal_feature_values.T\n",
        "\n",
        "        country_synthetic['cases'] = pd.Series(random_normal_feature_values[0])\n",
        "        country_synthetic['deaths'] = pd.Series(random_normal_feature_values[1])\n",
        "\n",
        "        \n",
        "        popData2018 = country_data_i['popData2018'].values[0]\n",
        "        population_change_percentage = 1+ (np.random.normal(0,1) / 100)\n",
        "        population_new = np.round(population_change_percentage*popData2018)\n",
        "\n",
        "        country_synthetic['popData2018'] = population_new\n",
        "        country_synthetic['_country_code'] = country_data_i['_country_code'].values[0]\n",
        "        country_synthetic['countriesAndTerritories'] = country_data_i['countriesAndTerritories'].values[0]\n",
        "        country_synthetic['countryterritoryCode'] = country_data_i['countryterritoryCode'].values[0]\n",
        "        country_synthetic['geoId'] = country_data_i['geoId'].values[0]\n",
        "        country_synthetic['continentExp'] = country_data_i['continentExp'].values[0]\n",
        "        country_synthetic['synthesis_seq'] = k\n",
        "        synthetic_data = synthetic_data.append(country_synthetic, ignore_index=True)\n",
        "    return synthetic_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_gPqH0O2nc2",
        "colab_type": "code",
        "outputId": "954716de-f0e8-4d86-a14e-4b0de55b35d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "run_id = 552022 #ddmyyhh\n",
        "for i in range(10):\n",
        "  window_mean = i+6\n",
        "  window_std= 1\n",
        "  synthetic_data  = generate_synthetic_data(original_data, window_mean = window_mean,  window_std= window_std, sythesize_ratio = 14)\n",
        "  synthetic_data.to_csv(TARGET_DIR + '/synthetic_run_'+str(i)+'_'+str(run_id)+'_dt_27_04'+'_wmean_'+str(window_mean)+'_wstd_'+str(window_std)+'.csv',  index=False )\n",
        "  print('synth step : ', i, ' len : ', len(synthetic_data))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "synth step :  0  len :  170608\n",
            "synth step :  1  len :  167677\n",
            "synth step :  2  len :  165013\n",
            "synth step :  3  len :  162152\n",
            "synth step :  4  len :  159277\n",
            "synth step :  5  len :  156423\n",
            "synth step :  6  len :  153683\n",
            "synth step :  7  len :  150793\n",
            "synth step :  8  len :  147907\n",
            "synth step :  9  len :  145099\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhdOTRKLt5qv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "4a88b973-4570-464a-b305-85d5506a76bb"
      },
      "source": [
        "import os\n",
        "print(os.listdir(TARGET_DIR))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['synthetic_run_0_552022_dt_27_04_wmean_6_wstd_1.csv', 'synthetic_run_1_552022_dt_27_04_wmean_7_wstd_1.csv', 'synthetic_run_2_552022_dt_27_04_wmean_8_wstd_1.csv', 'synthetic_run_3_552022_dt_27_04_wmean_9_wstd_1.csv', 'synthetic_run_4_552022_dt_27_04_wmean_10_wstd_1.csv', 'synthetic_run_5_552022_dt_27_04_wmean_11_wstd_1.csv', 'synthetic_run_6_552022_dt_27_04_wmean_12_wstd_1.csv', 'synthetic_run_7_552022_dt_27_04_wmean_13_wstd_1.csv', 'synthetic_run_8_552022_dt_27_04_wmean_14_wstd_1.csv', 'synthetic_run_9_552022_dt_27_04_wmean_15_wstd_1.csv']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBKDgh7kftRJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#for development purpose. To generate data, there is a cell below.\n",
        "for i in range(5):\n",
        "  synthetic_data = generate_synthetic_data(original_data, window_mean = 10, window_std = 2, sythesize_ratio = 1, forced_reverse= True)\n",
        "\n",
        "first_country  = [c for c in  original_data['countryterritoryCode'].unique()][3]\n",
        "plt1, = plt.plot(synthetic_data[synthetic_data['countryterritoryCode'] == first_country]['cases'].tolist())\n",
        "plt2, = plt.plot(original_data[original_data['countryterritoryCode'] == first_country]['cases'].tolist())\n",
        "plt.xlabel('number of days')\n",
        "plt.ylabel('confirmed case count')\n",
        "plt.title(first_country)\n",
        "\n",
        "plt.legend([plt1,plt2], ['Synthetic '+ str(len(synthetic_data[synthetic_data['countryterritoryCode'] == first_country]['cases']))\n",
        "                         ,'Real ' + str(len(original_data[original_data['countryterritoryCode'] == first_country]['cases']))])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RTd16sTrEhQF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#testing for accuracy\n",
        "def verify_synthetic_data(synthetic_data,original_data, print_results = False):\n",
        "  countryterritoryCode ='CN'\n",
        "  #test_synthetic_data = synthetic_data[synthetic_data['countryterritoryCode'] == countryterritoryCode]\n",
        "  test_synthetic_data = synthetic_data\n",
        "  control_set = original_data #[original_data['countryterritoryCode']==countryterritoryCode]\n",
        "  case_mean = np.mean(control_set['cases'].to_numpy()) - np.mean(test_synthetic_data['cases'].to_numpy())\n",
        "  death_mean = np.mean(control_set['deaths'].to_numpy()) - np.mean(test_synthetic_data['deaths'].to_numpy())\n",
        "  pop_mean = np.mean(control_set['popData2018'].to_numpy()) - np.mean(test_synthetic_data['popData2018'].to_numpy())\n",
        "\n",
        "  case_std = np.std(control_set['cases'].to_numpy()) - np.mean(test_synthetic_data['cases'].to_numpy())\n",
        "  death_std = np.std(control_set['deaths'].to_numpy()) - np.mean(test_synthetic_data['deaths'].to_numpy())\n",
        "  pop_std = np.std(control_set['popData2018'].to_numpy()) - np.mean(test_synthetic_data['popData2018'].to_numpy())\n",
        "\n",
        "  if print_results:\n",
        "    print(test_synthetic_data.head())\n",
        "    print(control_set.head())\n",
        "    print('original data length : ', len(control_set))\n",
        "    print('synthetic data length : ',len(test_synthetic_data))\n",
        "    print('min sequence length : ',min([len(test_synthetic_data[test_synthetic_data['countryterritoryCode']==country]) for country in test_synthetic_data['countryterritoryCode'].unique()]))\n",
        "    print('max sequence length : ',max([len(test_synthetic_data[test_synthetic_data['countryterritoryCode']==country]) for country in test_synthetic_data['countryterritoryCode'].unique()]))\n",
        "\n",
        "    print('Mean diff. cases : ', case_mean)\n",
        "    print('Mean diff. deaths : ', death_mean)\n",
        "    print('Mean diff. population : ', pop_mean)\n",
        "  \n",
        "  return  [case_mean, death_mean, pop_mean, case_std , death_std , pop_std ]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JovUZAh6IRXw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\"\"\"\n",
        "#for development\n",
        "results = []\n",
        "for i in range(5,15):\n",
        "   synthetic_data = generate_synthetic_data(original_data, window_mean = i, window_std = 1, sythesize_ratio = 1)\n",
        "   results.append(verify_synthetic_data(synthetic_data,original_data))\n",
        "   plt.plot(np.array(results).T[0])\n",
        "plt.plot(np.array(results).T[1])\n",
        "\"\"\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hz965wRiksO2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qum_ts4-nLtg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "run_id = 2842001 #ddmyyhh\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h0ziv32DCx7O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}