data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3a02475b-ce89-40a3-80fa-aadea842fb68",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from glob import glob\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from collections import Counter\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "06b1b448-7cb5-4dc1-b8de-0c4bd8b5cce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13ae712d-cbc3-4cbe-9ef9-a2c8d0d4eaf6",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# utils functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b3500689-d7a2-4238-afe5-65c33842cf91",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_anom_pairs(y):\n",
    "    anom_pairs = []\n",
    "    anom_index = np.where(y==1)[0]\n",
    "    tmp_seg = []\n",
    "    for i in anom_index:\n",
    "        tmp_seg.append(i)\n",
    "        if i + 1 not in anom_index:\n",
    "            anom_pairs.append((tmp_seg[0], tmp_seg[-1]))\n",
    "            tmp_seg = []\n",
    "    return anom_pairs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "607e3562-956d-472f-904c-e9d67d8c05d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def seqs_split(data, label, abnormal_class, seed=42, downsample_rate=1.0, pure=True):\n",
    "    # split into normal sequences and abnormal sequences \n",
    "    normal_data = []\n",
    "    abnormal_data = []\n",
    "    for d, l in zip(data, label):\n",
    "        if l not in abnormal_class:\n",
    "            normal_data.append(d)\n",
    "        else:\n",
    "            abnormal_data.append(d)\n",
    "\n",
    "    normal_data = np.array(normal_data)\n",
    "    abnormal_data = np.array(abnormal_data)\n",
    "\n",
    "    rng = np.random.RandomState(seed)\n",
    "    normal_data = normal_data[rng.permutation(len(normal_data))]\n",
    "\n",
    "    # downsample abnormal class\n",
    "    rng = np.random.RandomState(seed)\n",
    "    ds = int(len(abnormal_data) * downsample_rate)\n",
    "    abnormal_data = abnormal_data[rng.permutation(len(abnormal_data))]\n",
    "    abnormal_data = abnormal_data[:ds]\n",
    "\n",
    "    print(downsample_rate, normal_data.shape, abnormal_data.shape)\n",
    "\n",
    "    \n",
    "    # split train/test data\n",
    "    if pure:\n",
    "        split = int(0.6*len(normal_data))\n",
    "\n",
    "        rng = np.random.RandomState(seed)\n",
    "        data_train = normal_data[:split]\n",
    "        data_train = data_train[rng.permutation(len(data_train))]\n",
    "        label_train = np.zeros(len(data_train), dtype=int)\n",
    "\n",
    "        rng = np.random.RandomState(seed)\n",
    "        normal_data_test =normal_data[split:]\n",
    "        data_test = np.vstack([normal_data_test, abnormal_data])\n",
    "        label_test = np.hstack([np.zeros(len(normal_data_test), dtype=int), np.ones(len(abnormal_data), dtype=int)])\n",
    "        idx = rng.permutation(len(data_test))\n",
    "        data_test = data_test[idx]\n",
    "        label_test = label_test[idx]\n",
    "    \n",
    "    else:\n",
    "        split1 = int(0.6*len(normal_data))\n",
    "        split2 = int(0.6*len(abnormal_data))\n",
    "        \n",
    "        data_train1 = normal_data[:split1]\n",
    "        data_train2 = abnormal_data[:split2]\n",
    "        data_train = np.vstack([data_train1, data_train2])\n",
    "        label_train = np.hstack([np.zeros(len(data_train1), dtype=int), np.ones(len(data_train2), dtype=int)])\n",
    "        \n",
    "        rng = np.random.RandomState(seed)\n",
    "        idx = rng.permutation(len(data_train))\n",
    "        data_train = data_train[idx]\n",
    "        label_train = label_train[idx]\n",
    "\n",
    "        data_test1 = normal_data[split1:]\n",
    "        data_test2 = abnormal_data[split2:]\n",
    "        data_test = np.vstack([data_test1, data_test2])\n",
    "        label_test = np.hstack([np.zeros(len(data_test1), dtype=int), np.ones(len(data_test2), dtype=int)])\n",
    "        \n",
    "        rng = np.random.RandomState(seed)\n",
    "        idx = rng.permutation(len(data_test))\n",
    "        data_test = data_test[idx]\n",
    "        label_test = label_test[idx]\n",
    "\n",
    "    # reshape\n",
    "    dim = data_train.shape[-1]\n",
    "    seq_len = data_train.shape[1]\n",
    "    data_train = data_train.reshape(-1, dim)\n",
    "    label_train = label_train.repeat(seq_len)\n",
    "    data_test = data_test.reshape(-1, dim)\n",
    "    label_test = label_test.repeat(seq_len)\n",
    "\n",
    "    # save\n",
    "    df_train = pd.DataFrame(data_train, columns=['A'+str(i) for i in range(dim)])\n",
    "    df_train['label'] = label_train\n",
    "\n",
    "    df_test = pd.DataFrame(data_test, columns=['A'+str(i) for i in range(dim)])\n",
    "    df_test['label'] = label_test\n",
    "\n",
    "    print(Counter(df_train['label']), Counter(df_test['label']))\n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f68599a8-96c7-4c4e-8264-eff6e0eca273",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_path = 'data_processed/'\n",
    "os.makedirs(save_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d92c53e-b7cd-421e-8b09-2d9e54745d6e",
   "metadata": {},
   "source": [
    "# ASD"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb41d9d8-dd67-4493-b458-1ef63081d98c",
   "metadata": {},
   "source": [
    "download link: https://github.com/zhhlee/InterFusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "d9a48f12-cccb-40eb-81a9-1daa56d1fa91",
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(train_df, test_df, machine_idx, output_dir):\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    train_df.to_csv(os.path.join(output_dir, machine_idx + '_train.csv'))\n",
    "    test_df.to_csv(os.path.join(output_dir, machine_idx + '_test.csv'))\n",
    "def create_df(train, test, test_label):\n",
    "    col = ['A' + str(i) for i in range(train.shape[1])]\n",
    "    train_df = pd.DataFrame(train, columns=col)\n",
    "    test_df = pd.DataFrame(test, columns=col)\n",
    "\n",
    "    train_df['label'] = 0\n",
    "    test_df['label'] = test_label\n",
    "    return train_df, test_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "bbe672de-53e5-42c3-a233-4454d82e621f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_folder = os.path.join('data/', 'ASD/processed/')\n",
    "output_root_dir = 'data_processed/'\n",
    "dataset = 'ASD'\n",
    "\n",
    "\n",
    "full_lst = os.listdir(dataset_folder)\n",
    "machine_idx_lst = [a.split('_')[0] for a in full_lst]\n",
    "for machine_idx in sorted(machine_idx_lst):\n",
    "    train = pickle.load(open(dataset_folder + machine_idx + '_train.pkl', 'rb'))\n",
    "    test = pickle.load(open(dataset_folder + machine_idx + '_test.pkl', 'rb'))\n",
    "    test_label = pickle.load(open(dataset_folder + machine_idx + '_test_label.pkl', 'rb'))\n",
    "    train_df, test_df = create_df(train, test, test_label)\n",
    "\n",
    "    output_dir = os.path.join(output_root_dir, dataset, machine_idx)\n",
    "    save(train_df, test_df, machine_idx, output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c2bf962-923d-4ac5-b149-db765352008e",
   "metadata": {},
   "source": [
    "# SMD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "535eacb9-9c72-45a4-b309-6d9c563f16ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "machine-3-1.txt\n",
      "machine-3-11.txt\n",
      "machine-3-9.txt\n"
     ]
    }
   ],
   "source": [
    "dataset_folder = os.path.join('data/', 'SMD/')\n",
    "output_root_dir = 'data_processed/'\n",
    "dataset = 'SMD'\n",
    "\n",
    "# machine_lst = os.listdir(os.path.join(dataset_folder, 'train/'))\n",
    "machine_lst = ['machine-3-1.txt', 'machine-3-11.txt', 'machine-3-9.txt']\n",
    "for machine in sorted(machine_lst):\n",
    "    print(machine)\n",
    "    train = np.genfromtxt(os.path.join(dataset_folder, 'train', machine),\n",
    "                          dtype=np.float32, delimiter=',')\n",
    "    test = np.genfromtxt(os.path.join(dataset_folder, 'test', machine),\n",
    "                         dtype=np.float32, delimiter=',')\n",
    "    test_label = np.genfromtxt(os.path.join(dataset_folder, 'test_label', machine),\n",
    "                               dtype=np.float32, delimiter=',')\n",
    "    train_df, test_df = create_df(train, test, test_label)\n",
    "\n",
    "    machine_idx = os.path.splitext(machine)[0]\n",
    "    output_dir = os.path.join(output_root_dir, dataset, machine_idx)\n",
    "    save(train_df, test_df, machine_idx, output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bf98f07-0e90-4ff4-a8b2-af2a377d132e",
   "metadata": {},
   "source": [
    "# SWaT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "de4278a2-5849-4d09-a1b0-cd6d68807633",
   "metadata": {},
   "outputs": [],
   "source": [
    "# the following code is adapted from the source code in [Zhihan Li et al. KDD21]\n",
    "# preprocess for SWaT. SWaT.A2_Dec2015, version 0\n",
    "dataset_folder = os.path.join('data/', 'SWaT')\n",
    "\n",
    "test_df = pd.read_csv(os.path.join(dataset_folder, 'SWaT_Dataset_Attack_v0.csv'))\n",
    "\n",
    "test_df = test_df.set_index(' Timestamp')\n",
    "test_df['label'] = np.where(test_df['Normal/Attack'] == 'Attack', 1, 0)\n",
    "# test_df.apply(lambda x: 1 if test_df['Normal/Attack'] == 'Attack' else 0)\n",
    "test_df = test_df.drop('Normal/Attack', axis=1)\n",
    "assert test_df.shape == (449919, 52)\n",
    "\n",
    "train_df = pd.read_csv(os.path.join(dataset_folder, 'SWaT_Dataset_Normal_v0.csv'))\n",
    "# train_df = train_df.drop(columns=['Unnamed: 0', 'Unnamed: 52'])\n",
    "train_df = train_df.set_index(' Timestamp')\n",
    "train_df['label'] = np.where(train_df['Normal/Attack'] == 'Attack', 1, 0)\n",
    "train_df = train_df.drop('Normal/Attack', axis=1)\n",
    "\n",
    "# following [Zhihan Li et al. KDD21] & [Dan Li. ICANN. 2019]\n",
    "# fow SWaT data, due to the cold start of the system, starting point is 21600\n",
    "train_df = train_df.iloc[21600:]\n",
    "assert train_df.shape == (475200, 52)\n",
    "\n",
    "\n",
    "output_dir = 'data_processed/SWaT/'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "train_df.to_csv(os.path.join(output_dir, 'SWaT_train.csv'))\n",
    "test_df.to_csv(os.path.join(output_dir, 'SWaT_test.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a719c131-9d43-441a-a3cf-2075e865f514",
   "metadata": {},
   "source": [
    "note that some columne names in SWaT_test.csv contain spaces, please manually remove these spaces."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37401e3d-e374-4ca7-9edf-28fcf0ad1574",
   "metadata": {},
   "source": [
    "# WaQ"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3dd08f8d-a602-4588-b825-003cf5975811",
   "metadata": {},
   "source": [
    "download link:  https://www.spotseven.de/gecco/gecco-challenge/gecco-challenge-2018/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c779119e-fd05-4698-afd7-9e23c40e1382",
   "metadata": {},
   "source": [
    "use original train/test split  \n",
    "trian set is contaminated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "1919ffa0-05aa-496d-913b-3bfee6f81b27",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyreadr\n",
    "\n",
    "train = pyreadr.read_r('data/GECCO/water.RDS') # also works for RData\n",
    "df_train = train[None] # extract the pandas data frame \n",
    "\n",
    "test = pyreadr.read_r('data/GECCO/water_test.RDS')\n",
    "df_test = test[None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "2e0b0eea-26d8-4dc9-8b2e-0e1d9747fc1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(139566, 11) (139566, 11)\n"
     ]
    }
   ],
   "source": [
    "print(df_train.shape, df_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "a4f98779-4ce2-4dea-b009-bf970dd7f0f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['label'] = df_train['EVENT']  + 0\n",
    "df_train = df_train.drop(['EVENT', 'Time'], axis=1)\n",
    "df_train = df_train.dropna()\n",
    "\n",
    "df_test['label'] = df_test['EVENT']  + 0\n",
    "df_test = df_test.drop(['EVENT', 'Time'], axis=1)\n",
    "df_test = df_test.dropna()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "10d27701-fe76-4f65-ad2b-b50cca7c21cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_train[df_train.isna().sum(axis=1) !=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "197e2fd8-293a-4a18-bf84-a04db4f2a72b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1726, 2329)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['label'].sum(), df_test['label'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "af91855b-f91b-4815-b5a5-fe2e624b8907",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((138521, 10), (115086, 10))"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape, df_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "95b3e266-99dc-42d0-af31-5dd23028213f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tp</th>\n",
       "      <th>Cl</th>\n",
       "      <th>pH</th>\n",
       "      <th>Redox</th>\n",
       "      <th>Leit</th>\n",
       "      <th>Trueb</th>\n",
       "      <th>Cl_2</th>\n",
       "      <th>Fm</th>\n",
       "      <th>Fm_2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.5</td>\n",
       "      <td>0.17</td>\n",
       "      <td>8.36</td>\n",
       "      <td>749.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.118</td>\n",
       "      <td>1677.0</td>\n",
       "      <td>695.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.5</td>\n",
       "      <td>0.17</td>\n",
       "      <td>8.36</td>\n",
       "      <td>749.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.118</td>\n",
       "      <td>1561.0</td>\n",
       "      <td>696.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.5</td>\n",
       "      <td>0.17</td>\n",
       "      <td>8.35</td>\n",
       "      <td>749.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.117</td>\n",
       "      <td>1581.0</td>\n",
       "      <td>696.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.5</td>\n",
       "      <td>0.17</td>\n",
       "      <td>8.35</td>\n",
       "      <td>749.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.118</td>\n",
       "      <td>1579.0</td>\n",
       "      <td>693.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.5</td>\n",
       "      <td>0.17</td>\n",
       "      <td>8.35</td>\n",
       "      <td>749.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.118</td>\n",
       "      <td>1567.0</td>\n",
       "      <td>689.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Tp    Cl    pH  Redox   Leit  Trueb   Cl_2      Fm   Fm_2  label\n",
       "0  6.5  0.17  8.36  749.0  211.0  0.011  0.118  1677.0  695.0      0\n",
       "1  6.5  0.17  8.36  749.0  211.0  0.011  0.118  1561.0  696.0      0\n",
       "2  6.5  0.17  8.35  749.0  211.0  0.011  0.117  1581.0  696.0      0\n",
       "3  6.5  0.17  8.35  749.0  211.0  0.011  0.118  1579.0  693.0      0\n",
       "4  6.5  0.17  8.35  749.0  211.0  0.011  0.118  1567.0  689.0      0"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "786e946d-4cf6-43f6-b95d-4a3c18877443",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(os.path.join(save_path, f'WaQ/'), exist_ok=True)\n",
    "df_train.to_csv(os.path.join(save_path, f'WaQ/WaQ_train.csv'))\n",
    "df_test.to_csv(os.path.join(save_path, f'WaQ/WaQ_test.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e7c66ae-313f-431d-a324-a4a1898e1960",
   "metadata": {
    "tags": []
   },
   "source": [
    "# DSADS"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb200665-4670-484f-bbc3-3b0d40ebbb6e",
   "metadata": {},
   "source": [
    "this dataset can be downloaded from https://github.com/zhangyuxin621/AMSL"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76d9d4fb-cf68-4daf-a091-6dab055957dd",
   "metadata": {
    "tags": []
   },
   "source": [
    "1,sitting,\n",
    "\n",
    "2,standing,\n",
    "\n",
    "3,lying on back,\n",
    "\n",
    "4,lying on right side,\n",
    "\n",
    "5,ascending stairs,\n",
    "\n",
    "6,descending stairs,\n",
    "\n",
    "7,standing in an elevator still,\n",
    "\n",
    "8,moving around in an elevator,\n",
    "\n",
    "9,walking in a parking lot,\n",
    "\n",
    "10,walking on a treadmill with a speed of 4 kmh,\n",
    "\n",
    "11,walking in flat and 15 deg inclined positions,\n",
    "\n",
    "12,running on a treadmill with a speed of 8 kmh,\n",
    "\n",
    "13,exercising on a stepper,\n",
    "\n",
    "14,exercising on a cross trainer,\n",
    "\n",
    "15,cycling on an exercise bike in horizontal positions,\n",
    "\n",
    "16,cycling on an exercise bike in vertical positions,\n",
    "\n",
    "17,rowing,\n",
    "\n",
    "18,jumping,\n",
    "\n",
    "19,playing basketball"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee976048-65d4-41d2-98f8-745abf307457",
   "metadata": {},
   "source": [
    "use running, rowing, and jumping as anomalies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1ca9ece1-db09-4658-9727-57f37d21e713",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_person = 8\n",
    "persons = ['p' + str(i) for i in range(1, 9)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "49de2012-dc18-4fa3-9089-997d9c7d52fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = sorted([os.path.split(f)[1] for f in glob('data/DASADS/a*')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9e094dea-3304-4516-8a59-11b66c79919c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p1 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n",
      "p2 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n",
      "p3 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n",
      "p4 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n",
      "p5 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n",
      "p6 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n",
      "p7 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n",
      "p8 (1140, 125, 45) Counter({'a01': 60, 'a02': 60, 'a03': 60, 'a04': 60, 'a05': 60, 'a06': 60, 'a07': 60, 'a08': 60, 'a09': 60, 'a10': 60, 'a11': 60, 'a12': 60, 'a13': 60, 'a14': 60, 'a15': 60, 'a16': 60, 'a17': 60, 'a18': 60, 'a19': 60})\n"
     ]
    }
   ],
   "source": [
    "all_data = {}\n",
    "all_label = {}\n",
    "\n",
    "for p in persons:\n",
    "    data = []\n",
    "    label = []\n",
    "    for c in classes:\n",
    "        f_lst = glob(f'data/DASADS/{c}/{p}/*')\n",
    "        \n",
    "        seqs = []\n",
    "        for f in f_lst:\n",
    "            seq = np.loadtxt(f, delimiter= ',')\n",
    "            seqs.append(seq)\n",
    "        seqs = np.array(seqs)\n",
    "        data.extend(seqs)\n",
    "        label.extend([c] * len(seqs))\n",
    "    \n",
    "    data = np.array(data)\n",
    "    label = np.array(label)\n",
    "    \n",
    "    # idx = np.random.permutation(len(data))\n",
    "    # data = data[idx]\n",
    "    # label = label[idx]\n",
    "    \n",
    "    all_data[p] = data\n",
    "    all_label[p] = label\n",
    "    \n",
    "    print(p, data.shape, Counter(label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "c5bda863-7257-4ff2-9c07-7db36b47427e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(all_data, open('data/DASADS/all_data.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "760896cc-d953-4372-b620-93a15f742c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(all_label, open('data/DASADS/all_label.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "91c2d5c0-4b4f-4af5-b2ad-33180498bb38",
   "metadata": {},
   "outputs": [],
   "source": [
    "dim = all_data['p1'].shape[-1]\n",
    "seq_len = all_data['p1'].shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c98245f1-ad5d-4946-9675-e725ee277cc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "downsample_rate = 1.0\n",
    "# abnormal_class = ['a05', 'a06', 'a12', 'a17', 'a18']\n",
    "abnormal_class = ['a12', 'a17', 'a18']\n",
    "seed=42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c9912216-00ff-4466-be11-a63a86456b0b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000}) Counter({0: 48000, 1: 22500})\n"
     ]
    }
   ],
   "source": [
    "for p in persons:\n",
    "    # print(p)\n",
    "    data = all_data[p]\n",
    "    label = all_label[p]\n",
    "    df_train, df_test = seqs_split(data, label, abnormal_class=abnormal_class, seed=seed, downsample_rate=downsample_rate, pure=True)\n",
    "    # print(df_train.iloc[13])\n",
    "    \n",
    "    # os.makedirs(os.path.join(save_path, f'DASADS/{p}/'), exist_ok=True)    \n",
    "    # df_train.to_csv(os.path.join(save_path, f'DASADS/{p}/{p}_train.csv'))\n",
    "    # df_test.to_csv(os.path.join(save_path, f'DASADS/{p}/{p}_test.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "09777b2d-f698-4b26-a9fd-48a766895045",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p1\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n",
      "p2\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n",
      "p3\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n",
      "p4\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n",
      "p5\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n",
      "p6\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n",
      "p7\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n",
      "p8\n",
      "1.0 (960, 125, 45) (180, 125, 45)\n",
      "Counter({0: 72000, 1: 13500}) Counter({0: 48000, 1: 9000})\n"
     ]
    }
   ],
   "source": [
    "downsample_rate = 1.0\n",
    "# abnormal_class = ['a05', 'a06', 'a12', 'a17', 'a18']\n",
    "abnormal_class = ['a12', 'a17', 'a18']\n",
    "seed=42\n",
    "\n",
    "for p in persons:\n",
    "    print(p)\n",
    "    data = all_data[p]\n",
    "    label = all_label[p]\n",
    "    df_train, df_test = seqs_split(data, label, abnormal_class=abnormal_class, seed=seed, downsample_rate=downsample_rate, pure=False)\n",
    "    # print(df_train.iloc[13])\n",
    "    \n",
    "    os.makedirs(os.path.join(save_path, f'DSADS/{p}/'), exist_ok=True)    \n",
    "    df_train.to_csv(os.path.join(save_path, f'DSADS/{p}/{p}_train.csv'))\n",
    "    df_test.to_csv(os.path.join(save_path, f'DSADS/{p}/{p}_test.csv'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1612473-cb40-4ca5-aff2-69c8e4e56c0b",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Epilepsy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30c590b7-c45e-472c-8c0a-7b95f762796d",
   "metadata": {},
   "source": [
    "this is downloaded from the released repository of NeuTraL data  \n",
    "https://github.com/boschresearch/NeuTraL-AD/tree/NTL_full/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0085ffa4-1a7f-4e8d-87f5-717fd1cd20ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((137, 206, 3), (137,))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'data/'\n",
    "name = 'epilepsy/'\n",
    "train_x = np.load(path + name + 'train_array.npy')\n",
    "train_y = np.load(path + name + 'train_label.npy')\n",
    "test_x = np.load(path + name + 'test_array.npy')\n",
    "test_y = np.load(path + name + 'test_label.npy')\n",
    "train_x.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "5c29750d-acaa-489d-945c-81a0d4565bb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((275, 206, 3),\n",
       " (275,),\n",
       " Counter({'EPILEPSY': 68, 'WALKING': 74, 'RUNNING': 73, 'SAWING': 60}))"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = np.concatenate([train_x, test_x])\n",
    "label = np.hstack([train_y, test_y])\n",
    "data.shape, label.shape, Counter(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "56d90bbb-be87-4837-835d-9f6165365435",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0 (207, 206, 3) (68, 206, 3)\n",
      "Counter({0: 25544, 1: 8240}) Counter({0: 17098, 1: 5768})\n"
     ]
    }
   ],
   "source": [
    "# # contaminated training set\n",
    "seed=42\n",
    "abnormal_class = ['EPILEPSY']\n",
    "df_train, df_test = seqs_split(data, label, abnormal_class, seed=seed, downsample_rate=1.0, pure=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "2d6656d9-a858-42c7-a4fe-f5f6506da4dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(os.path.join(save_path, f'Epilepsy/'), exist_ok=True)    \n",
    "df_train.to_csv(os.path.join(save_path, f'Epilepsy/Epilepsy_train.csv'))\n",
    "df_test.to_csv(os.path.join(save_path, f'Epilepsy/Epilepsy_test.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "201b6b40-082b-4401-a143-1e6d49f2c2d2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cuda113",
   "language": "python",
   "name": "cuda113"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  },
  "toc-autonumbering": true
 },
 "nbformat": 4,
 "nbformat_minor": 5
}