dataset_prep.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "eN1_3vP8fFPH"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JkSiFu51zrJF"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import Dataset, DataLoader"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ypnwITglj8Rm"
      },
      "source": [
        "# Application-specific Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1LBZ6H88oDqs"
      },
      "source": [
        "Train - Val Split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "PcvD1Z-Ul4A-"
      },
      "outputs": [],
      "source": [
        "# Accepts a data frame of run-to-failure instances from multiple similar machines\n",
        "from numpy.random import default_rng\n",
        "\n",
        "rng = default_rng()\n",
        "def train_val_split(df,val_percent):\n",
        "  all_instances = pd.unique(df['unit_id'])\n",
        "  num_val = int(np.floor(val_percent*len(all_instances)))\n",
        "  val_instances = (rng.choice(max(all_instances), size=num_val, replace=False))+1\n",
        "  # val_instances = np.random.randint(low=1, high=max(all_instances)+1, size=num_val)\n",
        "  train_instances = np.argwhere(np.isin(all_instances, val_instances, invert=True))+1\n",
        "\n",
        "  # Validation set\n",
        "  frames = []\n",
        "  for instance in val_instances:\n",
        "    z = df[df.unit_id == instance]\n",
        "    frames.append(z)\n",
        "  val_df = pd.concat(frames)\n",
        "\n",
        "  # Train set\n",
        "  frames = []\n",
        "  for instance in np.squeeze(train_instances):\n",
        "    z = df[df.unit_id == instance]\n",
        "    frames.append(z)\n",
        "  train_df = pd.concat(frames)\n",
        "\n",
        "  return train_df, val_df, train_instances, val_instances\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nvMx32yCkNid"
      },
      "source": [
        "Clustering\n",
        "\n",
        "Predict the cluster each sample in the dataset belongs to with pre-trained k-means clustering model.\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5VEF65tIwAd4"
      },
      "source": [
        "Dataframe"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "6_QO6480v_Iu"
      },
      "outputs": [],
      "source": [
        "# Create clusters with pre-trained k-means clustering model\n",
        "\n",
        "def kMeansClustering(kmeans_model, data, op_condts_labels):\n",
        "  # Let's predict the cluster each sample in the dataset belongs to\n",
        "  cluster_labels = kmeans_model.predict(data[op_condts_labels])\n",
        "  return cluster_labels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JnydWmwctO5i"
      },
      "source": [
        "Normalize by cluster\n",
        "\n",
        "\n",
        "1.   Find and store the mean and std of each cluster of the training dataset in a numpy array.\n",
        "1.   Normalize train, val and test datasets by clusters mean and std (Standardization)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "ERNo-7f2BuNR"
      },
      "outputs": [],
      "source": [
        "# Cluster parameters function\n",
        "no_clusters=6\n",
        "def parameters_form(no_clusters, data, cluster_labels):\n",
        "  parameters_mean_list = []\n",
        "  parameters_std_list = []\n",
        "  for label in range(no_clusters):\n",
        "    cluster = data[cluster_labels == label]\n",
        "    meaan = np.mean(cluster, axis = 0)\n",
        "    stdd = np.std(cluster, axis = 0)\n",
        "    parameters_mean_list.append(meaan)\n",
        "    parameters_std_list.append(stdd)\n",
        "  print(len(parameters_mean_list), len(parameters_std_list))\n",
        "  return parameters_mean_list, parameters_std_list\n",
        "\n",
        "# Normalize function\n",
        "def normalize(X,mean,std):\n",
        "  return (X - mean) / std\n",
        "\n",
        "# Normalized data\n",
        "def normalize_regime(temp_data, cluster_labels, parameters_mean_list, parameters_std_list):\n",
        "  # New Normalized with clusters dataset before split\n",
        "  normalized_unwrap_train_data = np.zeros_like(temp_data)\n",
        "  for clu in np.unique(cluster_labels):\n",
        "    mm = normalize(temp_data[np.argwhere(cluster_labels==clu)].squeeze(),parameters_mean_list[clu],parameters_std_list[clu])\n",
        "    # Fill newdataset @ every iteration\n",
        "    normalized_unwrap_train_data [np.argwhere(cluster_labels==clu).squeeze()] = mm\n",
        "  return normalized_unwrap_train_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AlBHr3rXkoy9"
      },
      "source": [
        "Feature extraction"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "L-0OvdJNm0AC"
      },
      "outputs": [],
      "source": [
        "# Accepts a numpy array and extracts desired columns.\n",
        "\n",
        "def featureExtraction(data, desired_sensors):\n",
        "  cols = desired_sensors+4\n",
        "  desired_cols = np.append(np.array([0]), cols)\n",
        "  data = data[:,[desired_cols]]\n",
        "  return data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q_W86zcFf3rf"
      },
      "source": [
        "Expanding Window: Varying Sequence Lenghts (Train and Val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "DWqDb-6K8_8v"
      },
      "outputs": [],
      "source": [
        "\"\"\"Build a list of numpy arrays with varying lengths.\n",
        "Args:\n",
        "    data: normalized dataset with relevant features\n",
        "        - X: numpy array of shape (T, feat_dim)\n",
        "Returns:\n",
        "    X: (len(list), varying lenght (T), feat_dim) list of arrays\n",
        "\"\"\"\n",
        "def expandWindow(data, min_len=5):\n",
        "  ensem_list = []\n",
        "  for i in np.unique(data[:,0]):\n",
        "    x_ens = data[data[:,0] == i]\n",
        "\n",
        "    # expanding window\n",
        "    # min_len = min_len\n",
        "    ens_len = x_ens.shape[0]\n",
        "    ens_max_len = ens_len - min_len + 1\n",
        "\n",
        "    start = 0\n",
        "    while start < ens_max_len:\n",
        "      ensem_list.append(x_ens[:start+min_len, 1:])\n",
        "      start+=1\n",
        "  return ensem_list"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Sliding Window: Varying Sequence Lenghts (Train and Val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def slidingWindow(data, T):\n",
        "  ensem_list = []\n",
        "  for i in np.unique(data[:,0]):\n",
        "    x_ens = data[data[:,0] == i]\n",
        "\n",
        "    # sliding window\n",
        "    ens_len = x_ens.shape[0]\n",
        "    ens_max_len = ens_len - T + 1\n",
        "\n",
        "    start = 0\n",
        "    while start < ens_max_len:\n",
        "      ensem_list.append(x_ens[start:start+T, 1:])\n",
        "      start+=1\n",
        "  return ensem_list\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Expanding and Sliding window processes for test dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def testExpandWindow(data):\n",
        "  x_test_list = []\n",
        "  for i in np.unique(data[:,0]):\n",
        "    dat = data[data[:,0]==i]\n",
        "    dat = dat[:,1:]\n",
        "    x_test_list.append(dat)\n",
        "  return x_test_list\n",
        "\n",
        "def testSlidingWindow(data, T):\n",
        "  x_test_list = []\n",
        "  for i in np.unique(data[:,0]):\n",
        "    dat = data[data[:,0]==i]\n",
        "    dat = dat[-T:,1:]\n",
        "    x_test_list.append(dat)\n",
        "  return x_test_list"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yyNDGd9nQGkW"
      },
      "source": [
        "Expanding Window - Y prep"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "n51RM2Rv0hdv"
      },
      "outputs": [],
      "source": [
        "# Y-prep min_len/T\n",
        "def yPrep(data, min_len):\n",
        "  y_list = []\n",
        "  for pe in np.unique(data[:,0]):\n",
        "    y_temp = data[data[:,0]==pe][min_len-1:,-1]\n",
        "    y_list.append(y_temp)\n",
        "  return y_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "0jTERtnG0hdv"
      },
      "outputs": [],
      "source": [
        "# total no of new dataset\n",
        "def checkY(y_list):\n",
        "  su = 0\n",
        "  for m in y_list:\n",
        "    su += len(m)\n",
        "  return su"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DXyL4Vj7kGtO"
      },
      "source": [
        "# General"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nQ1ltwPixCog"
      },
      "source": [
        "Unsupervised pre-training Dataset Class and functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "Gvy_9R_-uxIQ"
      },
      "outputs": [],
      "source": [
        "# It accepts a list of arrays of varying sequence lengths and targets\n",
        "class supDataset(Dataset):\n",
        "  def __init__(self, data_list, targets):\n",
        "    self.data_list = data_list\n",
        "    self.targets = targets\n",
        "\n",
        "  # Returns len of dataset\n",
        "  def __len__(self):\n",
        "    return len(self.data_list)\n",
        "\n",
        "  # Takes indices of data len, returns a dictionary of tensors\n",
        "  def __getitem__(self, idx):\n",
        "    X = self.data_list[idx]\n",
        "    y = self.targets[idx]\n",
        "    # return X, y\n",
        "    # return torch.tensor(X, dtype=torch.float),  torch.tensor(y, dtype=torch.int64)\n",
        "    return torch.tensor(X, dtype=torch.float), y\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "V1rMQUtMuxIQ"
      },
      "outputs": [],
      "source": [
        "def padding_mask(lengths, max_len=None):\n",
        "    \"\"\"\n",
        "    Used to mask padded positions: creates a (batch_size, max_len) boolean mask from a tensor of sequence lengths,\n",
        "    where 1 means the values at time step (t) are used to compute attention weights\n",
        "    \"\"\"\n",
        "    batch_size = lengths.numel()\n",
        "    max_len = max_len or lengths.max_val()  # trick works because of overloading of 'or' operator for non-boolean types\n",
        "    return (torch.arange(0, max_len, device=lengths.device)\n",
        "            .type_as(lengths)\n",
        "            .repeat(batch_size, 1)\n",
        "            .lt(lengths.unsqueeze(1)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def collate_superv(data, max_len=None):\n",
        "    \"\"\"Build mini-batch tensors from a list of (X, y) tuples.\n",
        "    Args:\n",
        "        data: len(batch_size) list of tuples (X, y).\n",
        "            - X: torch tensor of shape (seq_length, feat_dim); variable seq_length.\n",
        "            - y: torch tensor of shape (1);\n",
        "        max_len: global fixed sequence length. Used for architectures requiring fixed length input,\n",
        "            where the batch length cannot vary dynamically. Longer sequences are clipped, shorter are padded with 0s\n",
        "    Returns:\n",
        "        X: (batch_size, padded_length, feat_dim) torch tensor of masked features (input)\n",
        "        y: (batch_size,1)\n",
        "        padding_masks: (batch_size, padded_length) boolean tensor, 1 means keep vector at this position, 0 ignore (padding)\n",
        "    \"\"\"\n",
        "\n",
        "    batch_size = len(data)\n",
        "    features, targets = zip(*data)\n",
        "\n",
        "    # Stack and pad features and masks (convert 2D to 3D tensors, i.e. add batch dimension)\n",
        "    lengths = [X.shape[0] for X in features]  # original sequence length for each time series\n",
        "    if max_len is None:\n",
        "        max_len = max(lengths)\n",
        "\n",
        "    X = torch.zeros(batch_size, max_len, features[0].shape[-1])  # (batch_size, padded_length, feat_dim)\n",
        "\n",
        "    for i in range(batch_size):\n",
        "        end = min(lengths[i], max_len)\n",
        "        X[i, :end, :] = features[i][:end, :]\n",
        "\n",
        "    padding_masks = padding_mask(torch.tensor(lengths, dtype=torch.int16), max_len=max_len)  # (batch_size, padded_length) boolean tensor, \"1\" means keep\n",
        "    # X = x.clone().detach().requires_grad_(True)\n",
        "    X = X.clone().detach().requires_grad_(True).type('torch.FloatTensor')\n",
        "    targets = torch.tensor(targets, dtype=torch.float).reshape(-1,1)\n",
        "    return X, targets, padding_masks"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}