wavenet_pipeline.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.12",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "nvidiaTeslaT4",
      "dataSources": [
        {
          "sourceId": 59093,
          "databundleVersionId": 7469972,
          "sourceType": "competition"
        },
        {
          "sourceId": 7392733,
          "sourceType": "datasetVersion",
          "datasetId": 4297749
        },
        {
          "sourceId": 7465251,
          "sourceType": "datasetVersion",
          "datasetId": 4317718
        }
      ],
      "dockerImageVersionId": 30636,
      "isInternetEnabled": false,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "This notebook is the Modified version by us of a starter notebook by Chris Deotte.    \n",
        "https://www.kaggle.com/code/cdeotte/efficientnetb0-starter-lb-0-43\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "qtJS6KqAOtfn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# WaveNet notebook using RAW EEG Features!\n",
        "This notebook our WaveNet training .\n",
        "We sed 8 features grouped as 4 chains. Downsample time 5x .     \n",
        "\n",
        "This is our model architecture .   \n",
        "![](https://raw.githubusercontent.com/cdeotte/Kaggle_Images/main/Jan-2024/wave-model.png)\n",
        "\n",
        "Adding training via channel flipping leads to **CV 0.65 LB 0.45**\n",
        "and the addition of two step training **CV 0.79 LB 0.37**"
      ],
      "metadata": {
        "id": "1dhPbrxiOKXd"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load Train Data"
      ],
      "metadata": {
        "id": "rJ5L0fidOKXm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd, numpy as np, os\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "train = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n",
        "print( train.shape )\n",
        "display( train.head() )\n",
        "\n",
        "# CHOICE TO CREATE OR LOAD EEGS FROM NOTEBOOK VERSION 1\n",
        "CREATE_EEGS = False\n",
        "TRAIN_MODEL = True"
      ],
      "metadata": {
        "trusted": true,
        "id": "yU43FxomOKXm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Raw EEG Features"
      ],
      "metadata": {
        "id": "s9fJLCilOKXm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_parquet('/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/1000913311.parquet')\n",
        "FEATS = df.columns\n",
        "print(f'There are {len(FEATS)} raw eeg features')\n",
        "print( list(FEATS) )"
      ],
      "metadata": {
        "trusted": true,
        "id": "YuW6SY0YOKXn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('We will use the following subset of raw EEG features:')\n",
        "FEATS = ['Fp1','T3','C3','O1','Fp2','C4','T4','O2']\n",
        "FEAT2IDX = {x:y for x,y in zip(FEATS,range(len(FEATS)))}\n",
        "print( list(FEATS) )"
      ],
      "metadata": {
        "trusted": true,
        "id": "vYgarxy4OKXn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def eeg_from_parquet(parquet_path, display=False):\n",
        "\n",
        "    # EXTRACT MIDDLE 50 SECONDS\n",
        "    eeg = pd.read_parquet(parquet_path, columns=FEATS)\n",
        "    rows = len(eeg)\n",
        "    offset = (rows-10_000)//2\n",
        "    eeg = eeg.iloc[offset:offset+10_000]\n",
        "\n",
        "    if display:\n",
        "        plt.figure(figsize=(10,5))\n",
        "        offset = 0\n",
        "\n",
        "    # CONVERT TO NUMPY\n",
        "    data = np.zeros((10_000,len(FEATS)))\n",
        "    for j,col in enumerate(FEATS):\n",
        "\n",
        "        # FILL NAN\n",
        "        x = eeg[col].values.astype('float32')\n",
        "        m = np.nanmean(x)\n",
        "        if np.isnan(x).mean()<1: x = np.nan_to_num(x,nan=m)\n",
        "        else: x[:] = 0\n",
        "\n",
        "        data[:,j] = x\n",
        "\n",
        "        if display:\n",
        "            if j!=0: offset += x.max()\n",
        "            plt.plot(range(10_000),x-offset,label=col)\n",
        "            offset -= x.min()\n",
        "\n",
        "    if display:\n",
        "        plt.legend()\n",
        "        name = parquet_path.split('/')[-1]\n",
        "        name = name.split('.')[0]\n",
        "        plt.title(f'EEG {name}',size=16)\n",
        "        plt.show()\n",
        "\n",
        "    return data"
      ],
      "metadata": {
        "trusted": true,
        "id": "5STq1HuHOKXn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "\n",
        "all_eegs = {}\n",
        "DISPLAY = 4\n",
        "EEG_IDS = train.eeg_id.unique()\n",
        "PATH = '/kaggle/input/hms-harmful-brain-activity-classification/train_eegs/'\n",
        "\n",
        "for i,eeg_id in enumerate(EEG_IDS):\n",
        "    if (i%100==0)&(i!=0): print(i,', ',end='')\n",
        "\n",
        "    # SAVE EEG TO PYTHON DICTIONARY OF NUMPY ARRAYS\n",
        "    data = eeg_from_parquet(f'{PATH}{eeg_id}.parquet', display=i<DISPLAY)\n",
        "    all_eegs[eeg_id] = data\n",
        "\n",
        "    if i==DISPLAY:\n",
        "        if CREATE_EEGS:\n",
        "            print(f'Processing {train.eeg_id.nunique()} eeg parquets... ',end='')\n",
        "        else:\n",
        "            print(f'Reading {len(EEG_IDS)} eeg NumPys from disk.')\n",
        "            break\n",
        "\n",
        "if CREATE_EEGS:\n",
        "    np.save('eegs',all_eegs)\n",
        "else:\n",
        "    all_eegs = np.load('/kaggle/input/brain-eegs/eegs.npy',allow_pickle=True).item()"
      ],
      "metadata": {
        "trusted": true,
        "id": "q6XpaAv7OKXn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Deduplicate Train EEG Id"
      ],
      "metadata": {
        "id": "3OEnYEbUOKXn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# LOAD TRAIN\n",
        "df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n",
        "TARGETS = df.columns[-6:]\n",
        "TARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\n",
        "TARS2 = {x:y for y,x in TARS.items()}\n",
        "\n",
        "train = df.groupby('eeg_id')[['patient_id']].agg('first')\n",
        "\n",
        "tmp = df.groupby('eeg_id')[TARGETS].agg('sum')\n",
        "for t in TARGETS:\n",
        "    train[t] = tmp[t].values\n",
        "\n",
        "y_data = train[TARGETS].values\n",
        "y_data = y_data / y_data.sum(axis=1,keepdims=True)\n",
        "train[TARGETS] = y_data\n",
        "\n",
        "tmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\n",
        "train['target'] = tmp\n",
        "\n",
        "train = train.reset_index()\n",
        "train = train.loc[train.eeg_id.isin(EEG_IDS)]\n",
        "print('Train Data with unique eeg_id shape:', train.shape )\n",
        "train.head()"
      ],
      "metadata": {
        "trusted": true,
        "id": "5gMtvGHIOKXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Butter Low-Pass Filter"
      ],
      "metadata": {
        "id": "oOyfqsUUOKXo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy.signal import butter, lfilter\n",
        "\n",
        "def butter_lowpass_filter(data, cutoff_freq=20, sampling_rate=200, order=4):\n",
        "    nyquist = 0.5 * sampling_rate\n",
        "    normal_cutoff = cutoff_freq / nyquist\n",
        "    b, a = butter(order, normal_cutoff, btype='low', analog=False)\n",
        "    filtered_data = lfilter(b, a, data, axis=0)\n",
        "    return filtered_data"
      ],
      "metadata": {
        "trusted": true,
        "id": "W_sqnyatOKXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "FREQS = [1,2,4,8,16][::-1]\n",
        "x = [all_eegs[EEG_IDS[0]][:,0]]\n",
        "for k in FREQS:\n",
        "    x.append( butter_lowpass_filter(x[0], cutoff_freq=k) )\n",
        "\n",
        "plt.figure(figsize=(20,20))\n",
        "plt.plot(range(10_000),x[0], label='without filter')\n",
        "for k in range(1,len(x)):\n",
        "    plt.plot(range(10_000),x[k]-k*(x[0].max()-x[0].min()), label=f'with filter {FREQS[k-1]}Hz')\n",
        "plt.legend()\n",
        "plt.title('Butter Low-Pass Filter Examples',size=18)\n",
        "plt.show()"
      ],
      "metadata": {
        "trusted": true,
        "id": "9rsbO219OKXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Loader with Butter Low-Pass Filter"
      ],
      "metadata": {
        "id": "6ClPAlEJOKXo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "class DataGenerator(tf.keras.utils.Sequence):\n",
        "    'Generates data for Keras'\n",
        "    def __init__(self, data, batch_size=16, shuffle=False, eegs=all_eegs, mode='train',\n",
        "                 downsample=5):\n",
        "\n",
        "        self.data = data\n",
        "        self.batch_size = batch_size\n",
        "        self.shuffle = shuffle\n",
        "        self.eegs = eegs\n",
        "        self.mode = mode\n",
        "        self.downsample = downsample\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        'Denotes the number of batches per epoch'\n",
        "        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n",
        "        return ct\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        'Generate one batch of data'\n",
        "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
        "        X, y = self.__data_generation(indexes)\n",
        "        return X[:,::self.downsample,:], y\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        'Updates indexes after each epoch'\n",
        "        self.indexes = np.arange( len(self.data) )\n",
        "        if self.shuffle: np.random.shuffle(self.indexes)\n",
        "\n",
        "    def __data_generation(self, indexes):\n",
        "        'Generates data containing batch_size samples'\n",
        "\n",
        "        X = np.zeros((len(indexes),10_000,8),dtype='float32')\n",
        "        y = np.zeros((len(indexes),6),dtype='float32')\n",
        "\n",
        "        sample = np.zeros((10_000,X.shape[-1]))\n",
        "        for j,i in enumerate(indexes):\n",
        "            row = self.data.iloc[i]\n",
        "            data = self.eegs[row.eeg_id]\n",
        "\n",
        "            # FEATURE ENGINEER\n",
        "            sample[:,0] = data[:,FEAT2IDX['Fp1']] - data[:,FEAT2IDX['T3']]\n",
        "            sample[:,1] = data[:,FEAT2IDX['T3']] - data[:,FEAT2IDX['O1']]\n",
        "\n",
        "            sample[:,2] = data[:,FEAT2IDX['Fp1']] - data[:,FEAT2IDX['C3']]\n",
        "            sample[:,3] = data[:,FEAT2IDX['C3']] - data[:,FEAT2IDX['O1']]\n",
        "\n",
        "            sample[:,4] = data[:,FEAT2IDX['Fp2']] - data[:,FEAT2IDX['C4']]\n",
        "            sample[:,5] = data[:,FEAT2IDX['C4']] - data[:,FEAT2IDX['O2']]\n",
        "\n",
        "            sample[:,6] = data[:,FEAT2IDX['Fp2']] - data[:,FEAT2IDX['T4']]\n",
        "            sample[:,7] = data[:,FEAT2IDX['T4']] - data[:,FEAT2IDX['O2']]\n",
        "\n",
        "            # STANDARDIZE\n",
        "            sample = np.clip(sample,-1024,1024)\n",
        "            sample = np.nan_to_num(sample, nan=0) / 32.0\n",
        "\n",
        "            # BUTTER LOW-PASS FILTER\n",
        "            sample = butter_lowpass_filter(sample)\n",
        "\n",
        "            X[j,] = sample\n",
        "            if self.mode!='test':\n",
        "                y[j] = row[TARGETS]\n",
        "\n",
        "        return X,y"
      ],
      "metadata": {
        "trusted": true,
        "id": "IVyFsyUiOKXo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Display Data Loader"
      ],
      "metadata": {
        "id": "9MqZV8usOKXp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gen = DataGenerator(train, shuffle=False)\n",
        "\n",
        "for x,y in gen:\n",
        "    for k in range(4):\n",
        "        plt.figure(figsize=(20,4))\n",
        "        offset = 0\n",
        "        for j in range(x.shape[-1]):\n",
        "            if j!=0: offset -= x[k,:,j].min()\n",
        "            plt.plot(range(2_000),x[k,:,j]+offset,label=f'feature {j+1}')\n",
        "            offset += x[k,:,j].max()\n",
        "        tt = f'{y[k][0]:0.1f}'\n",
        "        for t in y[k][1:]:\n",
        "            tt += f', {t:0.1f}'\n",
        "        plt.title(f'EEG_Id = {EEG_IDS[k]}\\nTarget = {tt}',size=14)\n",
        "        plt.legend()\n",
        "        plt.show()\n",
        "    break"
      ],
      "metadata": {
        "trusted": true,
        "id": "y_qLl0E9OKXp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Initialize GPUs"
      ],
      "metadata": {
        "id": "eM-tuM9bOKXp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0,1\"\n",
        "import tensorflow as tf\n",
        "print('TensorFlow version =',tf.__version__)\n",
        "\n",
        "# USE MULTIPLE GPUS\n",
        "gpus = tf.config.list_physical_devices('GPU')\n",
        "if len(gpus)<=1:\n",
        "    strategy = tf.distribute.OneDeviceStrategy(device=\"/gpu:0\")\n",
        "    print(f'Using {len(gpus)} GPU')\n",
        "else:\n",
        "    strategy = tf.distribute.MirroredStrategy()\n",
        "    print(f'Using {len(gpus)} GPUs')"
      ],
      "metadata": {
        "trusted": true,
        "id": "Je3nzsycOKXp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# USE MIXED PRECISION\n",
        "MIX = True\n",
        "if MIX:\n",
        "    tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})\n",
        "    print('Mixed precision enabled')\n",
        "else:\n",
        "    print('Using full precision')"
      ],
      "metadata": {
        "trusted": true,
        "id": "UaxHiDhyOKXp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Build WaveNet Model"
      ],
      "metadata": {
        "id": "R0pkfSjgOKXp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# TRAIN SCHEDULE\n",
        "def lrfn(epoch):\n",
        "        return [1e-3,1e-3,1e-3,1e-4,1e-5][epoch]\n",
        "LR = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n",
        "EPOCHS = 5"
      ],
      "metadata": {
        "trusted": true,
        "id": "3pOHiFyjOKXp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.layers import Input, Dense, Multiply, Add, Conv1D, Concatenate\n",
        "\n",
        "def wave_block(x, filters, kernel_size, n):\n",
        "    dilation_rates = [2**i for i in range(n)]\n",
        "    x = Conv1D(filters = filters,\n",
        "               kernel_size = 1,\n",
        "               padding = 'same')(x)\n",
        "    res_x = x\n",
        "    for dilation_rate in dilation_rates:\n",
        "        tanh_out = Conv1D(filters = filters,\n",
        "                          kernel_size = kernel_size,\n",
        "                          padding = 'same',\n",
        "                          activation = 'tanh',\n",
        "                          dilation_rate = dilation_rate)(x)\n",
        "        sigm_out = Conv1D(filters = filters,\n",
        "                          kernel_size = kernel_size,\n",
        "                          padding = 'same',\n",
        "                          activation = 'sigmoid',\n",
        "                          dilation_rate = dilation_rate)(x)\n",
        "        x = Multiply()([tanh_out, sigm_out])\n",
        "        x = Conv1D(filters = filters,\n",
        "                   kernel_size = 1,\n",
        "                   padding = 'same')(x)\n",
        "        res_x = Add()([res_x, x])\n",
        "    return res_x"
      ],
      "metadata": {
        "trusted": true,
        "id": "uDlo3nBFOKXq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "![](https://raw.githubusercontent.com/cdeotte/Kaggle_Images/main/Jan-2024/wave-model.png)"
      ],
      "metadata": {
        "id": "Rd_FliYUOKXq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def build_model():\n",
        "\n",
        "    # INPUT\n",
        "    inp = tf.keras.Input(shape=(2_000,8))\n",
        "\n",
        "    ############\n",
        "    # FEATURE EXTRACTION SUB MODEL\n",
        "    inp2 = tf.keras.Input(shape=(2_000,1))\n",
        "    x = wave_block(inp2, 8, 4, 6)\n",
        "    x = wave_block(x, 16, 4, 6)\n",
        "    x = wave_block(x, 32, 4, 6)\n",
        "    x = wave_block(x, 64, 4, 6)\n",
        "    model2 = tf.keras.Model(inputs=inp2, outputs=x)\n",
        "    ###########\n",
        "\n",
        "    # LEFT TEMPORAL CHAIN\n",
        "    x1 = model2(inp[:,:,0:1])\n",
        "    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n",
        "    x2 = model2(inp[:,:,1:2])\n",
        "    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n",
        "    z1 = tf.keras.layers.Average()([x1,x2])\n",
        "\n",
        "    # LEFT PARASAGITTAL CHAIN\n",
        "    x1 = model2(inp[:,:,2:3])\n",
        "    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n",
        "    x2 = model2(inp[:,:,3:4])\n",
        "    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n",
        "    z2 = tf.keras.layers.Average()([x1,x2])\n",
        "\n",
        "    # RIGHT PARASAGITTAL CHAIN\n",
        "    x1 = model2(inp[:,:,4:5])\n",
        "    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n",
        "    x2 = model2(inp[:,:,5:6])\n",
        "    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n",
        "    z3 = tf.keras.layers.Average()([x1,x2])\n",
        "\n",
        "    # RIGHT TEMPORAL CHAIN\n",
        "    x1 = model2(inp[:,:,6:7])\n",
        "    x1 = tf.keras.layers.GlobalAveragePooling1D()(x1)\n",
        "    x2 = model2(inp[:,:,7:8])\n",
        "    x2 = tf.keras.layers.GlobalAveragePooling1D()(x2)\n",
        "    z4 = tf.keras.layers.Average()([x1,x2])\n",
        "\n",
        "    # COMBINE CHAINS\n",
        "    y = tf.keras.layers.Concatenate()([z1,z2,z3,z4])\n",
        "    y = tf.keras.layers.Dense(64, activation='relu')(y)\n",
        "    y = tf.keras.layers.Dense(6,activation='softmax', dtype='float32')(y)\n",
        "\n",
        "    # COMPILE MODEL\n",
        "    model = tf.keras.Model(inputs=inp, outputs=y)\n",
        "    opt = tf.keras.optimizers.Adam(learning_rate = 1e-3)\n",
        "    loss = tf.keras.losses.KLDivergence()\n",
        "    model.compile(loss=loss, optimizer = opt)\n",
        "\n",
        "    return model"
      ],
      "metadata": {
        "trusted": true,
        "id": "g2WvE_krOKXq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train Group KFold"
      ],
      "metadata": {
        "id": "8hz9cGHAOKXq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "VERBOSE = 1\n",
        "FOLDS_TO_TRAIN = 5\n",
        "if not os.path.exists('WaveNet_Model'):\n",
        "    os.makedirs('WaveNet_Model')\n",
        "\n",
        "from sklearn.model_selection import KFold, GroupKFold\n",
        "import tensorflow.keras.backend as K, gc\n",
        "\n",
        "all_oof = []; all_oof2 = []; all_true = []\n",
        "gkf = GroupKFold(n_splits=5)\n",
        "for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):\n",
        "\n",
        "    print('#'*25)\n",
        "    print(f'### Fold {i+1}')\n",
        "    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=16)\n",
        "    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n",
        "    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n",
        "    print('#'*25)\n",
        "\n",
        "    # TRAIN MODEL\n",
        "    K.clear_session()\n",
        "    with strategy.scope():\n",
        "        model = build_model()\n",
        "    if TRAIN_MODEL:\n",
        "        model.fit(train_gen, verbose=VERBOSE,\n",
        "              validation_data = valid_gen,\n",
        "              epochs=EPOCHS, callbacks = [LR])\n",
        "        model.save_weights(f'WaveNet_Model/WaveNet_fold{i}.h5')\n",
        "\n",
        "    # WAVENET OOF\n",
        "    oof = model.predict(valid_gen, verbose=VERBOSE)\n",
        "    all_oof.append(oof)\n",
        "    all_true.append(train.iloc[valid_index][TARGETS].values)\n",
        "\n",
        "    # TRAIN MEAN OOF\n",
        "    y_train = train.iloc[train_index][TARGETS].values\n",
        "    y_valid = train.iloc[valid_index][TARGETS].values\n",
        "    oof = y_valid.copy()\n",
        "    for j in range(6):\n",
        "        oof[:,j] = y_train[:,j].mean()\n",
        "    oof = oof / oof.sum(axis=1,keepdims=True)\n",
        "    all_oof2.append(oof)\n",
        "\n",
        "    del model, oof, y_train, y_valid\n",
        "    gc.collect()\n",
        "\n",
        "    if i==FOLDS_TO_TRAIN-1: break\n",
        "\n",
        "all_oof = np.concatenate(all_oof)\n",
        "all_oof2 = np.concatenate(all_oof2)\n",
        "all_true = np.concatenate(all_true)"
      ],
      "metadata": {
        "trusted": true,
        "id": "F8y4ylnfOKXq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# CV Score for WaveNet"
      ],
      "metadata": {
        "id": "4nmTqlJKOKXq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append('/kaggle/input/kaggle-kl-div')\n",
        "from kaggle_kl_div import score\n",
        "\n",
        "oof = pd.DataFrame(all_oof.copy())\n",
        "oof['id'] = np.arange(len(oof))\n",
        "\n",
        "true = pd.DataFrame(all_true.copy())\n",
        "true['id'] = np.arange(len(true))\n",
        "\n",
        "cv = score(solution=true, submission=oof, row_id_column_name='id')\n",
        "print('CV Score with WaveNet Raw EEG =',cv)"
      ],
      "metadata": {
        "trusted": true,
        "id": "kQDAV9SWOKXq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# CV Score using Train Means"
      ],
      "metadata": {
        "id": "d9LxCWVIOKXr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "oof = pd.DataFrame(all_oof2.copy())\n",
        "oof['id'] = np.arange(len(oof))\n",
        "\n",
        "true = pd.DataFrame(all_true.copy())\n",
        "true['id'] = np.arange(len(true))\n",
        "\n",
        "cv = score(solution=true, submission=oof, row_id_column_name='id')\n",
        "print('CV Score with Train Means =',cv)"
      ],
      "metadata": {
        "trusted": true,
        "id": "FDVjIjmaOKXr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training with Flipped Channel"
      ],
      "metadata": {
        "id": "aedtaYQzOKXr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class DataGenerator(tf.keras.utils.Sequence):\n",
        "    'Generates data for Keras'\n",
        "    def __init__(self, data, batch_size=16, shuffle=False, eegs=all_eegs, mode='train',\n",
        "                 downsample=5):\n",
        "\n",
        "        self.data = data\n",
        "        self.batch_size = batch_size\n",
        "        self.shuffle = shuffle\n",
        "        self.eegs = eegs\n",
        "        self.mode = mode\n",
        "        self.downsample = downsample\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        'Denotes the number of batches per epoch'\n",
        "        ct = int( np.ceil( len(self.data) / self.batch_size ) )\n",
        "        return ct\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        'Generate one batch of data'\n",
        "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
        "        X, y = self.__data_generation(indexes)\n",
        "        return X[:,::self.downsample,:], y\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        'Updates indexes after each epoch'\n",
        "        self.indexes = np.arange( len(self.data) )\n",
        "        if self.shuffle: np.random.shuffle(self.indexes)\n",
        "\n",
        "    def __data_generation(self, indexes):\n",
        "        'Generates data containing batch_size samples'\n",
        "\n",
        "        X = np.zeros((len(indexes),10_000,8),dtype='float32')\n",
        "        y = np.zeros((len(indexes),6),dtype='float32')\n",
        "\n",
        "        sample = np.zeros((10_000,X.shape[-1]))\n",
        "        sample1 = np.zeros((10_000,X.shape[-1]))\n",
        "        for j,i in enumerate(indexes):\n",
        "            row = self.data.iloc[i]\n",
        "            data = self.eegs[row.eeg_id]\n",
        "\n",
        "            # FEATURE ENGINEER\n",
        "            sample[:,0] = data[:,FEAT2IDX['Fp1']] - data[:,FEAT2IDX['T3']]\n",
        "            sample[:,1] = data[:,FEAT2IDX['T3']] - data[:,FEAT2IDX['O1']]\n",
        "\n",
        "            sample[:,2] = data[:,FEAT2IDX['Fp1']] - data[:,FEAT2IDX['C3']]\n",
        "            sample[:,3] = data[:,FEAT2IDX['C3']] - data[:,FEAT2IDX['O1']]\n",
        "\n",
        "            sample[:,4] = data[:,FEAT2IDX['Fp2']] - data[:,FEAT2IDX['C4']]\n",
        "            sample[:,5] = data[:,FEAT2IDX['C4']] - data[:,FEAT2IDX['O2']]\n",
        "\n",
        "            sample[:,6] = data[:,FEAT2IDX['Fp2']] - data[:,FEAT2IDX['T4']]\n",
        "            sample[:,7] = data[:,FEAT2IDX['T4']] - data[:,FEAT2IDX['O2']]\n",
        "\n",
        "            # STANDARDIZE\n",
        "            sample = np.clip(sample,-1024,1024)\n",
        "            sample = np.nan_to_num(sample, nan=0) / 32.0\n",
        "\n",
        "\n",
        "            # BUTTER LOW-PASS FILTER\n",
        "            sample = butter_lowpass_filter(sample)\n",
        "\n",
        "            sample1[:,0] = sample[:,6]\n",
        "            sample1[:,1] = sample[:,7]\n",
        "            sample1[:,2] = sample[:,4]\n",
        "            sample1[:,3] = sample[:,5]\n",
        "            sample1[:,4] = sample[:,2]\n",
        "            sample1[:,5] = sample[:,3]\n",
        "            sample1[:,6] = sample[:,0]\n",
        "            sample1[:,7] = sample[:,1]\n",
        "            X[j,] = sample1\n",
        "\n",
        "            if self.mode!='test':\n",
        "                y[j] = row[TARGETS]\n",
        "\n",
        "        return X,y"
      ],
      "metadata": {
        "id": "OTcfwHlSOKXr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def tranform_tta(x):\n",
        "    x_1 = np.zeros((x.shape[0],x.shape[1]))\n",
        "    x_1[:,0] = x[:,6]\n",
        "    x_1[:,1] = x[:,7]\n",
        "    x_1[:,2] = x[:,4]\n",
        "    x_1[:,3] = x[:,5]\n",
        "    x_1[:,4] = x[:,2]\n",
        "    x_1[:,5] = x[:,3]\n",
        "    x_1[:,6] = x[:,0]\n",
        "    x_1[:,7] = x[:,1]\n",
        "    return x_1"
      ],
      "metadata": {
        "id": "PY7_xoeGOKXr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "VERBOSE = 1\n",
        "FOLDS_TO_TRAIN = 5\n",
        "if not os.path.exists('WaveNet_Model'):\n",
        "    os.makedirs('WaveNet_Model')\n",
        "\n",
        "from sklearn.model_selection import KFold, GroupKFold\n",
        "import tensorflow.keras.backend as K, gc\n",
        "\n",
        "all_oof = []; all_oof2 = []; all_true = []\n",
        "gkf = GroupKFold(n_splits=5)\n",
        "for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):\n",
        "\n",
        "    print('#'*25)\n",
        "    print(f'### Fold {i+1}')\n",
        "    train_gen = DataGenerator(train.iloc[train_index], shuffle=True, batch_size=16)\n",
        "    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n",
        "    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n",
        "    print('#'*25)\n",
        "\n",
        "    # TRAIN MODEL\n",
        "    K.clear_session()\n",
        "    with strategy.scope():\n",
        "        model = build_model()\n",
        "    if TRAIN_MODEL:\n",
        "        model.load_weights(f'WaveNet_Model/WaveNet_fold{i}.h5')\n",
        "        best_weights_filepath = f'WaveNet_Model/WaveNet_channel_flipped_fold{i}.h5'\n",
        "        earlyStopping= tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10,\n",
        "                                               verbose=1, mode='auto')\n",
        "        saveBestModel = tf.keras.callbacks.ModelCheckpoint(best_weights_filepath,\n",
        "                                                   monitor='val_loss', verbose=1,\n",
        "                                                   save_best_only=True, mode='auto')\n",
        "        model.fit(train_gen, verbose=VERBOSE,\n",
        "              validation_data = valid_gen,\n",
        "              epochs=EPOCHS,callbacks = [PrintLossCallback(),LR,earlyStopping, saveBestModel])\n",
        "\n",
        "    # WAVENET OOF\n",
        "    pred = np.zeros(((len(valid_index)),6))\n",
        "    f = 0\n",
        "    for x,_ in valid_gen:\n",
        "        x_1 = np.zeros((x.shape[0],x.shape[1],x.shape[2]))\n",
        "        for j in range(0,x.shape[0]):\n",
        "            x_1[j] = tranform_tta((x[j])).reshape(x.shape[1],x.shape[2])\n",
        "        pred_2 = model.predict(x, verbose=1)\n",
        "        pred_1 = model.predict(x_1, verbose=1)\n",
        "        pred_1 = (((pred_2 + pred_1)/2))\n",
        "        pred[f:f+pred_1.shape[0]] = pred_1\n",
        "        f += pred_1.shape[0]\n",
        "\n",
        "    all_oof.append(pred)\n",
        "    all_true.append(train.iloc[valid_index][TARGETS].values)\n",
        "\n",
        "    # TRAIN MEAN OOF\n",
        "    y_train = train.iloc[train_index][TARGETS].values\n",
        "    y_valid = train.iloc[valid_index][TARGETS].values\n",
        "    oof = y_valid.copy()\n",
        "    for j in range(6):\n",
        "        oof[:,j] = y_train[:,j].mean()\n",
        "    oof = oof / oof.sum(axis=1,keepdims=True)\n",
        "    all_oof2.append(oof)\n",
        "\n",
        "    del model, oof, y_train, y_valid\n",
        "    gc.collect()\n",
        "\n",
        "    if i==FOLDS_TO_TRAIN-1: break\n",
        "\n",
        "all_oof = np.concatenate(all_oof)\n",
        "all_oof2 = np.concatenate(all_oof2)\n",
        "all_true = np.concatenate(all_true)"
      ],
      "metadata": {
        "id": "SHwAvcxrOKXr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# CV with Channel Flip"
      ],
      "metadata": {
        "id": "1zWmomazOKXs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append('/kaggle/input/kaggle-kl-div')\n",
        "from kaggle_kl_div import score\n",
        "\n",
        "oof = pd.DataFrame(all_oof.copy())\n",
        "oof['id'] = np.arange(len(oof))\n",
        "\n",
        "true = pd.DataFrame(all_true.copy())\n",
        "true['id'] = np.arange(len(true))\n",
        "\n",
        "cv = score(solution=true, submission=oof, row_id_column_name='id')\n",
        "print('CV Score with WaveNet Raw EEG =',cv)"
      ],
      "metadata": {
        "id": "6RGycqwrOKXs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2 Stage Training"
      ],
      "metadata": {
        "id": "s7nBZwUxOKXs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n",
        "df['total_evaluators'] = df[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].sum(axis=1)\n",
        "df"
      ],
      "metadata": {
        "id": "nFO5qM0LOKXs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('/kaggle/input/hms-harmful-brain-activity-classification/train.csv')\n",
        "df['total_evaluators'] = df[['seizure_vote', 'lpd_vote', 'gpd_vote', 'lrda_vote', 'grda_vote', 'other_vote']].sum(axis=1)\n",
        "TARGETS = df.columns[-7:-1]\n",
        "TARS = {'Seizure':0, 'LPD':1, 'GPD':2, 'LRDA':3, 'GRDA':4, 'Other':5}\n",
        "TARS2 = {x:y for y,x in TARS.items()}\n",
        "\n",
        "train = df.groupby('eeg_id')[['patient_id']].agg('first')\n",
        "tmp = df.groupby('eeg_id')[['total_evaluators']].agg('mean')\n",
        "train['total_evaluators'] = tmp\n",
        "tmp = df.groupby('eeg_id')[TARGETS].agg('sum')\n",
        "for t in TARGETS:\n",
        "    train[t] = tmp[t].values\n",
        "\n",
        "y_data = train[TARGETS].values\n",
        "y_data = y_data / y_data.sum(axis=1,keepdims=True)\n",
        "train[TARGETS] = y_data\n",
        "\n",
        "tmp = df.groupby('eeg_id')[['expert_consensus']].agg('first')\n",
        "train['target'] = tmp\n",
        "\n",
        "train = train.reset_index()\n",
        "train = train.loc[train.eeg_id.isin(EEG_IDS)]\n",
        "print('Train Data with unique eeg_id shape:', train.shape )\n",
        "train.head()"
      ],
      "metadata": {
        "id": "Ve14rilVOKXt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "all_oof = []; all_oof2 = []; all_true = []\n",
        "gkf = GroupKFold(n_splits=5)\n",
        "for i, (train_index, valid_index) in enumerate(gkf.split(train, train.target, train.patient_id)):\n",
        "\n",
        "    print('#'*25)\n",
        "    print(f'### Fold {i+1}')\n",
        "    train_gen2 = DataGenerator(train.iloc[train_index][train.iloc[train_index][\"total_evaluators\"]>=10], shuffle=True, batch_size=16)\n",
        "    valid_gen2 = DataGenerator(train.iloc[valid_index][train.iloc[valid_index][\"total_evaluators\"]>=10], shuffle=False, batch_size=64,mode='valid')\n",
        "    valid_gen = DataGenerator(train.iloc[valid_index], shuffle=False, batch_size=64, mode='valid')\n",
        "    print(f'### train size {len(train_index)}, valid size {len(valid_index)}')\n",
        "    print('#'*25)\n",
        "\n",
        "    # TRAIN MODEL\n",
        "    K.clear_session()\n",
        "    with strategy.scope():\n",
        "        model = build_model()\n",
        "    if TRAIN_MODEL:\n",
        "        model.load_weights(f'WaveNet_Model/WaveNet_channel_flipped_fold{i}.h5')\n",
        "        best_weights_filepath = f'WaveNet_Model/WaveNet_Final_fold{i}.h5'\n",
        "        earlyStopping= tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10,\n",
        "                                               verbose=1, mode='auto')\n",
        "        saveBestModel = tf.keras.callbacks.ModelCheckpoint(best_weights_filepath,\n",
        "                                                   monitor='val_loss', verbose=1,\n",
        "                                                   save_best_only=True, mode='auto')\n",
        "        model.fit(train_gen2, verbose=VERBOSE,\n",
        "              validation_data = valid_gen2,\n",
        "              epochs=EPOCHS,callbacks = [PrintLossCallback(),LR,earlyStopping, saveBestModel])\n",
        "\n",
        "    # WAVENET OOF\n",
        "    pred = np.zeros(((len(valid_index)),6))\n",
        "    f = 0\n",
        "    for x,_ in valid_gen:\n",
        "        x_1 = np.zeros((x.shape[0],x.shape[1],x.shape[2]))\n",
        "        for j in range(0,x.shape[0]):\n",
        "            x_1[j] = tranform_tta((x[j])).reshape(x.shape[1],x.shape[2])\n",
        "        pred_2 = model.predict(x, verbose=1)\n",
        "        pred_1 = model.predict(x_1, verbose=1)\n",
        "        pred_1 = (((pred_2 + pred_1)/2))\n",
        "        pred[f:f+pred_1.shape[0]] = pred_1\n",
        "        f += pred_1.shape[0]\n",
        "\n",
        "    all_oof.append(pred)\n",
        "    all_true.append(train.iloc[valid_index][TARGETS].values)\n",
        "\n",
        "    # TRAIN MEAN OOF\n",
        "    y_train = train.iloc[train_index][TARGETS].values\n",
        "    y_valid = train.iloc[valid_index][TARGETS].values\n",
        "    oof = y_valid.copy()\n",
        "    for j in range(6):\n",
        "        oof[:,j] = y_train[:,j].mean()\n",
        "    oof = oof / oof.sum(axis=1,keepdims=True)\n",
        "    all_oof2.append(oof)\n",
        "\n",
        "    del model, oof, y_train, y_valid\n",
        "    gc.collect()\n",
        "\n",
        "    if i==FOLDS_TO_TRAIN-1: break\n",
        "\n",
        "all_oof = np.concatenate(all_oof)\n",
        "all_oof2 = np.concatenate(all_oof2)\n",
        "all_true = np.concatenate(all_true)"
      ],
      "metadata": {
        "id": "b2Epz7TVOKXt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "oAcUqxUcOKXt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "CAA7oBIfOKXu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "hcINQ4I-OKXu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}