Histogram_Dichotomizor_SVM_WithDiff.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<mne.utils.use_log_level at 0xd1b27b8>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import mne\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import itertools\n",
    "from scipy import signal\n",
    "from scipy.stats import entropy\n",
    "mne.utils.use_log_level('error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Add new featres in feature list and again in getFeatures method's \"flist\"\n",
    "feature_list = ['Mean', 'Median', 'StdDev','CoV','Skew','Kurt']\n",
    "columns = ['Subject','Test', 'Channel', *feature_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#subject = subject number\n",
    "#testtype = VR or Non-VR\n",
    "def loadRawData(subject, testtype):\n",
    "    path = \"Preprocessed_Data/Subject-\"+str(subject)+\"_\"+testtype+\".fif\"\n",
    "    raw = mne.io.read_raw_fif(path, verbose='error')\n",
    "    return raw\n",
    "\n",
    "def getFeatures(raw, time, segment=10):\n",
    "    features = pd.DataFrame(columns = feature_list)\n",
    "    inc = (int)(segment*.25)\n",
    "    for i in range(time[0],time[1],inc): \n",
    "        end = i+inc\n",
    "        if(i+inc>time[1]):\n",
    "            end = time[1]           \n",
    "        start, stop = raw.time_as_index([i, end])\n",
    "        \n",
    "        #only looking at CH 4\n",
    "        picks = mne.pick_types(raw.info, eeg=True, exclude=['CH 1','CH 2','CH 3','CH 5','CH 6','CH 7','CH 8'])\n",
    "        \n",
    "        try:    \n",
    "            d, t = raw[picks[:], start:stop]\n",
    "            ds = pd.DataFrame(d[0])\n",
    "            #normalize\n",
    "            #ds = (ds-ds.mean())/(ds.max()-ds.min())\n",
    "            \n",
    "            mean = ds.mean()[0]\n",
    "            median = ds.median()[0]\n",
    "            std = ds.std()[0]\n",
    "            mos = mean/std\n",
    "            skew = ds.skew()[0]\n",
    "            kurt = ds.kurt()[0]\n",
    "            \n",
    "            #add new features here too \n",
    "            flist = [mean, median, std, mos, skew, kurt]\n",
    "            features = features.append(pd.Series(flist, index=feature_list), ignore_index=True)\n",
    "        except:\n",
    "            continue\n",
    "    return features\n",
    "\n",
    "def getDataFrame(f, data):\n",
    "    for j in range(len(f)):\n",
    "        fl = [None]*len(feature_list)\n",
    "        k=0\n",
    "        for feature in feature_list:\n",
    "            fl[k] = f[feature][j]\n",
    "            k+=1\n",
    "        data = data.append(pd.Series([i, 'VR', 'CH 4', *fl], index=columns), ignore_index=True)\n",
    "    return data\n",
    "\n",
    "\n",
    "def getDiff(df):\n",
    "    diffDF = df.copy()\n",
    "    if('Subject' in df.columns):\n",
    "        diffDF.drop(['Subject'], axis=1, inplace=True)\n",
    "        diffDF.drop(['Test'], axis=1, inplace=True)\n",
    "        diffDF.drop(['Channel'], axis=1, inplace=True)\n",
    "    diffDF = diffDF.diff().dropna()\n",
    "    diffDF = diffDF.applymap(np.abs)\n",
    "    return diffDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test 1\n",
    "#testtype = VR or Non-VR\n",
    "df1 = pd.DataFrame(columns = columns)\n",
    "for i in range(1,33):\n",
    "    data = loadRawData(i,\"VR\")\n",
    "    f = getFeatures(raw=data, time=(60,120))\n",
    "    df1 = getDataFrame(f,df1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = getDiff(df1)\n",
    "df3 = getDiff(df2)\n",
    "\n",
    "df1.reset_index(drop=True, inplace=True)\n",
    "df2.reset_index(drop=True, inplace=True)\n",
    "df3.reset_index(drop=True, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df1)/29"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_list = ['Mean', 'Median', 'StdDev','CoV','Skew','Kurt', \n",
    "                'Dif1 Mean', 'Dif1 Median', 'Dif1 StdDev','Dif1 CoV','Dif1 Skew','Dif1 Kurt',\n",
    "                'Dif2 Mean', 'Dif2 Median', 'Dif2 StdDev','Dif2 CoV','Dif2 Skew','Dif2 Kurt']\n",
    "columns = ['Subject','Test', 'Channel', *feature_list]\n",
    "\n",
    "df = pd.concat([df1, df2, df3], axis=1)\n",
    "df.columns = columns\n",
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subject</th>\n",
       "      <th>Test</th>\n",
       "      <th>Channel</th>\n",
       "      <th>Mean</th>\n",
       "      <th>Median</th>\n",
       "      <th>StdDev</th>\n",
       "      <th>CoV</th>\n",
       "      <th>Skew</th>\n",
       "      <th>Kurt</th>\n",
       "      <th>Dif1 Mean</th>\n",
       "      <th>...</th>\n",
       "      <th>Dif1 StdDev</th>\n",
       "      <th>Dif1 CoV</th>\n",
       "      <th>Dif1 Skew</th>\n",
       "      <th>Dif1 Kurt</th>\n",
       "      <th>Dif2 Mean</th>\n",
       "      <th>Dif2 Median</th>\n",
       "      <th>Dif2 StdDev</th>\n",
       "      <th>Dif2 CoV</th>\n",
       "      <th>Dif2 Skew</th>\n",
       "      <th>Dif2 Kurt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>VR</td>\n",
       "      <td>CH 4</td>\n",
       "      <td>-0.016555</td>\n",
       "      <td>-0.853062</td>\n",
       "      <td>7.532125</td>\n",
       "      <td>-0.002198</td>\n",
       "      <td>1.284384</td>\n",
       "      <td>6.718239</td>\n",
       "      <td>0.080915</td>\n",
       "      <td>...</td>\n",
       "      <td>1.264397</td>\n",
       "      <td>0.009514</td>\n",
       "      <td>1.109892</td>\n",
       "      <td>4.897208</td>\n",
       "      <td>0.025475</td>\n",
       "      <td>0.213015</td>\n",
       "      <td>0.680844</td>\n",
       "      <td>0.003937</td>\n",
       "      <td>0.015292</td>\n",
       "      <td>2.386970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>VR</td>\n",
       "      <td>CH 4</td>\n",
       "      <td>0.064359</td>\n",
       "      <td>-0.309380</td>\n",
       "      <td>8.796523</td>\n",
       "      <td>0.007316</td>\n",
       "      <td>0.174492</td>\n",
       "      <td>1.821031</td>\n",
       "      <td>0.106390</td>\n",
       "      <td>...</td>\n",
       "      <td>1.945242</td>\n",
       "      <td>0.013451</td>\n",
       "      <td>1.094600</td>\n",
       "      <td>2.510239</td>\n",
       "      <td>0.201895</td>\n",
       "      <td>0.012938</td>\n",
       "      <td>2.393204</td>\n",
       "      <td>0.016478</td>\n",
       "      <td>0.200489</td>\n",
       "      <td>0.335245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>VR</td>\n",
       "      <td>CH 4</td>\n",
       "      <td>-0.042031</td>\n",
       "      <td>-0.640048</td>\n",
       "      <td>6.851281</td>\n",
       "      <td>-0.006135</td>\n",
       "      <td>1.269092</td>\n",
       "      <td>4.331269</td>\n",
       "      <td>0.308285</td>\n",
       "      <td>...</td>\n",
       "      <td>4.338446</td>\n",
       "      <td>0.029929</td>\n",
       "      <td>0.894111</td>\n",
       "      <td>2.845484</td>\n",
       "      <td>0.004509</td>\n",
       "      <td>0.046104</td>\n",
       "      <td>0.516354</td>\n",
       "      <td>0.000182</td>\n",
       "      <td>0.582203</td>\n",
       "      <td>0.592728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>VR</td>\n",
       "      <td>CH 4</td>\n",
       "      <td>0.266254</td>\n",
       "      <td>-0.296442</td>\n",
       "      <td>11.189727</td>\n",
       "      <td>0.023795</td>\n",
       "      <td>0.374981</td>\n",
       "      <td>1.485785</td>\n",
       "      <td>0.312794</td>\n",
       "      <td>...</td>\n",
       "      <td>3.822092</td>\n",
       "      <td>0.030111</td>\n",
       "      <td>0.311909</td>\n",
       "      <td>2.252756</td>\n",
       "      <td>0.208722</td>\n",
       "      <td>0.190916</td>\n",
       "      <td>2.962448</td>\n",
       "      <td>0.013286</td>\n",
       "      <td>1.267148</td>\n",
       "      <td>1.101471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>VR</td>\n",
       "      <td>CH 4</td>\n",
       "      <td>-0.046540</td>\n",
       "      <td>0.093268</td>\n",
       "      <td>7.367635</td>\n",
       "      <td>-0.006317</td>\n",
       "      <td>0.063072</td>\n",
       "      <td>3.738542</td>\n",
       "      <td>0.104072</td>\n",
       "      <td>...</td>\n",
       "      <td>0.859644</td>\n",
       "      <td>0.016826</td>\n",
       "      <td>1.579057</td>\n",
       "      <td>3.354227</td>\n",
       "      <td>0.011449</td>\n",
       "      <td>0.367240</td>\n",
       "      <td>0.290256</td>\n",
       "      <td>0.001256</td>\n",
       "      <td>0.576975</td>\n",
       "      <td>2.442410</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Subject Test Channel      Mean    Median     StdDev       CoV      Skew  \\\n",
       "0       1   VR    CH 4 -0.016555 -0.853062   7.532125 -0.002198  1.284384   \n",
       "1       1   VR    CH 4  0.064359 -0.309380   8.796523  0.007316  0.174492   \n",
       "2       1   VR    CH 4 -0.042031 -0.640048   6.851281 -0.006135  1.269092   \n",
       "3       1   VR    CH 4  0.266254 -0.296442  11.189727  0.023795  0.374981   \n",
       "4       1   VR    CH 4 -0.046540  0.093268   7.367635 -0.006317  0.063072   \n",
       "\n",
       "       Kurt  Dif1 Mean  ...  Dif1 StdDev  Dif1 CoV  Dif1 Skew  Dif1 Kurt  \\\n",
       "0  6.718239   0.080915  ...     1.264397  0.009514   1.109892   4.897208   \n",
       "1  1.821031   0.106390  ...     1.945242  0.013451   1.094600   2.510239   \n",
       "2  4.331269   0.308285  ...     4.338446  0.029929   0.894111   2.845484   \n",
       "3  1.485785   0.312794  ...     3.822092  0.030111   0.311909   2.252756   \n",
       "4  3.738542   0.104072  ...     0.859644  0.016826   1.579057   3.354227   \n",
       "\n",
       "   Dif2 Mean  Dif2 Median  Dif2 StdDev  Dif2 CoV  Dif2 Skew  Dif2 Kurt  \n",
       "0   0.025475     0.213015     0.680844  0.003937   0.015292   2.386970  \n",
       "1   0.201895     0.012938     2.393204  0.016478   0.200489   0.335245  \n",
       "2   0.004509     0.046104     0.516354  0.000182   0.582203   0.592728  \n",
       "3   0.208722     0.190916     2.962448  0.013286   1.267148   1.101471  \n",
       "4   0.011449     0.367240     0.290256  0.001256   0.576975   2.442410  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also\n",
    "    #print(df)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed(14)\n",
    "\n",
    "def distanceCalculation(df):\n",
    "    subs = df['Subject'].unique()    # All subjects\n",
    "    all_subs= list(itertools.combinations(subs, 2)) # All possible combination for all subjects\n",
    "\n",
    "    distance_col = ['Subject',*feature_list, 'Type']\n",
    "    intra_data = pd.DataFrame(columns = distance_col)\n",
    "\n",
    "    #Intra Distance Computation (Same Person)\n",
    "    for sub in subs:\n",
    "        rows = df.loc[df['Subject'] == sub]\n",
    "        each_comb = list(itertools.combinations(rows.index, 2))\n",
    "        for i in range(len(each_comb)):\n",
    "            comb = each_comb[i]\n",
    "            fdr = absDistance(df, feature_list, comb[0], comb[1])\n",
    "            intra_data = intra_data.append(pd.Series([sub,*fdr,0], index=distance_col), ignore_index=True)\n",
    "    \n",
    "    #Inter Distance Computation (Different Person)\n",
    "    inter_data = pd.DataFrame(columns = distance_col)\n",
    "    inter_total = len(intra_data)*2\n",
    "    while(len(inter_data)<inter_total):\n",
    "        v1 = random.randint(0, len(df)-1)\n",
    "        v2 = random.randint(0, len(df)-1)\n",
    "        while(df.iloc[v1]['Subject']==df.iloc[v2]['Subject']):\n",
    "            v1 = random.randint(0, len(df)-1)\n",
    "            v2 = random.randint(0, len(df)-1)      \n",
    "        fdr = absDistance(df, feature_list, v1, v2)\n",
    "        sub_pair = (df.iloc[v1]['Subject'],df.iloc[v2]['Subject'])\n",
    "        inter_data = inter_data.append(pd.Series([sub_pair, *fdr, 1], index=distance_col), ignore_index=True)    \n",
    "    return intra_data, inter_data\n",
    "\n",
    "def absDistance(df, features, s1, s2):\n",
    "    r=0\n",
    "    fdr = [None]*len(features)\n",
    "    for feature in features:\n",
    "        f1 = df.iloc[s1][feature] \n",
    "        f2 = df.iloc[s2][feature] \n",
    "        Inter_dis = np.absolute(f1-f2) # absolute difference\n",
    "        fdr[r] = Inter_dis\n",
    "        r+=1\n",
    "    return fdr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#15 min to run (6 fetures), 18 min (8 features) - 4/17 - 15 minutes (18 features)\n",
    "intra1, inter1 = distanceCalculation(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Intra length: \"+str(len(intra1)))\n",
    "print(\"Inter length: \"+str(len(inter1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(1234)\n",
    "#takes subframe and returns a more managble table for SVM\n",
    "def get_SVM_Table(intra, inter):\n",
    "    svmTable = pd.DataFrame()\n",
    "    len1 = (len(intra)//100)*100\n",
    "    len2 = (len1/2)*3\n",
    "    rands = random.sample(range(0, len(intra)), len1)\n",
    "    for rand in rands:\n",
    "        svmTable = svmTable.append(intra.iloc[rand],ignore_index=True)\n",
    "    \n",
    "    rands = random.sample(range(0, len(inter)), len2)\n",
    "    for rand in rands:\n",
    "        svmTable = svmTable.append(inter.iloc[rand],ignore_index=True)\n",
    "    return svmTable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm1 = get_SVM_Table(intra1,inter1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "#all_svm = pd.concat([svm1, svm2], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "def svmTest(svm):\n",
    "    X = svm.drop(['Subject','Type'], axis=1)\n",
    "    y = svm[\"Type\"]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=101)\n",
    "    \n",
    "    model = SVC(C=1.0, kernel = 'rbf', degree=3, gamma='auto')\n",
    "    model.fit(X_train, y_train)\n",
    "    prediction = model.predict(X_test)\n",
    "    confusionMatrix = confusion_matrix(y_test, prediction)\n",
    "    print(confusion_matrix(y_test, prediction))\n",
    "    print(classification_report(y_test, prediction))\n",
    "    print(\"Accuracy: \"+str(accuracy_score(y_test, prediction)))\n",
    "    return confusionMatrix, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test 1:\n",
      "[[1027  484]\n",
      " [ 434 1805]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.70      0.68      0.69      1511\n",
      "         1.0       0.79      0.81      0.80      2239\n",
      "\n",
      "   micro avg       0.76      0.76      0.76      3750\n",
      "   macro avg       0.75      0.74      0.74      3750\n",
      "weighted avg       0.75      0.76      0.75      3750\n",
      "\n",
      "Accuracy: 0.7552\n"
     ]
    }
   ],
   "source": [
    "print('Test 1:')\n",
    "vr_cm, vr_model = svmTest(svm1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False Acceptance: 0.12906666666666666\n",
      "False Rejection: 0.11573333333333333\n",
      "0.7552\n"
     ]
    }
   ],
   "source": [
    "TN = vr_cm[0][0]\n",
    "FN = vr_cm[1][0]\n",
    "TP = vr_cm[1][1]\n",
    "FP = vr_cm[0][1]\n",
    "sums = TN+TP+FN+FP\n",
    "\n",
    "acc = (TN+TP)/sums\n",
    "\n",
    "print('False Acceptance: '+str(FP/sums))\n",
    "print('False Rejection: '+str(FN/sums))\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}