phonelab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "from scipy import sparse\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.colors as clr\n",
    "# plt.style.use('science')\n",
    "# plt.style.use(['science','notebook'])\n",
    "# plt.style.use('seaborn-ticks')\n",
    "# plt.rcParams['figure.dpi'] = 300\n",
    "# plt.rcParams['savefig.dpi'] = 300\n",
    "# import seaborn as sns\n",
    "# sns.set_style('ticks')\n",
    "# sns.set_context('notebook')\n",
    "# sns.set(rc={'figure.dpi': 300, 'savefig.dpi': 300})\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.paths import *\n",
    "from graph.phonelab import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Database"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Create"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It has been done in scripts. Check `code` folder."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Laod"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read from DB without any filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# db = os.path.join(paths.PHONELAB_DB, 'phonelab.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# query = f'SELECT user,ssid,time FROM logs WHERE connect = 1 ORDER BY time,user,ssid;'\n",
    "# query = f'SELECT user,bssid,time FROM logs WHERE connect = 1 ORDER BY time,user,bssid;'\n",
    "# query = f'SELECT user,ssid,bssid,time FROM logs WHERE connect = 1 ORDER BY time,user,ssid,bssid;'\n",
    "# query = f'SELECT user,ssid,time FROM logs WHERE connect = 0 ORDER BY time,user,ssid;'\n",
    "# query = f'SELECT user,bssid,time FROM logs WHERE connect = 0 ORDER BY time,user,bssid;'\n",
    "# query = f'SELECT user,ssid,bssid,time FROM logs WHERE connect = 0 ORDER BY time,user,ssid,bssid;'\n",
    "# df = db_select_df(db, query)\n",
    "# df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read from DB while filtering dates and connection type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connect = 1\n",
    "# t = datetime.fromisoformat('2014-11-07')\n",
    "# query = f'SELECT user,ssid,bssid,time FROM logs WHERE connect = {connect} AND date = \\'{t.date()}\\' ORDER BY time,user,ssid,bssid;'\n",
    "# df = db_select_df(db, query)\n",
    "# df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Create"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Selected SSIDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# phonelab_selected_ssid(selected=2, share=False, output=True)\n",
    "\n",
    "# for i in range(2,6):\n",
    "#     phonelab_selected_ssid(selected=i, share=False, output=False)\n",
    "# phonelab_selected_ssid(selected=4, share=True, output=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Spatial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# phonelab_spatial_selected(selected=2, label_file_in='connect', output=True)\n",
    "\n",
    "# for i in range(1,6):\n",
    "#     phonelab_spatial_selected(selected=i, label_file_in='connect', output=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Spatial-Temporal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# phonelab_spatial_temporal_selected(selected=2, label_file_in='connect', output=True)\n",
    "\n",
    "# for i in range(1,6):\n",
    "#     phonelab_spatial_temporal_selected(selected=i, label_file_in='connect', output=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "User Spatial Connect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(270, 1176)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l1 = 'user'\n",
    "l2 = 'spatial'\n",
    "l3 = 'connect'\n",
    "selected = 0\n",
    "row_scaled = ''\n",
    "f1 = f'{row_scaled}selected_{selected}'\n",
    "d1 = os.path.join(paths.PHONELAB_DATA, f1, l2)\n",
    "path = os.path.join(d1, f'{l1}_{l2}_{l3}.npz')\n",
    "M = sparse.load_npz(path).toarray()\n",
    "M.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "User Day Spatial Connect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l1 = 'user_day'\n",
    "# l2 = 'spatial'\n",
    "# l3 = 'connect'\n",
    "# selected = 0\n",
    "# row_scaled = ''\n",
    "# f1 = f'{row_scaled}selected_{selected}'\n",
    "# d1 = os.path.join(paths.PHONELAB_DATA, f1, l2)\n",
    "# path = os.path.join(d1, f'{l1}_{l2}_{l3}.npz')\n",
    "# M = sparse.load_npz(path).toarray()\n",
    "# M.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "User Spatial Temporal Connect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l1 = 'user'\n",
    "# l2 = 'spatial_temporal'\n",
    "# l3 = 'connect'\n",
    "# selected = 0\n",
    "# row_scaled = ''\n",
    "# f1 = f'{row_scaled}selected_{selected}'\n",
    "# d1 = os.path.join(paths.PHONELAB_DATA, f1, l2)\n",
    "# path = os.path.join(d1, f'{l1}_{l2}_{l3}.npz')\n",
    "# M = sparse.load_npz(path).toarray()\n",
    "# M.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "User Day Spatial Temporal Connect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l1 = 'user_day'\n",
    "# l2 = 'spatial_temporal'\n",
    "# l3 = 'connect'\n",
    "# selected = 0\n",
    "# row_scaled = ''\n",
    "# f1 = f'{row_scaled}selected_{selected}'\n",
    "# d1 = os.path.join(paths.PHONELAB_DATA, f1, l2)\n",
    "# path = os.path.join(d1, f'{l1}_{l2}_{l3}.npz')\n",
    "# M = sparse.load_npz(path).toarray()\n",
    "# M.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "User -> Location (Connect) has 270 users (row) and 1,176 SSID (column)    \n",
    "User-Day -> SSID-Hour (Connect) has 19,144 rows and 28,224 columns (=1176x24)    \n",
    "User-Day -> Selected-SSID-Hour (Connect) has 5,151 rows and 6,072 columns (253=x24)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert M to matrix of zeros and ones\n",
    "M[M > 0] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above code, turn matrix M to unweighted i.e. element is 1 if user connected to  \n",
    "SSID and 0 if not connected (before the element was the number of connection)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "user_dist = np.array(np.sum(M, axis=1, dtype=int).reshape(1, -1))[0]\n",
    "print(np.where(user_dist == 0)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is to check to see if there is any USER that has not been connected to any    \n",
    "SSID or visited a location, i.e., if the sum of row (representing a user's data)    \n",
    "has no information, which then implies that there has been a bug in code    \n",
    "(in fact there was one in earlier version of code) and then if the list is empty    \n",
    "we can move on to next step of analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[163 240  39 246  76 294  89  32  91  92]\n",
      "[(0, 163), (1, 240), (2, 39), (3, 246), (4, 76), (5, 294), (6, 89), (7, 32), (8, 91), (9, 92)]\n"
     ]
    }
   ],
   "source": [
    "print(user_dist[:10])\n",
    "print(list(enumerate(user_dist))[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking the number of time the firs 10 users {1,...,10} connected to SSIDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close('all')\n",
    "fig, ax = plt.subplots(figsize=(6, 4))\n",
    "ax.hist(\n",
    "    user_dist,\n",
    "    bins=range(min(user_dist),\n",
    "               max(user_dist) + 1, 1),\n",
    "    # cumulative=True,\n",
    "    alpha=0.7,\n",
    "    rwidth=0.5,\n",
    "    align='left'\n",
    ")\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('Number of Connected SSID by Users Histogram')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "25 users connected to 1 SSID only, and at maximum 1 user connected to 38 unique  \n",
    "SSIDs. Using cumulative plotting, we can see that considering users connecting  \n",
    "to 1 to 5 SSID would add up to about 130 users which is half of 270 users.\n",
    "\n",
    "Now the question is, if we consider top 2 or 3 connected SSID of each user then  \n",
    "how many important SSID would we have at the end ?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "node 14 only has connected to one ssid 51\n",
      "node 18 only has connected to one ssid 72\n",
      "node 19 only has connected to one ssid 51\n",
      "node 31 only has connected to one ssid 1\n",
      "node 32 only has connected to one ssid 51\n",
      "node 43 only has connected to one ssid 1\n",
      "node 47 only has connected to one ssid 1\n",
      "node 54 only has connected to one ssid 266\n",
      "node 58 only has connected to one ssid 293\n",
      "node 83 only has connected to one ssid 407\n",
      "node 89 only has connected to one ssid 85\n",
      "node 94 only has connected to one ssid 1\n",
      "node 130 only has connected to one ssid 645\n",
      "node 140 only has connected to one ssid 691\n",
      "node 165 only has connected to one ssid 1\n",
      "node 181 only has connected to one ssid 1\n",
      "node 197 only has connected to one ssid 722\n",
      "node 198 only has connected to one ssid 1\n",
      "node 234 only has connected to one ssid 1\n",
      "node 238 only has connected to one ssid 1\n",
      "node 250 only has connected to one ssid 1\n",
      "node 260 only has connected to one ssid 1\n",
      "node 264 only has connected to one ssid 51\n",
      "node 266 only has connected to one ssid 100\n",
      "node 267 only has connected to one ssid 1\n"
     ]
    }
   ],
   "source": [
    "# Number of important SSID extracted from each user\n",
    "max_ssid = 2\n",
    "imp_ssids = []\n",
    "imp_ssids_users = {}\n",
    "\n",
    "for idx, row in enumerate(M):\n",
    "\n",
    "    row = row.astype(int)\n",
    "    # Get number of non-zero elements i.e. SSIDs that each user connected to\n",
    "    connected_ssids = np.where(row > 0)[0]\n",
    "\n",
    "    if len(connected_ssids) == 1:\n",
    "        imp_ssids.append(connected_ssids[0])\n",
    "        if connected_ssids[0] not in imp_ssids_users:\n",
    "            imp_ssids_users[connected_ssids[0]] = [idx]\n",
    "        else:\n",
    "            imp_ssids_users[connected_ssids[0]].append(idx)\n",
    "        print('node', idx, 'only has connected to one ssid', *connected_ssids)\n",
    "\n",
    "    if len(connected_ssids) > 1:\n",
    "        row_sort_idx = np.argsort(row)\n",
    "        for ssid in row_sort_idx[::-1][:max_ssid]:\n",
    "            imp_ssids.append(ssid)\n",
    "            if ssid not in imp_ssids_users:\n",
    "                imp_ssids_users[ssid] = [idx]\n",
    "            else:\n",
    "                imp_ssids_users[ssid].append(idx)\n",
    "        # print(*row_sort_idx[::-1][:max_ssid], sep='\\t')\n",
    "        # print(*row[row_sort_idx][::-1][:max_ssid], sep='\\t')\n",
    "\n",
    "# Convert important SSID list to set in order to remove duplicates\n",
    "imp_ssids = sorted(list(set(imp_ssids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "253 important AP has been selected by taking top 2 from each user.\n",
      "The important APs are:\n",
      "[0, 1, 3, 7, 8, 9, 10, 13, 16, 43, 46, 48, 51, 57, 62, 72, 75, 76, 77, 78, 79, 85, 87, 100, 102, 103, 106, 108, 111, 112, 114, 115, 117, 119, 123, 126, 129, 133, 134, 139, 141, 146, 147, 149, 152, 154, 155, 156, 157, 158, 160, 162, 166, 168, 171, 179, 182, 183, 192, 194, 205, 209, 211, 212, 219, 220, 224, 230, 232, 235, 236, 242, 243, 244, 245, 246, 249, 258, 262, 266, 267, 269, 271, 278, 293, 297, 299, 305, 310, 313, 316, 322, 323, 324, 326, 328, 329, 330, 332, 336, 340, 348, 353, 354, 357, 368, 369, 378, 380, 381, 384, 385, 388, 389, 390, 391, 402, 403, 404, 407, 422, 424, 431, 432, 436, 438, 447, 468, 469, 474, 475, 484, 485, 486, 492, 498, 499, 503, 506, 507, 508, 519, 520, 521, 523, 527, 547, 549, 550, 552, 553, 566, 569, 576, 579, 582, 585, 589, 590, 591, 594, 596, 600, 602, 607, 618, 624, 626, 627, 628, 629, 630, 639, 640, 645, 646, 668, 669, 679, 680, 688, 691, 692, 696, 714, 719, 721, 722, 743, 745, 747, 754, 785, 795, 802, 806, 809, 814, 816, 818, 822, 826, 832, 838, 841, 850, 851, 859, 864, 869, 888, 893, 898, 900, 901, 902, 904, 905, 909, 918, 929, 932, 933, 964, 966, 968, 975, 976, 983, 989, 991, 1006, 1014, 1017, 1018, 1028, 1034, 1036, 1046, 1053, 1064, 1067, 1071, 1077, 1080, 1087, 1091, 1095, 1104, 1105, 1136, 1138, 1162]\n"
     ]
    }
   ],
   "source": [
    "print(f'{len(imp_ssids)} important AP has been selected by taking top {max_ssid} from each user.')\n",
    "print('The important APs are:')\n",
    "print(imp_ssids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save selected SSIDs\n",
    "path = os.path.join(paths.PHONELAB_DATA, f'selected_{max_ssid}', 'spatial', 'selected_ssid.csv')\n",
    "np.savetxt(path, imp_ssids, delimiter=',', fmt='%s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interestingly, many of these nodes (exactly 12) connected to SSID = 1 which we  \n",
    "think is the main university Wi-Fi network's SSID and then 4 of users connected  \n",
    "to SSID = 4 which we don't know what it is, but the rest only has been connected  \n",
    "by 1 user only.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we use 2 top connected SSIDs for each user we end up with 253 SSIDs at the  \n",
    "end, but if we use 3 top SSIDs we end up with 366 unique SSID. The distribution  \n",
    "if how many users connected to each SSID is analyzed be below ...\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of connection to SSID: frequency\n",
      "{1: 210, 2: 28, 3: 9, 8: 2, 170: 1, 4: 1, 25: 1, 7: 1}\n",
      "SSID: number of connections\n",
      "{1: 170, 43: 25, 51: 8, 85: 8, 72: 7, 8: 4}\n"
     ]
    }
   ],
   "source": [
    "num_users_per_ssid = []\n",
    "num_users_ssid = {}\n",
    "for ssid in imp_ssids_users:\n",
    "    # print(f'{ssid}:')\n",
    "    # print(f'\\t{len(imp_ssids_users[ssid])} users: {imp_ssids_users[ssid]}')\n",
    "    num_users_per_ssid.append(len(imp_ssids_users[ssid]))\n",
    "    if len(imp_ssids_users[ssid]) > 3:\n",
    "        num_users_ssid[ssid] = len(imp_ssids_users[ssid])\n",
    "counter = Counter(num_users_per_ssid)\n",
    "# print(counter)\n",
    "# print(sorted(counter,key=counter.get, reverse=True))\n",
    "print('Number of connection to SSID: frequency')\n",
    "print(dict(counter.most_common()))\n",
    "print('SSID: number of connections')\n",
    "print(\n",
    "    dict(\n",
    "        sorted(num_users_ssid.items(), key=lambda item: item[1], reverse=True)\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using 2 top AP for each user\n",
    "210 (out of 253) SSID were only connected by 1 user, meaning 83% of 253 important AP are only being connected by 1 user (i.e. 83% of APs only important for one user and unimportant for the rest) and thus users in total connect to only 17% of shared APs. At the highest, we can see one of the SSIDs (SSID # 1) has been connected by 170 users, and after that, the second and third most popular SSIDs (in terms of number of connections) are 43, and 51 with 25 and 8 connections from users."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "ssid_dist = np.array(np.sum(M, axis=0, dtype=int).reshape(1, -1))[0]\n",
    "print(len(np.where(ssid_dist == 0)[0]))\n",
    "print(np.where(ssid_dist == 0)[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "User-Day -> SSID-Hour has 17,872 zero columns out of 28,224 or 63%, i.e., many    \n",
    "columns are zero and thus PCA could compact information nicely.    \n",
    "\n",
    "User-Day -> (253)Selected-SSID-Hour has 3,669 zero columns out of 6,072 or 60%    \n",
    "which is fairly similar to un-selected SSID case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  1 200   1   1   2   9   1   4  29   2]\n",
      "[(0, 1), (1, 200), (2, 1), (3, 1), (4, 2), (5, 9), (6, 1), (7, 4), (8, 29), (9, 2)]\n"
     ]
    }
   ],
   "source": [
    "print(ssid_dist[:10])\n",
    "print(list(enumerate(ssid_dist))[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking the number of time the firs 10 SSIDs {1,...,10} were connected by users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.close('all')\n",
    "fig, ax = plt.subplots(figsize=(6, 4))\n",
    "ax.hist(\n",
    "    ssid_dist,\n",
    "    bins=range(min(ssid_dist),\n",
    "               max(ssid_dist) + 1, 1),\n",
    "    # cumulative=True,\n",
    "    alpha=0.7,\n",
    "    rwidth=0.5,\n",
    "    align='left'\n",
    ")\n",
    "ax.set_ylim((0, 50))\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('Number of Connected Users for each SSID Histogram')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we sort the SSIDs based on how many time they have been connected to    \n",
    "in general (any user could visited same location more than once)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(1, 200), (43, 68), (72, 45), (85, 31), (8, 29), (115, 26), (133, 25), (44, 21), (51, 21), (147, 20)]\n",
      "SSIDs sorted based on number of users conncting them:\n",
      "(1, 43, 72, 85, 8, 115, 133, 44, 51, 147, 168, 23, 62, 13, 122, 97, 128, 311, 378, 87, 183, 79, 506, 5, 419, 63, 141, 148, 475, 73, 126, 134, 174, 406, 465, 130, 158, 171, 233, 301, 7, 15, 36, 70, 78, 101, 123, 124, 195, 205, 208, 213, 232, 259, 264, 266, 323, 365, 369, 384, 426, 446, 468, 480, 537, 568, 719, 35, 40, 67, 80, 104, 109, 110, 131, 201, 211, 228, 234, 242, 243, 248, 250, 262, 281, 329, 349, 363, 501, 503, 540, 579, 639, 4, 9, 10, 12, 14, 19, 20, 22, 37, 42, 45, 57, 71, 86, 100, 105, 111, 112, 125, 129, 137, 138, 139, 140, 142, 143, 146, 149, 150, 151, 152, 153, 154, 156, 157, 159, 166, 186, 194, 212, 223, 226, 229, 230, 241, 244, 249, 251, 255, 257, 265, 276, 277, 282, 293, 299, 300, 305, 310, 312, 322, 330, 334, 337, 340, 357, 358, 362, 364, 368, 372, 373, 381, 386, 387, 388, 389, 403, 414, 436, 455, 464, 469, 476, 482, 484, 485, 486, 488, 505, 520, 523, 526, 527, 528, 530, 531, 536, 541, 542, 544, 546, 553, 580, 581, 585, 596, 600, 602, 607, 609, 618, 619, 626, 629, 630, 642, 669, 679, 680, 681, 685, 686, 687, 708, 710, 720, 722, 723, 741, 760, 768, 788, 795, 796, 802, 803, 804, 814, 832, 838, 850, 873, 918, 970, 993, 1017, 1038, 1119, 0, 2, 3, 6, 11, 16, 17, 18, 21, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 38, 39, 41, 46, 47, 48, 49, 50, 52, 53, 54, 55, 56, 58, 59, 60, 61, 64, 65, 66, 68, 69, 74, 75, 76, 77, 81, 82, 83, 84, 88, 89, 90, 91, 92, 93, 94, 95, 96, 98, 99, 102, 103, 106, 107, 108, 113, 114, 116, 117, 118, 119, 120, 121, 127, 132, 135, 136, 144, 145, 155, 160, 161, 162, 163, 164, 165, 167, 169, 170, 172, 173, 175, 176, 177, 178, 179, 180, 181, 182, 184, 185, 187, 188, 189, 190, 191, 192, 193, 196, 197, 198, 199, 200, 202, 203, 204, 206, 207, 209, 210, 214, 215, 216, 217, 218, 219, 220, 221, 222, 224, 225, 227, 231, 235, 236, 237, 238, 239, 240, 245, 246, 247, 252, 253, 254, 256, 258, 260, 261, 263, 267, 268, 269, 270, 271, 272, 273, 274, 275, 278, 279, 280, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 294, 295, 296, 297, 298, 302, 303, 304, 306, 307, 308, 309, 313, 314, 315, 316, 317, 318, 319, 320, 321, 324, 325, 326, 327, 328, 331, 332, 333, 335, 336, 338, 339, 341, 342, 343, 344, 345, 346, 347, 348, 350, 351, 352, 353, 354, 355, 356, 359, 360, 361, 366, 367, 370, 371)\n"
     ]
    }
   ],
   "source": [
    "# Make sure filter users to one connection in each row before ssid_dist calculation\n",
    "ssid_dist_tuple = sorted(\n",
    "    list(enumerate(ssid_dist)), key=lambda e: e[1], reverse=True\n",
    ")\n",
    "print(ssid_dist_tuple[:10])\n",
    "print('SSIDs sorted based on number of users conncting them:')\n",
    "print(list(zip(*ssid_dist_tuple[:len(imp_ssids)]))[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Top 2 APs\n",
    "As we can see, **108** SSIDs are shared between important SSID set from top connected 2 SSIDs for each user (out of 253). In other words, about 42% (or near half) of important SSIDs are also exist in highly connected SSIDs and the other half is only important for single user only.\n",
    "#### Top 3 APs\n",
    "This time, **180** SSIDs are shared out of **366** (112 or 48% increase compared to 253) and still about 49% (or near half) of important SSIDs exist in highly connected SSIDs.\n",
    "#### Top 4 APs\n",
    "**250** SSIDs are shared out of **470**. Since shared size is almost as picking only 2 top APs which is 253, we selected this special group and save in `selected_1` folder\n",
    "#### Summary\n",
    "One can conclude that, no mater the size of top connected SSIDs from each user, usually about half of the important SSIDs would be important to more than one user, and the other half would be more important to a specific user most of the time."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When we move up to top 5 SSIDs, the intersection is 250 out of 470 or 53% which    \n",
    "is higher than before, but our important SSID set is almost half the size of all    \n",
    "SSIDs in dataset, but the ratio is still about the half.\n",
    "\n",
    "One can conclude that if we no mater the size of top connected SSIDs from each    \n",
    "user, usually about half of those would be important to all users as well, and    \n",
    "the other half would be more important to that specific user mostly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250\n",
      "470\n",
      "0.5319148936170213\n",
      "[0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 13, 16, 20, 23, 35, 37, 42, 43, 44, 45, 46, 47, 48, 51, 52, 57, 62, 63, 70, 72, 73, 75, 76, 77, 78, 79, 83, 84, 85, 87, 97, 100, 101, 102, 103, 105, 106, 107, 108, 109, 111, 112, 113, 114, 115, 117, 118, 119, 120, 121, 123, 124, 126, 128, 129, 130, 131, 133, 134, 137, 139, 141, 145, 146, 147, 148, 149, 150, 152, 153, 154, 155, 156, 157, 158, 159, 160, 162, 165, 166, 167, 168, 170, 171, 179, 181, 182, 183, 184, 189, 192, 193, 194, 195, 202, 203, 204, 205, 206, 207, 209, 210, 211, 212, 213, 214, 216, 219, 220, 224, 228, 230, 232, 233, 234, 235, 236, 239, 241, 242, 243, 244, 245, 246, 248, 249, 250, 255, 257, 258, 262, 263, 266, 267, 268, 269, 271, 273, 277, 278, 284, 289, 293, 296, 297, 299, 305, 306, 310, 313, 315, 316, 321, 322, 323, 324, 325, 326, 328, 329, 330, 331, 332, 333, 334, 336, 337, 340, 348, 350, 353, 354, 355, 356, 357, 358, 359, 363, 365, 366, 368, 369, 378, 381, 384, 387, 388, 389, 403, 406, 419, 436, 446, 465, 468, 469, 475, 476, 484, 485, 486, 488, 503, 505, 506, 520, 523, 527, 528, 530, 553, 579, 585, 596, 600, 602, 607, 618, 626, 629, 630, 639, 669, 679, 680, 708, 719, 722, 741, 795, 796, 802, 804, 814, 832, 838, 850, 918, 993, 1017]\n"
     ]
    }
   ],
   "source": [
    "imp_ssid_inter = set(imp_ssids).intersection(\n",
    "    list(zip(*ssid_dist_tuple[:len(imp_ssids)]))[0]\n",
    ")\n",
    "print(len(imp_ssid_inter))\n",
    "print(len(imp_ssids))\n",
    "print(f'{len(imp_ssid_inter)/len(imp_ssids)}')\n",
    "print(sorted(list(imp_ssid_inter)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = f'/home/ali/Projects/Network/data/phonelab/data/sp_selected_1/selected_ssid.csv'\n",
    "np.savetxt(file_path, sorted(list(imp_ssid_inter)), delimiter=',', fmt='%s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User @ Day labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = pd.read_csv(\n",
    "    # '/home/ali/Projects/Network/data/phonelab/data/user_day_ssid_connect_label.csv',\n",
    "    '/home/ali/Projects/Network/data/phonelab/data/user_day_location_connect_label.csv',\n",
    "    index_col=False,\n",
    "    header=None,\n",
    "    names=['label']\n",
    ").astype(int).values\n",
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(np.unique(labels)))\n",
    "print(np.unique(labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalize the matrix (Min-Max)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MaxAbsScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max_scaler = MaxAbsScaler()\n",
    "M_scaled = min_max_scaler.fit_transform(M)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# min_max_scaler = MaxAbsScaler()\n",
    "# sM_scaled = min_max_scaler.fit_transform(sM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have 19,144 rows of data and 1176 SSID or location or 1,176 x 24 = 28,224 (SSID x Hour) spatio-temporal features.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standardize the matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standard_scaler = StandardScaler()\n",
    "# M_scaled= standard_scaler.fit_transform(M)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute covariance matrix\n",
    "\n",
    "Due to high computation, is executed on CC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cov_mat = np.cov(M_scaled.T)\n",
    "# eigen_vals, eigen_vecs = np.linalg.eig(cov_mat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load eigen vectors and eigen values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sE = sparse.load_npz('/home/ali/Projects/Network/data/phonelab/data/user_ssid_connect_eigen_vecs_sparse.npz')\n",
    "\n",
    "# sE = sparse.load_npz('/home/ali/Projects/Network/data/phonelab/data/user_day_ssid_connect_eigen_vecs_sparse.npz')\n",
    "\n",
    "# sE = sparse.load_npz('/home/ali/Projects/Network/data/phonelab/data/user_location_connect_eigen_vecs_sparse.npz')\n",
    "\n",
    "# sE = sparse.load_npz('/home/ali/Projects/Network/data/phonelab/data/user_day_location_connect_eigen_vecs_sparse.npz')\n",
    "\n",
    "# Selected SSID\n",
    "# sE = sparse.load_npz('/home/ali/Projects/Network/data/phonelab/data/selected_2/user_day_ssid_connect_eigen_vecs.npz')\n",
    "\n",
    "# E = sE.toarray()\n",
    "\n",
    "# V = np.load('/home/ali/Projects/Network/data/phonelab/data/user_ssid_connect_eigen_vals.npy')\n",
    "\n",
    "# V = np.load('/home/ali/Projects/Network/data/phonelab/data/user_day_ssid_connect_eigen_vals.npy')\n",
    "\n",
    "# V = np.load('/home/ali/Projects/Network/data/phonelab/data/user_location_connect_eigen_vals.npy')\n",
    "\n",
    "# V = np.load('/home/ali/Projects/Network/data/phonelab/data/user_day_location_connect_eigen_vals.npy')\n",
    "\n",
    "# Selected SSID\n",
    "V = np.load(\n",
    "    '/home/ali/Projects/Network/data/phonelab/data/selected_2/user_day_ssid_connect_eigen_vals.npy'\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variance ratio\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate accumulative variance ratio of eigen values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot = sum(V)\n",
    "var_exp = np.array([(i / tot) for i in sorted(V, reverse=True)])\n",
    "cum_var_exp = np.cumsum(var_exp).real"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Look at the first 10 pricipal component variance ratio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "var_exp[:10].real"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Look at the first 10 cumilative pricipal component variance ratio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cum_var_exp[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the number of needed components to represent 5,10,15, ... 100 % of data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_at = cum_var_exp.searchsorted(np.linspace(0, 1, 20, endpoint=False))\n",
    "split_at"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_var = [\n",
    "    '{:.0f}'.format(x * 100)\n",
    "    for x in np.linspace(0, 1, 20, endpoint=False)[20 - len(set(split_at)):]\n",
    "]\n",
    "cps = [x + 1 for x in sorted(list(set(split_at)))]\n",
    "cp_var = list(zip(split_var, cps))\n",
    "cp_var"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of components -> ratio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "    f'80 % of data can be captured using {split_at[-4] + 1} number of components or {np.round((split_at[-4] + 1) / len(V) * 100, 2)} % of features.'\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Exclude repeated ones because they are give us same number of components\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_at[1:]\n",
    "# split_at[2:]\n",
    "# split_at[3:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the variance ration at the identified component numbers  \n",
    "Note they should be increasing within 5% ratio ...\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cum_var_exp[split_at][1:]\n",
    "# cum_var_exp[split_at][2:]\n",
    "# cum_var_exp[split_at][3:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize variance ratio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(16, 12), dpi=400)\n",
    "plt.bar(\n",
    "    # range(1, len(V) + 1),\n",
    "    # var_exp.real,\n",
    "    range(1, 251),\n",
    "    var_exp[:250].real,\n",
    "    alpha=0.5,\n",
    "    align='center',\n",
    "    label='Explained Variance'\n",
    ")\n",
    "plt.ylabel('Explained Variance Ratio')\n",
    "plt.xlabel('Number of Principal Component')\n",
    "plt.legend(loc='best')\n",
    "PATH = os.path.join(DATA, 'pca_variance_ratio.pdf')\n",
    "# plt.savefig(PATH, dpi=400)\n",
    "plt.ylim((0, 0.055))\n",
    "plt.title('Variance Ratio')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(4, 3), dpi=400)\n",
    "plt.step(\n",
    "    range(1,\n",
    "          len(V) + 1),\n",
    "    cum_var_exp.real,\n",
    "    # range(1, 1051),\n",
    "    # cum_var_exp[0:1050].real,\n",
    "    where='mid',\n",
    "    label='Cumulative Explained Variance'\n",
    ")\n",
    "# plt.plot(cum_var_exp)\n",
    "plt.ylabel('Explained Variance Ratio')\n",
    "plt.xlabel('Number of Principal Component')\n",
    "plt.legend(loc='best')\n",
    "PATH = os.path.join(DATA, 'pca_variance_ratio.pdf')\n",
    "# plt.savefig(PATH, dpi=400)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(4, 3), dpi=400)\n",
    "plt.bar(\n",
    "    range(1, 251),\n",
    "    var_exp[:250].real,\n",
    "    alpha=0.5,\n",
    "    align='center',\n",
    "    label='Explained Variance'\n",
    ")\n",
    "plt.ylabel('Explained Variance Ratio')\n",
    "plt.xlabel('Number of Principal Component')\n",
    "plt.legend(loc='best')\n",
    "PATH = os.path.join(DATA, 'pca_variance_ratio.pdf')\n",
    "# plt.savefig(PATH, dpi=400)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(4, 3), dpi=400)\n",
    "plt.step(\n",
    "    range(1, 251),\n",
    "    cum_var_exp[0:250].real,\n",
    "    where='mid',\n",
    "    label='Cumulative Explained Variance'\n",
    ")\n",
    "# plt.plot(cum_var_exp)\n",
    "plt.ylabel('Explained Variance Ratio')\n",
    "plt.xlabel('Number of Principal Component')\n",
    "plt.legend(loc='best')\n",
    "PATH = os.path.join(DATA, 'pca_variance_ratio.pdf')\n",
    "# plt.savefig(PATH, dpi=400)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sort eigen vectors and eigen values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eigen_pairs = [(np.abs(V[i]), E[:, i]) for i in range(len(V))]\n",
    "eigen_pairs.sort(key=lambda k: k[0], reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# idx = eigen_vals.argsort()[::-1]\n",
    "# eigen_vals = eigen_vals[idx]\n",
    "# eigen_vecs = eigen_vecs[:,idx]\n",
    "# eigen_pairs = (eigen_vals, eigen_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "component_number = 250\n",
    "W = np.concatenate(\n",
    "    [eigen_pairs[i][1][:, np.newaxis] for i in range(component_number)],\n",
    "    axis=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "W.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# M_new = np.dot(M_minmax, W)\n",
    "M_new = M_minmax @ W\n",
    "\n",
    "# file_name = f'/home/ali/Projects/Network/data/phonelab/data/user_ssid_connect_transformed_{component_number}.npy'\n",
    "# np.save(file_name, M_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "M_new.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = '/home/ali/Projects/Network/data/phonelab/data/user_day_ssid_connect_label.csv'\n",
    "M_labels = pd.read_csv(PATH, index_col=False, header=None,\n",
    "                       names=['label']).iloc[:, 0]\n",
    "# y = pd.read_csv(PATH, index_col=False, header=None, names=['label']).astype(int).values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate the distribution of classes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "M_labels.value_counts().plot(kind='bar', figsize=(32, 6), rot=90)\n",
    "# plt.xticks(rotation=90)\n",
    "# Remoce odd X ticks\n",
    "ax = plt.gca()\n",
    "for label in ax.get_xaxis().get_ticklabels()[::2]:\n",
    "    label.set_visible(False)\n",
    "plt.xlabel('Label (i.e. User)', labelpad=10)\n",
    "plt.ylabel('Number of Instance (i.e. Days of Having Data)', labelpad=10)\n",
    "plt.title('Dataset Label Frequency', pad=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_cmap = clr.LinearSegmentedColormap.from_list(\n",
    "    'my_cmap', [\n",
    "        (0, 'skyblue'), (0.25, 'aqua'), (0.5, 'royalblue'), (0.75, 'darkblue'),\n",
    "        (1, 'black')\n",
    "    ],\n",
    "    N=len(M_labels.unique())\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_cmap = sns.color_palette(\n",
    "    'rocket', n_colors=len(M_labels.unique()), as_cmap=True\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def discrete_cmap(N, base_cmap=None):\n",
    "    '''Create an N-bin discrete colormap from the specified input map'''\n",
    "\n",
    "    # Note that if base_cmap is a string or None, you can simply do\n",
    "    #    return plt.cm.get_cmap(base_cmap, N)\n",
    "    # The following works for string, None, or a colormap instance:\n",
    "\n",
    "    base = plt.cm.get_cmap(base_cmap)\n",
    "    color_list = base(np.linspace(0, 1, N))\n",
    "    cmap_name = base.name + str(N)\n",
    "    return base.from_list(cmap_name, color_list, N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = len(M_labels.unique())\n",
    "plt.scatter(\n",
    "    M_new[:, 0],\n",
    "    M_new[:, 1],\n",
    "    c=M_labels,\n",
    "    edgecolor='none',\n",
    "    alpha=0.5,\n",
    "    s=10,\n",
    "    # cmap=plt.cm.get_cmap('spectral', 10) # Only if we have 10 classes\n",
    "    # cmap=plt.cm.get_cmap('viridis')\n",
    "    cmap=discrete_cmap(N, 'cubehelix')\n",
    "    # cmap = my_cmap\n",
    ")\n",
    "plt.xlabel('Component 1')\n",
    "plt.ylabel('Component 2')\n",
    "# plt.colorbar()\n",
    "cbar = plt.colorbar(ticks=range(0, N, 10))\n",
    "cbar.ax.tick_params(labelsize=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(\n",
    "    np.concatenate((M_new.real, M_labels[:, np.newaxis]), axis=1),\n",
    "    columns=['cp1', 'cp2', 'label']\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering on Dataset After PCA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create dataframe from transformed data after PCA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(\n",
    "    data=M.real, columns=['pc' + str(i) for i in range(M.shape[1])]\n",
    ")\n",
    "df['target'] = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A few of classes (i.e. labels) has only 1 instance which could affect the prediction. So we can (optionally) remove these to improve the classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "M_labels_counts = df['target'].value_counts()\n",
    "print(M_labels_counts[M_labels_counts < 2].index)\n",
    "\n",
    "df.drop(\n",
    "    df[df['target'].isin(M_labels_counts[M_labels_counts < 2].index)].index,\n",
    "    inplace=True\n",
    ")\n",
    "\n",
    "# M_labels_counts = df['target'].value_counts()\n",
    "# print(M_labels_counts[M_labels_counts < 2].index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split dataset into training and test sets (while considering distribution of classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_set, test_set = train_test_split(\n",
    "    df, test_size=0.2, random_state=1, stratify=df['target']\n",
    ")\n",
    "\n",
    "# print(train_set['target'].value_counts() / len(train_set))\n",
    "# print(test_set['target'].value_counts() / len(test_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another way of dataset split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import StratifiedShuffleSplit\n",
    "\n",
    "# strat_split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=1)\n",
    "# for train_index, test_index in strat_split.split(df, df['target']):\n",
    "#     train_set = df.iloc[train_index]\n",
    "#     test_set = df.iloc[test_index]\n",
    "\n",
    "# print(train_set['target'].value_counts() / len(train_set))\n",
    "# print(test_set['target'].value_counts() / len(test_set))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "print(df.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_set, test_set = train_test_split(df, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = train_set.iloc[:, :-1]\n",
    "y_train = train_set.iloc[:, -1]\n",
    "X_test = test_set.iloc[:, :-1]\n",
    "y_test = test_set.iloc[:, -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x_test.iloc[0].values.reshape(1, -1))\n",
    "print(y_test.iloc[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_parameters = [\n",
    "    {\n",
    "        'kernel': ['linear'],\n",
    "        'C': [1]\n",
    "    }, {\n",
    "        'kernel': ['rbf'],\n",
    "        'C': [1000],\n",
    "        'gamma': [0.1]\n",
    "    }\n",
    "]\n",
    "gs = GridSearchCV(\n",
    "    SVC(),\n",
    "    param_grid=grid_parameters,\n",
    "    scoring='precision_macro',\n",
    "    cv=5,\n",
    "    n_jobs=-1\n",
    ")\n",
    "gs.fit(X_train, y_train)\n",
    "y_pred = gs.predict(X_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report = pd.DataFrame(classification_report(y_test, y_pred,\n",
    "                                            output_dict=True)).transpose()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(report)\n",
    "# report.loc['micro avg', 'precision':'f1-score']\n",
    "report.loc['macro avg', ['precision', 'recall', 'f1-score']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = [cp_var[cp], cp]\n",
    "sc.extend(report.loc['micro avg', 'precision':'f1-score'])\n",
    "sc.append(gs.refit_time_)\n",
    "scores.append(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_clf = SVC(gamma='auto', random_state=42)\n",
    "svm_clf.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_clf.predict(x_test.iloc[0].values.reshape(1, -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# classifier = RandomForestClassifier(max_depth=2, random_state=0)\n",
    "classifier = RandomForestClassifier(max_depth=20, random_state=0)\n",
    "classifier.fit(x_train, y_train)\n",
    "\n",
    "# Predicting the Test set results\n",
    "y_pred = classifier.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "print(cm)\n",
    "print(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = len(M_labels.unique())\n",
    "base = plt.cm.get_cmap('cubehelix')\n",
    "color_list = base(np.linspace(0, 1, N))\n",
    "# print(color_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA(n_components=2, svd_solver='randomized')\n",
    "\n",
    "# M_new = pca.fit_transform(M)\n",
    "M_new = pca.fit_transform(M_minmax)\n",
    "\n",
    "print(pca.components_.shape)\n",
    "print(pca.explained_variance_ratio_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = np.arange(0, M_new.shape[0])\n",
    "plt.scatter(\n",
    "    M_new[:, 0],\n",
    "    M_new[:, 1],\n",
    "    c=target,\n",
    "    edgecolor='none',\n",
    "    alpha=0.5,\n",
    "    # cmap=plt.cm.get_cmap('spectral', 10) # Only if we have 10 classes\n",
    "    cmap=plt.cm.get_cmap('viridis')\n",
    ")\n",
    "plt.xlabel('Component 1')\n",
    "plt.ylabel('Component 2')\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "principalDf = pd.DataFrame(data=M_new, columns=['pc1', 'pc2'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(4, 4))\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "ax.set_xlabel('Principal Component 1')\n",
    "ax.set_ylabel('Principal Component 2')\n",
    "# ax.set_xlabel('Principal Component 1', fontsize = 15)\n",
    "# ax.set_ylabel('Principal Component 2', fontsize = 15)\n",
    "ax.set_title('2 component PCA')\n",
    "# ax.set_title('2 component PCA', fontsize = 20)\n",
    "ax.scatter(principalDf['pc1'], principalDf['pc2'], s=1)\n",
    "# ax.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA().fit(M_minmax.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca.explained_variance_ratio_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca_com = []\n",
    "pca_var = []\n",
    "for dim in np.arange(1, M_minmax.shape[0]):\n",
    "    if dim <= M_minmax.shape[0]:\n",
    "        svd_solver = 'full'\n",
    "    else:\n",
    "        svd_solver = 'randomized'\n",
    "    pca = PCA(n_components=dim, svd_solver=svd_solver).fit(M_minmax)\n",
    "    pca_com.append(dim)\n",
    "    pca_var.append(sum(pca.explained_variance_ratio_))\n",
    "plt.plot(pca_com, pca_var)\n",
    "plt.xlabel('Number of components')\n",
    "plt.ylabel('Cumulative explained variance')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pca = PCA().fit(M_minmax)\n",
    "plt.plot(np.cumsum(pca.explained_variance_ratio_))\n",
    "plt.xlabel('Number of components')\n",
    "plt.ylabel('Cumulative explained variance')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "170d76527af6bb928a789dc7d6821516e71a979b7f8112ecc205a562846d3426"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}