FB_featurization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "168QPRizVFFg"
   },
   "source": [
    "<p style=\"font-size:32px;text-align:center\"> <b>Social network Graph Link Prediction - Facebook Challenge</b> </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q8lS7fVyVFFl"
   },
   "outputs": [],
   "source": [
    "#Importing Libraries\n",
    "# please do go through this python notebook: \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import csv\n",
    "import pandas as pd#pandas to create small dataframes \n",
    "import datetime #Convert to unix time\n",
    "import time #Convert to unix time\n",
    "# if numpy is not installed already : pip3 install numpy\n",
    "import numpy as np#Do aritmetic operations on arrays\n",
    "# matplotlib: used to plot graphs\n",
    "import matplotlib\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns#Plots\n",
    "from matplotlib import rcParams#Size of plots  \n",
    "from sklearn.cluster import MiniBatchKMeans, KMeans#Clustering\n",
    "import math\n",
    "import pickle\n",
    "import os\n",
    "# to install xgboost: pip3 install xgboost\n",
    "import xgboost as xgb\n",
    "\n",
    "import warnings\n",
    "import networkx as nx\n",
    "import pdb\n",
    "import pickle\n",
    "from pandas import HDFStore,DataFrame\n",
    "from pandas import read_hdf\n",
    "from scipy.sparse.linalg import svds, eigs\n",
    "import gc\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1znHayNeVFFt"
   },
   "source": [
    "# 1. Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Uq9HbHwEVFFv",
    "outputId": "b2aa525a-93d3-47c3-8216-416a811bc812"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "please run the FB_EDA.ipynb or download the files from drive\n"
     ]
    }
   ],
   "source": [
    "if os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_edatrain_pos_after_eda.csv\"):\n",
    "    train_graph=nx.read_edgelist(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\train_pos_after_eda.csv\",delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n",
    "    print(nx.info(train_graph))\n",
    "else:\n",
    "    print(\"please run the FB_EDA.ipynb or download the files from drive\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HmlUa64tVFF7"
   },
   "source": [
    "# 2. Similarity measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ivVMUMiWVFF9"
   },
   "source": [
    "## 2.1 Jaccard Distance:\n",
    "http://www.statisticshowto.com/jaccard-index/\n",
    "\n",
    "Construct set of followers. Example followers of U1 and followers of U2. Say U1 = {U5,U3,U4}, U2 = {U3,U4,U6}\n",
    "\n",
    "Since U1 and U2 have common followers so very high chance U1 and U2 will follow each other. \n",
    "\n",
    "So Jaccard distance = common folloewrs/total followers. So in above example Jaccard distance  = 2/4,it can be used in sets as well other than graph theory. \n",
    "\n",
    "I can do the samer thing for followee also like I did for followers. \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NoWCYuRBVFF_"
   },
   "source": [
    "\\begin{equation}\n",
    "j = \\frac{|X\\cap Y|}{|X \\cup Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Seo4z5SnVFGB"
   },
   "outputs": [],
   "source": [
    "#for followees\n",
    "def jaccard_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n",
    "    except:\n",
    "        return 0\n",
    "    return sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Oa9FMlS8VFGF",
    "outputId": "426a6833-1631-4024-c24a-d21ae7686472"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#one test case\n",
    "print(jaccard_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Gf8njOv6VFGK",
    "outputId": "8ba07727-a0ab-498e-819f-0d310876191c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#node 1635354 not in graph \n",
    "print(jaccard_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LO-a5ZkKVFGO"
   },
   "outputs": [],
   "source": [
    "#for followers\n",
    "def jaccard_for_followers(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                 (len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DlbX2t0jVFGQ",
    "outputId": "7e4b4536-442a-4b0c-ae02-fb442c1955db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(jaccard_for_followers(273084,470294))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OgeBW2LMVFGU",
    "outputId": "1e12fabe-d990-4506-bb6b-c86b01d1b0af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#node 1635354 not in graph \n",
    "print(jaccard_for_followees(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MnH2my2UVFGX"
   },
   "source": [
    "\n",
    "## 2.2 Cosine distance\n",
    "\n",
    "\n",
    "We did learn cosine distance in two vectors. Instead of vector, if we have set.. like example above of graph.\n",
    "\n",
    "if we have more overlap in sets, Cosine dist will be high. very similar to jaccard distance.  \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XNvdBGS2VFGY"
   },
   "source": [
    "\\begin{equation}\n",
    "CosineDistance = \\frac{|X\\cap Y|}{|X|\\cdot|Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Iznz67EdVFGZ"
   },
   "outputs": [],
   "source": [
    "#for followees\n",
    "def cosine_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H55ALjkMVFGc",
    "outputId": "531fceba-60f4-4e6b-97f4-f37733dc468f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "q0RGKgJFVFGf",
    "outputId": "41202fc6-f4aa-4a1d-d8f6-84f960a3fbba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1635354))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KJ_yGxA0VFGj"
   },
   "outputs": [],
   "source": [
    "def cosine_for_followers(a,b):\n",
    "    try:\n",
    "        \n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                     (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "75QrFJb6VFGm",
    "outputId": "f01e0558-f1e3-465f-ab14-0e4ca764f4aa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(2,470294))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-ut4k_F0VFGq",
    "outputId": "8bc9607a-7262-43e2-9de8-f71d276762fc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DaIHhWh6VFGv"
   },
   "source": [
    "## 3. Ranking Measures\n",
    "\n",
    "Very popular algo, it was used by good for ranking. Web page will have large value if lots of links are linked to it in the sense lots of web ages recieve links from that webpage. So would have higher value.  It gives direct graph. each vertex UI will give score to show how important the vertex is. Page rank will give reloave score of each vertex. so user has lot of  followers\n",
    "so that vertex will have higher value.  Like Bill gates  has high followers so that vertex will have high value.\n",
    "\n",
    "it is probabilkity value that random user will end up on this web page. \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6nfV1SprVFGx"
   },
   "source": [
    "https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.algorithms.link_analysis.pagerank_alg.pagerank.html\n",
    "\n",
    "PageRank computes a ranking of the nodes in the graph G based on the structure of the incoming links.\n",
    "\n",
    "<img src='PageRanks-Example.jpg'/>\n",
    "\n",
    "Mathematical PageRanks for a simple network, expressed as percentages. (Google uses a logarithmic scale.) Page C has a higher PageRank than Page E, even though there are fewer links to C; the one link to C comes from an important page and hence is of high value. If web surfers who start on a random page have an 85% likelihood of choosing a random link from the page they are currently visiting, and a 15% likelihood of jumping to a page chosen at random from the entire web, they will reach Page E 8.1% of the time. <b>(The 15% likelihood of jumping to an arbitrary page corresponds to a damping factor of 85%.) Without damping, all web surfers would eventually end up on Pages A, B, or C, and all other pages would have PageRank zero. In the presence of damping, Page A effectively links to all pages in the web, even though it has no outgoing links of its own.</b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GkkfYYZ6VFGy"
   },
   "source": [
    "## 3.1 Page Ranking\n",
    "\n",
    "https://en.wikipedia.org/wiki/PageRank\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AtvqwZ34VFGy"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\page_rank.p\"):\n",
    "    pr = nx.pagerank(train_graph, alpha=0.85)\n",
    "    pickle.dump(pr,open(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\page_rank.p\",'wb'))\n",
    "else:\n",
    "    pr = pickle.load(open(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\page_rank.p\",'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lXGKYYf6VFG2",
    "outputId": "bb3d1b7a-81f9-44ab-dbe7-3214ccd47179"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 1.6556497245737814e-07\n",
      "max 2.7098251341935827e-05\n",
      "mean 5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "# values calculated for each vertex.\n",
    "\n",
    "print('min',pr[min(pr, key=pr.get)])\n",
    "print('max',pr[max(pr, key=pr.get)])\n",
    "print('mean',float(sum(pr.values())) / len(pr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5xwlah4oVFG4",
    "outputId": "992fdfad-7ff6-4626-c9ee-d9bce220a680"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "#for imputing to nodes which are not there in Train data.  if no vertex, better to use mean page rank value.\n",
    "mean_pr = float(sum(pr.values())) / len(pr)\n",
    "print(mean_pr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HhPbSL1tVFG7"
   },
   "source": [
    "# 4. Other Graph Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AgsorCl7VFG8"
   },
   "source": [
    "## 4.1 Shortest path:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "E7teH2LCVFG9"
   },
   "source": [
    "Getting Shortest path between twoo nodes, if nodes have direct path i.e directly connected then we are removing that edge and calculating path. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RA076ovzVFG9"
   },
   "outputs": [],
   "source": [
    "#if has direct edge then deleting that edge and calculating shortest path\n",
    "def compute_shortest_path_length(a,b):\n",
    "    p=-1\n",
    "    try:\n",
    "        if train_graph.has_edge(a,b):\n",
    "            train_graph.remove_edge(a,b)\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "            train_graph.add_edge(a,b)\n",
    "        else:\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "        return p\n",
    "    except:\n",
    "        return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AxnKId11VFG_",
    "outputId": "15ca223a-6a04-4549-d010-54619b472a9e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#testing\n",
    "compute_shortest_path_length(77697, 826021)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0huWCNtRVFHC",
    "outputId": "6debfa4f-2067-48bc-84b3-ab86e2d9dea6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#testing\n",
    "compute_shortest_path_length(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "baE_95bzVFHF"
   },
   "source": [
    "## 4.2 Checking for same community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "15CIQqAbVFHG"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_graph' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-ef219cf9bbe2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#getting weekly connected edges from graph\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mwcc\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweakly_connected_components\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mbelongs_to_same_wcc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mtrain_graph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhas_edge\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'train_graph' is not defined"
     ]
    }
   ],
   "source": [
    "#getting weekly connected edges from graph \n",
    "wcc=list(nx.weakly_connected_components(train_graph))\n",
    "def belongs_to_same_wcc(a,b):\n",
    "    index = []\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    if train_graph.has_edge(a,b):\n",
    "            for i in wcc:\n",
    "                if a in i:\n",
    "                    index= i\n",
    "                    break\n",
    "            if (b in index):\n",
    "                train_graph.remove_edge(a,b)\n",
    "                if compute_shortest_path_length(a,b)==-1:\n",
    "                    train_graph.add_edge(a,b)\n",
    "                    return 0\n",
    "                else:\n",
    "                    train_graph.add_edge(a,b)\n",
    "                    return 1\n",
    "            else:\n",
    "                return 0\n",
    "    else:\n",
    "            for i in wcc:\n",
    "                if a in i:\n",
    "                    index= i\n",
    "                    break\n",
    "            if(b in index):\n",
    "                return 1\n",
    "            else:\n",
    "                return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fAzOHtCFVFHI",
    "outputId": "2b043a87-b460-42bf-f37e-4c04bbed6586"
   },
   "outputs": [],
   "source": [
    "belongs_to_same_wcc(861, 1659750)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HMdYpPuGVFHK",
    "outputId": "2005e22c-b60f-48d7-839b-650bf97cae35"
   },
   "outputs": [],
   "source": [
    "belongs_to_same_wcc(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "q74nth0OVFHN"
   },
   "source": [
    "## 4.3 Adamic/Adar Index:\n",
    "Adamic/Adar measures is defined as inverted sum of degrees of common neighbours for given two vertices.\n",
    "$$A(x,y)=\\sum_{u \\in N(x) \\cap N(y)}\\frac{1}{log(|N(u)|)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CeS98LI5VFHO"
   },
   "outputs": [],
   "source": [
    "#adar index\n",
    "def calc_adar_in(a,b):\n",
    "    sum=0\n",
    "    try:\n",
    "        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n",
    "        if len(n)!=0:\n",
    "            for i in n:\n",
    "                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n",
    "            return sum\n",
    "        else:\n",
    "            return 0\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KezFeRmyVFHQ",
    "outputId": "2f9c0e11-02d9-4f28-d67a-65e3d4943e99"
   },
   "outputs": [],
   "source": [
    "calc_adar_in(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vj_m89bBVFHV",
    "outputId": "68a0a099-2954-402f-c80f-6d436ffa1aba"
   },
   "outputs": [],
   "source": [
    "calc_adar_in(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pBUudhFAVFHY"
   },
   "source": [
    "## 4.4 Is persion was following back:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j_mwmopLVFHZ"
   },
   "outputs": [],
   "source": [
    "def follows_back(a,b):\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LdjUXIfbVFHb",
    "outputId": "ed3d8640-9834-4a95-e712-804292da70e9"
   },
   "outputs": [],
   "source": [
    "follows_back(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PmZtL65YVFHf",
    "outputId": "18ea6fe2-3f96-42c0-d116-ecb76ddba4b5"
   },
   "outputs": [],
   "source": [
    "follows_back(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "29Vrq2EXVFHi"
   },
   "source": [
    "## 4.5 Katz Centrality:\n",
    "https://en.wikipedia.org/wiki/Katz_centrality\n",
    "\n",
    "https://www.geeksforgeeks.org/katz-centrality-centrality-measure/\n",
    " Katz centrality computes the centrality for a node \n",
    "    based on the centrality of its neighbors. It is a \n",
    "    generalization of the eigenvector centrality. The\n",
    "    Katz centrality for node `i` is\n",
    " \n",
    "$$x_i = \\alpha \\sum_{j} A_{ij} x_j + \\beta,$$\n",
    "where `A` is the adjacency matrix of the graph G \n",
    "with eigenvalues $$\\lambda$$.\n",
    "\n",
    "The parameter $$\\beta$$ controls the initial centrality and \n",
    "\n",
    "$$\\alpha < \\frac{1}{\\lambda_{max}}.$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CN5OSqrkVFHj"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\katz.p'):\n",
    "    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n",
    "    pickle.dump(katz,open(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\katz.p','wb'))\n",
    "else:\n",
    "    katz = pickle.load(open(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\katz.p\",'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gcU83vw7VFHm",
    "outputId": "05f49ad4-46fe-4cf6-f32a-2fe4846b0714"
   },
   "outputs": [],
   "source": [
    "print('min',katz[min(katz, key=katz.get)])\n",
    "print('max',katz[max(katz, key=katz.get)])\n",
    "print('mean',float(sum(katz.values())) / len(katz))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qcboIksiVFHt",
    "outputId": "99f52422-9edb-479a-d5d9-e33401160da7"
   },
   "outputs": [],
   "source": [
    "mean_katz = float(sum(katz.values())) / len(katz)\n",
    "print(mean_katz)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SRZqGFgYVFHx"
   },
   "source": [
    "## 4.6 Hits Score\n",
    "The HITS algorithm computes two numbers for a node. Authorities estimates the node value based on the incoming links. Hubs estimates the node value based on outgoing links.\n",
    "\n",
    "https://en.wikipedia.org/wiki/HITS_algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WXNHRdzUVFHz"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\hits.p\"):\n",
    "    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n",
    "    pickle.dump(hits,open(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\hits.p\",'wb'))\n",
    "else:\n",
    "    hits = pickle.load(open(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\hits.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PSUwSZBVVFH3",
    "outputId": "77448253-5409-4229-f0be-b8dbc14d7f46"
   },
   "outputs": [],
   "source": [
    "print('min',hits[0][min(hits[0], key=hits[0].get)])\n",
    "print('max',hits[0][max(hits[0], key=hits[0].get)])\n",
    "print('mean',float(sum(hits[0].values())) / len(hits[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZZtowOLZVFH6"
   },
   "source": [
    "# 5. Featurization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "o6NnRWmLVFH6"
   },
   "source": [
    "## 5. 1 Reading a sample of Data from both train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wgHje1UVVFH8"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "if os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\train_after_eda.csv\"):\n",
    "    filename = \"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\train_after_eda.csv\"\n",
    "    # you uncomment this line, if you dont know the lentgh of the file name\n",
    "    # here we have hardcoded the number of lines as 15100030\n",
    "    # n_train = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    n_train =  15100028\n",
    "    s = 100000 #desired sample size\n",
    "    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\n",
    "    #https://stackoverflow.com/a/22259008/4084039"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zOzuRFFlVFH-"
   },
   "outputs": [],
   "source": [
    "if os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\train_after_eda.csv\"):\n",
    "    filename = \"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\test_after_eda.csv\"\n",
    "    # you uncomment this line, if you dont know the lentgh of the file name\n",
    "    # here we have hardcoded the number of lines as 3775008\n",
    "    # n_test = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    n_test = 3775006\n",
    "    s = 50000 #desired sample size\n",
    "    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\n",
    "    #https://stackoverflow.com/a/22259008/4084039"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3D_SeUCOVFH_",
    "outputId": "322902a4-0420-4b99-8606-5fd0de4bbea4"
   },
   "outputs": [],
   "source": [
    "print(\"Number of rows in the train data file:\", n_train)\n",
    "print(\"Number of rows we are going to elimiate in train data are\",len(skip_train))\n",
    "print(\"Number of rows in the test data file:\", n_test)\n",
    "print(\"Number of rows we are going to elimiate in test data are\",len(skip_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pCisf6PpVFID",
    "outputId": "daf2af43-3f98-4466-ad99-03bc54464714"
   },
   "outputs": [],
   "source": [
    "df_final_train = pd.read_csv(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\train_after_eda.csv\", skiprows=skip_train, names=['source_node', 'destination_node'])\n",
    "df_final_train['indicator_link'] = pd.read_csv(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\train_y.csv\", skiprows=skip_train, names=['indicator_link'])\n",
    "print(\"Our train matrix size \",df_final_train.shape)\n",
    "df_final_train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tFn1RkdyVFIH",
    "outputId": "1ca99e70-6d2a-45f2-f51c-fd3b1211ad20"
   },
   "outputs": [],
   "source": [
    "df_final_test = pd.read_csv(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\after_eda\\\\test_after_eda.csv\", skiprows=skip_test, names=['source_node', 'destination_node'])\n",
    "df_final_test['indicator_link'] = pd.read_csv(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\after_eda\\\\test_y.csv\", skiprows=skip_test, names=['indicator_link'])\n",
    "print(\"Our test matrix size \",df_final_test.shape)\n",
    "df_final_test.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gIaOWDaDVFIJ"
   },
   "source": [
    "## 5.2 Adding a set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>jaccard_followers</li>\n",
    "<li>jaccard_followees</li>\n",
    "<li>cosine_followers</li>\n",
    "<li>cosine_followees</li>\n",
    "<li>num_followers_s</li>\n",
    "<li>num_followees_s</li>\n",
    "<li>num_followers_d</li>\n",
    "<li>num_followees_d</li>\n",
    "<li>inter_followers</li>\n",
    "<li>inter_followees</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2qTkOiBcVFIJ"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage1.h5'):\n",
    "    #mapping jaccrd followers to train and test data\n",
    "    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping jaccrd followees to train and test data\n",
    "    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "\n",
    "        #mapping jaccrd followers to train and test data\n",
    "    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping jaccrd followees to train and test data\n",
    "    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fz2eZpSnVFIL"
   },
   "outputs": [],
   "source": [
    "def compute_features_stage1(df_final):\n",
    "    #calculating no of followers followees for source and destination\n",
    "    #calculating intersection of followers and followees for source and destination\n",
    "    num_followers_s=[]\n",
    "    num_followees_s=[]\n",
    "    num_followers_d=[]\n",
    "    num_followees_d=[]\n",
    "    inter_followers=[]\n",
    "    inter_followees=[]\n",
    "    for i,row in df_final.iterrows():\n",
    "        try:\n",
    "            s1=set(train_graph.predecessors(row['source_node']))\n",
    "            s2=set(train_graph.successors(row['source_node']))\n",
    "        except:\n",
    "            s1 = set()\n",
    "            s2 = set()\n",
    "        try:\n",
    "            d1=set(train_graph.predecessors(row['destination_node']))\n",
    "            d2=set(train_graph.successors(row['destination_node']))\n",
    "        except:\n",
    "            d1 = set()\n",
    "            d2 = set()\n",
    "        num_followers_s.append(len(s1))\n",
    "        num_followees_s.append(len(s2))\n",
    "\n",
    "        num_followers_d.append(len(d1))\n",
    "        num_followees_d.append(len(d2))\n",
    "\n",
    "        inter_followers.append(len(s1.intersection(d1)))\n",
    "        inter_followees.append(len(s2.intersection(d2)))\n",
    "    \n",
    "    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VFc60kcRVFIN"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage1.h5\"):\n",
    "    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n",
    "    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n",
    "    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n",
    "    \n",
    "    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n",
    "    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n",
    "    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n",
    "    \n",
    "    hdf = HDFStore(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage1.h5\")\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage1.h5\", 'train_df',mode='r')\n",
    "    df_final_test = read_hdf(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage1.h5\", 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "go_e8hxxVFIO"
   },
   "source": [
    "## 5.3 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>adar index</li>\n",
    "<li>is following back</li>\n",
    "<li>belongs to same weakly connect components</li>\n",
    "<li>shortest path between source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LqB0Peg0VFIP"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage2.h5\"):\n",
    "    #mapping adar index on train\n",
    "    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "    #mapping adar index on test\n",
    "    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping followback or not on train\n",
    "    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping followback or not on test\n",
    "    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping same component of wcc or not on train\n",
    "    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    ##mapping same component of wcc or not on train\n",
    "    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping shortest path on train \n",
    "    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "    #mapping shortest path on test\n",
    "    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    hdf = HDFStore(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage2.h5\")\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage2.h5\", 'train_df',mode='r')\n",
    "    df_final_test = read_hdf(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage2.h5\", 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HJ8Dbma_VFIR"
   },
   "source": [
    "## 5.4 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>Weight Features\n",
    "    <ul>\n",
    "        <li>weight of incoming edges</li>\n",
    "        <li>weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges * weight of outgoing edges</li>\n",
    "        <li>2*weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + 2*weight of outgoing edges</li>\n",
    "    </ul>\n",
    "</li>\n",
    "<li>Page Ranking of source</li>\n",
    "<li>Page Ranking of dest</li>\n",
    "<li>katz of source</li>\n",
    "<li>katz of dest</li>\n",
    "<li>hubs of source</li>\n",
    "<li>hubs of dest</li>\n",
    "<li>authorities_s of source</li>\n",
    "<li>authorities_s of dest</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iVHI2jtNVFIS"
   },
   "source": [
    "#### Weight Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rXmUYF9FVFIT"
   },
   "source": [
    "In order to determine the similarity of nodes, an edge weight value was calculated between nodes. Edge weight decreases as the neighbor count goes up. Intuitively, consider one million people following a celebrity on a social network then chances are most of them never met each other or the celebrity. On the other hand, if a user has 30 contacts in his/her social network, the chances are higher that many of them know each other. \n",
    "`credit` - Graph-based Features for Supervised Link Prediction\n",
    "William Cukierski, Benjamin Hamner, Bo Yang"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Qzbs2no7VFIV"
   },
   "source": [
    "\\begin{equation}\n",
    "W = \\frac{1}{\\sqrt{1+|X|}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kkzUPrWaVFIV"
   },
   "source": [
    "it is directed graph so calculated Weighted in and Weighted out differently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FgNMzzTbVFIW",
    "outputId": "7e8e6d88-8bd6-45f6-f80e-82b093c18974"
   },
   "outputs": [],
   "source": [
    "#weight for source and destination of each link\n",
    "Weight_in = {}\n",
    "Weight_out = {}\n",
    "for i in  tqdm(train_graph.nodes()):\n",
    "    s1=set(train_graph.predecessors(i))\n",
    "    w_in = 1.0/(np.sqrt(1+len(s1)))\n",
    "    Weight_in[i]=w_in\n",
    "    \n",
    "    s2=set(train_graph.successors(i))\n",
    "    w_out = 1.0/(np.sqrt(1+len(s2)))\n",
    "    Weight_out[i]=w_out\n",
    "    \n",
    "#for imputing with mean\n",
    "mean_weight_in = np.mean(list(Weight_in.values()))\n",
    "mean_weight_out = np.mean(list(Weight_out.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AF4yPhIOVFIY"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage3.h5\"):\n",
    "    #mapping to pandas train\n",
    "    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "    #mapping to pandas test\n",
    "    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "\n",
    "    #some features engineerings on the in and out weights\n",
    "    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n",
    "    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n",
    "    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n",
    "    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n",
    "\n",
    "    #some features engineerings on the in and out weights\n",
    "    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n",
    "    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n",
    "    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n",
    "    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uhxzhQ9aVFIa"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage3.h5\"):\n",
    "    \n",
    "    #page rank for source and destination in Train and Test\n",
    "    #if anything not there in train graph then adding mean page rank \n",
    "    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "\n",
    "    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    #================================================================================\n",
    "\n",
    "    #Katz centrality score for source and destination in Train and test\n",
    "    #if anything not there in train graph then adding mean katz score\n",
    "    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "\n",
    "    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    #================================================================================\n",
    "\n",
    "    #Hits algorithm score for source and destination in Train and test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "\n",
    "    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    #Hits algorithm score for source and destination in Train and Test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "\n",
    "    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    hdf = HDFStore(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage3.h5\")\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage3.h5\", 'train_df',mode='r')\n",
    "    df_final_test = read_hdf(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage3.h5\", 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p6xkDfD-VFIb"
   },
   "source": [
    "## 5.5 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>SVD features for both source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WQO6E65eVFIc"
   },
   "outputs": [],
   "source": [
    "def svd(x, S):\n",
    "    try:\n",
    "        z = sadj_dict[x]\n",
    "        return S[z]\n",
    "    except:\n",
    "        return [0,0,0,0,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9sOyLwvNVFId"
   },
   "outputs": [],
   "source": [
    "#for svd features to get feature vector creating a dict node val and inedx in svd vector\n",
    "sadj_col = sorted(train_graph.nodes())\n",
    "sadj_dict = { val:idx for idx,val in enumerate(sadj_col)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zLSt8fGVVFIg"
   },
   "outputs": [],
   "source": [
    "Adj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "soq-VAHlVFIh",
    "outputId": "3f9bfb32-004f-4698-e415-469243250130"
   },
   "outputs": [],
   "source": [
    "U, s, V = svds(Adj, k = 6)\n",
    "print('Adjacency matrix Shape',Adj.shape)\n",
    "print('U Shape',U.shape)\n",
    "print('V Shape',V.shape)\n",
    "print('s Shape',s.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ls5fqLFhVFIm"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage4.h5\"):\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "\n",
    "    hdf = HDFStore(\"C:\\\\VipinML\\\\InputData\\\\FB\\\\fea_sample\\\\storage_sample_stage4.h5\")\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0-hBtlkzVFIn"
   },
   "outputs": [],
   "source": [
    "# prepared and stored the data from machine learning models\n",
    "# pelase check the FB_Models.ipynb"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "DaIHhWh6VFGv",
    "GkkfYYZ6VFGy",
    "AgsorCl7VFG8",
    "baE_95bzVFHF",
    "pBUudhFAVFHY",
    "29Vrq2EXVFHi",
    "SRZqGFgYVFHx"
   ],
   "name": "FB_featurization.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}