preproc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import sys\n",
    "import pandas as pd\n",
    "import math\n",
    "import matplotlib.pylab as plt\n",
    "import matplotlib as mpl\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import subprocess\n",
    "import re\n",
    "import itertools\n",
    "import mmh3\n",
    "import json\n",
    "import pickle\n",
    "from datetime import datetime as dt\n",
    "from datetime import time\n",
    "from random import randrange, gauss, shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ----------------- #\n",
    "## Handle raw data ##\n",
    "# ----------------- #\n",
    "\n",
    "def load_raw_month_data():\n",
    "    df_short = pd.read_csv('../Data/raw_data/short.csv', sep=\" \").loc[:,['#','user1','user2','timestamp']]\n",
    "    df_all = pd.read_csv('../Data/raw_data/all.csv', sep=\" \").loc[:,['#','user1','user2','timestamp']]\n",
    "\n",
    "    # Fix bad column name assignment\n",
    "    df_all.columns = ['user1','user2','timestamp','duration']\n",
    "    df_short.columns = ['user1','user2','timestamp','duration']\n",
    "    \n",
    "    return df_short, df_all\n",
    "\n",
    "\n",
    "def time_bin_network(df, window=3600):\n",
    "    \"\"\"Return iterator to partition network in 'window'-sized bins.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    df : network dataframe\n",
    "        Pandas dataframe with network loaded straight from csv.\n",
    "    window : bin-size integer\n",
    "        Size of network bins in seconds. This number controls\n",
    "        the amount of bins.\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    df_slice : iterator\n",
    "        Iterator object that yields one bin at a time.\n",
    "    \"\"\"\n",
    "    \n",
    "    try:\n",
    "        assert window >= 300\n",
    "        window += 0.01\n",
    "    except AssertionError:\n",
    "        raise AssertionError(\"\"\"'window' must be greater than, \\\n",
    "or equal to 300.\"\"\")\n",
    "        \n",
    "    # Get timestamp column and extreme values\n",
    "    col_t = df[\"timestamp\"]\n",
    "    min_t = min(col_t)\n",
    "    max_t = max(col_t)    \n",
    "    \n",
    "    # Get timespan and number of network splits\n",
    "    delta_time = max_t - min_t\n",
    "    \n",
    "    n_splits = int(math.ceil(delta_time/float(window)))\n",
    "\n",
    "    for i in range(n_splits):\n",
    "        lower_bound = df[\"timestamp\"] > min_t + i*window\n",
    "        upper_bound = df[\"timestamp\"] < min_t + (i+1)*window\n",
    "        df_slice = df[lower_bound][upper_bound]\n",
    "        yield df_slice\n",
    "        \n",
    "        \n",
    "def dump_binned_network(df, binsize, filename):\n",
    "    \"\"\"Store binned network into local file\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    df : network dataframe\n",
    "        Pandas dataframe with network\n",
    "    binsize : int\n",
    "        Size of bins in resulting network\n",
    "    filename : str\n",
    "        Name of resulting file\n",
    "    \"\"\"\n",
    "    \n",
    "    binned_network = list(time_bin_network(df, window=binsize))\n",
    "    \n",
    "    with open('../Data/processed_data/binned_networks/'+filename+'.pickle', 'w') as outfile:\n",
    "        pickle.dump(binned_network, outfile)\n",
    "        \n",
    "        \n",
    "def create_loadable_bins():\n",
    "    \"\"\"Store a series of binned networks locally\n",
    "    \"\"\"\n",
    "    \n",
    "    bins = [300, 900, 1800, 3600, 86400, 604800]\n",
    "    bin_names = ['5mins', '15mins', '30mins', 'hourly', 'daily', 'weekly']\n",
    "    dataframes = [df_short, df_all]\n",
    "    dataframe_names = ['_short', '_all']\n",
    "\n",
    "    for df_, dfn_ in zip(dataframes,dataframe_names):\n",
    "        for b,n in zip(bins,bin_names):\n",
    "            print dfn_,n\n",
    "            dump_binned_network(df_,b,n+dfn_)\n",
    "            \n",
    "def load_binned_network(kind,filename):\n",
    "    with open('../Data/processed_data/binned_networks/'+kind+'/'+filename+'.pickle', 'r') as infile:\n",
    "        return pickle.load(infile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ----------------------- #\n",
    "## Reformat layered data ##\n",
    "# ----------------------- #\n",
    "\n",
    "def network_reformat_multiplex(layers, halflife=-1):\n",
    "    \"\"\"Return multiplex representation of multiplex network\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    halflife : number\n",
    "        Halflife in seconds of relax-rate decay between layers.\n",
    "        Defaults to -1.\n",
    "    layers : pandas df formatted layers\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    net_file : string\n",
    "        A network string in multiplex format\n",
    "    int_to_hash : dict\n",
    "        Key-value pairs of node integer id and original hash id\n",
    "    \"\"\"\n",
    "    \n",
    "    # Infomap will only work with node ids as indices.\n",
    "    \n",
    "    # Get all node ids in original md5 hash values\n",
    "    nodes = set()\n",
    "    for l, df in enumerate(layers):\n",
    "        layer_nodes = set()\n",
    "        layer_nodes.update(df[\"user1\"])\n",
    "        layer_nodes.update(df[\"user2\"])\n",
    "        nodes.update(layer_nodes)\n",
    "        \n",
    "    \n",
    "    ##########################\n",
    "    ## Add vertices to file ##\n",
    "    ##########################\n",
    "    \n",
    "    out_file = \"*Vertices %d\" % len(nodes)\n",
    "    \n",
    "    # Node name book-keeping, and adding to file\n",
    "    hashid_to_intid = {}\n",
    "    intid_to_hashid = {}\n",
    "    for i,n in enumerate(nodes):\n",
    "        intid = i+1\n",
    "        hashid = str(n)\n",
    "        out_file += '\\n%d \"Node %s\" 1.0' % (intid,hashid)\n",
    "        hashid_to_intid[hashid] = intid\n",
    "        intid_to_hashid[intid] = hashid\n",
    "\n",
    "        \n",
    "    #############################\n",
    "    ## Add Intra-edges to file ##\n",
    "    #############################\n",
    "    \n",
    "    out_file += \"\\n*Multiplex\\n# Intra edges: layer node layer node weight\"\n",
    "    \n",
    "    for l, df in enumerate(layers):\n",
    "        user1 = df[\"user1\"]\n",
    "        user2 = df[\"user2\"]\n",
    "            \n",
    "        edges = zip(user1, user2)\n",
    "        \n",
    "        # Add weights. REDUNDANT FOR 5MINS TIMESLICES BECAUSE PPL ONLY MEET ONCE HERE.\n",
    "        edges = [(e[0],e[1],w) for e,w in Counter(edges).items()]\n",
    "        \n",
    "        # Add Intra-edges to file\n",
    "        for i,j,w in edges:\n",
    "            out_file += '\\n%d %s %d %s %d' % (l+1,hashid_to_intid[i], l+1,hashid_to_intid[j],w) #+1 because 1 is first layer index\n",
    "            \n",
    "    \n",
    "    #############################\n",
    "    ## Add Inter-edges to file ##\n",
    "    #############################\n",
    "    \n",
    "    out_file += \"# Intra edges: layer node layer node weight\"\n",
    "    \n",
    "    # Infinte halflife (represented as -1)\n",
    "    if halflife == -1:\n",
    "        return out_file\n",
    "    \n",
    "    # Relax decay function\n",
    "    def N(t):\n",
    "        tau = halflife/np.log(2)\n",
    "        return np.exp(-t/float(tau))\n",
    "                        \n",
    "    for l1, df1 in enumerate(layers):\n",
    "        nodes1 = set(list(df1['user1'].values)+list(df1['user2'].values))\n",
    "        for l2, df2 in enumerate(layers):    \n",
    "            if not l2 > l1:\n",
    "                continue   \n",
    "            \n",
    "            nodes2 = set(list(df2['user1'].values)+list(df2['user2'].values))\n",
    "            common_nodes = nodes1 & nodes2\n",
    "            time_diff = df2['timestamp'].values[0] - df1['timestamp'].values[0]\n",
    "            \n",
    "            for n in common_nodes:\n",
    "                out_file += '\\n%d %s %d %s %f' % (l1+1,hashid_to_intid[n],l2+1,hashid_to_intid[n],N(time_diff))\n",
    "    \n",
    "    return out_file, intid_to_hashid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ----------------------- #\n",
    "## Run Infomap algorithm ##\n",
    "# ----------------------- #\n",
    "\n",
    "def community_detection_multiplex(layers, halflife=2400):\n",
    "    \"\"\"Run multiplex community detection because Python implementation has no docs\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    halflife : number\n",
    "        Halflife in seconds of relax-rate decay between layers.\n",
    "        Defaults to -1.\n",
    "    layers : layers (pandas dfs)\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    communities : list of lists\n",
    "    \"\"\"\n",
    "    \n",
    "    def parse_communities():\n",
    "        with open('output/'+multiplex_network_filename+\"_expanded.clu\", 'r') as infile:\n",
    "            multiplex_network_clusters = infile.read()\n",
    "\n",
    "        # Get layers, nodes and clusters from _extended.clu file\n",
    "        la_no_clu = re.findall(r'\\d+ \\d+ \\d+ \\d\\.\\d+', multiplex_network_clusters) # [\"30 1 2 0.00800543\",...]\n",
    "        la_no_clu = [tuple(i.split()) for i in la_no_clu]\n",
    "\n",
    "        communities_json = {}\n",
    "        for layer, node, cluster, _ in la_no_clu:\n",
    "            try:\n",
    "                communities_json[int(layer)].add((intid_to_hashid[int(node)], int(cluster)))\n",
    "            except KeyError:\n",
    "                # Will run once for every layer\n",
    "                communities_json.update({int(layer): {(intid_to_hashid[int(node)], int(cluster))}})\n",
    "\n",
    "        return communities_json\n",
    "\n",
    "    \n",
    "    # Get network in mutliplex string format and define filename\n",
    "    multiplex_network, intid_to_hashid = network_reformat_multiplex(layers, halflife=halflife)\n",
    "    multiplex_network_filename = 'multiplex-network'\n",
    "\n",
    "    # Store locally\n",
    "    with open(\"input/\"+multiplex_network_filename+\".net\", 'w') as outfile:\n",
    "        outfile.write(multiplex_network)\n",
    "    \n",
    "    # Run Infomap for multiplex network\n",
    "    subprocess.call(['./Infomap/Infomap', 'input/'+multiplex_network_filename+\".net\", \n",
    "                     'output/', '-i', 'multiplex', '--overlapping', '--map', '--clu', '--tree', '--expanded'])\n",
    "    \n",
    "    parsed_communities = parse_communities()\n",
    "    \n",
    "    hash_clu = [item for sublist in parsed_communities.values() for item in sublist]\n",
    "    communities = dict()\n",
    "    for key, group in itertools.groupby(hash_clu, lambda x: x[1]):\n",
    "        for thing in group:\n",
    "            try:\n",
    "                communities[key].append(thing[0])\n",
    "            except KeyError:\n",
    "                communities[thing[1]] = [thing[0]]\n",
    "    communities = dict((k,set(v)) for k,v in communities.items())\n",
    "\n",
    "    d3_ready = {}\n",
    "    for layer, group in parsed_communities.items():\n",
    "        layer_communities = {}\n",
    "        for no, clu in group:\n",
    "            try:\n",
    "                layer_communities[clu].append(no)\n",
    "            except KeyError:\n",
    "                layer_communities[clu] = [no]\n",
    "        d3_ready[layer] = layer_communities\n",
    "\n",
    "    return communities, d3_ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# -------------------------------- #\n",
    "## Compute community similarities ##\n",
    "# -------------------------------- #\n",
    "\n",
    "def get_similarity(i,j,communities, count=False):\n",
    "    \"\"\"communities : Communities in each layer\"\"\"\n",
    "    sim_counter = len(communities[i] & communities[j])\n",
    "    tot_counter = len(communities[i] | communities[j])\n",
    "    \n",
    "    if count:\n",
    "        return sim_counter / float(tot_counter), sim_counter\n",
    "    else:\n",
    "        return sim_counter / float(tot_counter)\n",
    "\n",
    "def compute_similarity_matrix(communities):\n",
    "    dim = len(communities)\n",
    "    X = {}\n",
    "    for i in range(dim):\n",
    "        for j in range(dim):\n",
    "            sim, count = get_similarity(i+1,j+1,communities, count=True)\n",
    "            try:\n",
    "                X['c'+str(i+1)].update({'c'+str(j+1): {'sim': sim, 'count': count}})\n",
    "            except KeyError:\n",
    "                X['c'+str(i+1)] = {'c'+str(j+1): {'sim': sim, 'count': count}}\n",
    "\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## --------------------- ##\n",
    "## Create binned network ##\n",
    "## --------------------- ##\n",
    "\n",
    "df_short, df_all = load_raw_month_data()\n",
    "create_loadable_bins()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## ------------------- ##\n",
    "## Load binned network ##\n",
    "## ------------------- ##\n",
    "\n",
    "#month_all_5mins = load_binned_network('1month_data','5mins_all')\n",
    "month_sho_5mins = load_binned_network('1month_data','5mins_short')\n",
    "#month_all_15mins = load_binned_network('1month_data','15mins_all')\n",
    "#month_sho_15mins = load_binned_network('1month_data','15mins_short')\n",
    "#month_all_30mins = load_binned_network('1month_data','30mins_all')\n",
    "#month_sho_30mins = load_binned_network('1month_data','30mins_short')\n",
    "#month_all_hourly = load_binned_network('1month_data','hourly_all')\n",
    "##month_sho_hourly = load_binned_network('1month_data','hourly_short')\n",
    "#month_all_daily = load_binned_network('1month_data','daily_all')\n",
    "#month_sho_daily = load_binned_network('1month_data','daily_short')\n",
    "#month_all_weekly = load_binned_network('1month_data','weekly_all')\n",
    "#month_sho_weekly = load_binned_network('1month_data','weekly_short')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## ------------------------ ##\n",
    "## Define LAYERS to analyse ##\n",
    "## ------------------------ ##\n",
    "dow = 6\n",
    "layers = month_sho_5mins[288*(dow+2):288*(dow+3)]\n",
    "\n",
    "## ------------------- ##\n",
    "## Compute COMMUNITIES ##\n",
    "## ------------------- ##\n",
    "communities, layer_communities = community_detection_multiplex(layers, halflife=300)\n",
    "similarities = compute_similarity_matrix(communities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt.fromtimestamp(layers[0]['timestamp'].values[0]).weekday()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build dataset for d3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Canvas parameters\n",
    "actual_width = len(set([item for sublist in communities.values() for item in sublist])) #number of nodes\n",
    "actual_height = len(layers)\n",
    "\n",
    "min_group_size = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def is_valid_location(new_block, existing_blocks):\n",
    "    \"\"\"Check whether random horizontal location for block is unoccupied\"\"\"\n",
    "    pad = 5\n",
    "    nb_x = new_block['x']\n",
    "    nb_w = new_block['w']\n",
    "    nb_range = set(range(int(nb_x-pad),int(nb_x+nb_w+pad)))\n",
    "    for block in existing_blocks:\n",
    "        eb_x = block['x']\n",
    "        eb_w = block['w']\n",
    "        eb_range = set(range(int(eb_x-pad),int(eb_x+eb_w+pad)))\n",
    "        if len(nb_range & eb_range) != 0:\n",
    "            return False\n",
    "\n",
    "    return True\n",
    "\n",
    "# Set working width and height\n",
    "width = actual_width*2\n",
    "height = actual_height\n",
    "\n",
    "\n",
    "#------#\n",
    "# Time #\n",
    "#------#\n",
    "\n",
    "# Start, termination, timestep variables\n",
    "t0 = str(dt.fromtimestamp(layers[0]['timestamp'].values[0])) # 2014-02-03 00:05:00\n",
    "tt = str(dt.fromtimestamp(layers[-1]['timestamp'].values[-1])) # 2014-02-04 00:00:00\n",
    "d_t = str((layers[1]['timestamp'].values[0]-layers[0]['timestamp'].values[0])/60) # 5 (minutes)\n",
    "\n",
    "thickness = layers[1]['timestamp'].values[0] - \\\n",
    "            layers[0]['timestamp'].values[0] # 300 (seconds)\n",
    "\n",
    "# Lines marking important points in time\n",
    "grid_times = [time(h) for h in [8,12,13,17]]\n",
    "grid_ticks = dict((i+1,str(dt.fromtimestamp(l['timestamp'].values[-1]))) \n",
    "                   for i, l in enumerate(layers) \n",
    "                   if dt.fromtimestamp(l['timestamp'].values[-1]).time() in grid_times)\n",
    "\n",
    "# Time tick labels\n",
    "label_times = [time(h) for h in [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23]]\n",
    "label_ticks = dict((i+1,str(dt.fromtimestamp(l['timestamp'].values[-1]))) \n",
    "                  for i, l in enumerate(layers) \n",
    "                  if dt.fromtimestamp(l['timestamp'].values[-1]).time() in label_times)\n",
    "\n",
    "\n",
    "#----------------------#\n",
    "# Build data structure #\n",
    "#----------------------#\n",
    "\n",
    "# Initiate data structure\n",
    "ds = {\n",
    "    'meta': {'w': width, 'h': height},\n",
    "    'time': {'t0': t0,\n",
    "             'tt': tt,\n",
    "             'dt': d_t,\n",
    "             'ticks': {'label_ticks': label_ticks,\n",
    "                       'grid_ticks': grid_ticks}\n",
    "            },\n",
    "    'sims': similarities,\n",
    "    'coms': {},\n",
    "    'layer_networks': {}\n",
    "}\n",
    "\n",
    "# Compute geometry and location of community blocks\n",
    "preferred_locations = {}\n",
    "prev_layer_blocks = {}\n",
    "for l,coms in layer_communities.items():\n",
    "    # Community-size pairs\n",
    "    com_size_pairs = [(k, len(v)) for k,v in coms.items() if len(v) >= min_group_size]\n",
    "    \n",
    "    # Compute blocks also in previous layer (EXISTING BLOCKS)\n",
    "    this_layer_blocks = {}\n",
    "    not_in_prev = []\n",
    "    for c,s in com_size_pairs:\n",
    "        \n",
    "        com = \"c\"+str(c)\n",
    "        if com in prev_layer_blocks:\n",
    "            prev_block = prev_layer_blocks[com]\n",
    "            this_x = (prev_block['w']-s)/2.0+prev_block['x']\n",
    "            block = {'x':this_x, 'w': s}\n",
    "            this_layer_blocks[com] = block\n",
    "            \n",
    "            points = [[this_x,l-1], [this_x+s,l-1], [this_x,l], [this_x+s,l]]\n",
    "            for p in points:\n",
    "                ds['coms'][com]['blocks'][-1]['points'].insert(\n",
    "                    len(ds['coms'][com]['blocks'][-1]['points'])/2, p\n",
    "                )\n",
    "                \n",
    "                ds['coms'][com]['tt'] = layers[l-1]['timestamp'].values[-1]\n",
    "                \n",
    "                ds['coms'][com]['duration'] = \\\n",
    "                    ds['coms'][com]['tt'] - \\\n",
    "                    ds['coms'][com]['t0']\n",
    "                \n",
    "                ds['coms'][com]['avg_size'] += s\n",
    "                ds['coms'][com]['avg_size'] /= 2.0\n",
    "                if s > ds['coms'][com]['max_size']:\n",
    "                    ds['coms'][com]['max_size'] = s\n",
    "                elif s < ds['coms'][com]['min_size']:\n",
    "                    ds['coms'][com]['min_size'] = s\n",
    "        else:\n",
    "            not_in_prev.append((c,s))\n",
    "\n",
    "    # Compute blocks not in previous layer (NEW BLOCKS)\n",
    "    existing_blocks = this_layer_blocks.values()\n",
    "    for c, s in not_in_prev:\n",
    "        com = \"c\"+str(c)\n",
    "        \n",
    "        # First try to put the block at its preferred location\n",
    "        if com in preferred_locations:\n",
    "            \n",
    "            this_x = preferred_locations[com]-s/2.0\n",
    "            block = {'x': this_x, 'w': s}\n",
    "            \n",
    "            if is_valid_location(block, existing_blocks):\n",
    "                \n",
    "                this_layer_blocks[com] = block\n",
    "                existing_blocks = this_layer_blocks.values()\n",
    "                \n",
    "                points = [[this_x,l-1], [this_x+s,l-1], [this_x,l], [this_x+s,l]]\n",
    "                for i, p in enumerate(points):\n",
    "                    if com not in ds['coms']:\n",
    "                        ds['coms'][com] = {'blocks': [], \n",
    "                                           't0': layers[l-1]['timestamp'].values[0], \n",
    "                                           'tt': layers[l-1]['timestamp'].values[-1],\n",
    "                                           'duration': thickness,\n",
    "                                           'abs_size': len(communities[c]),\n",
    "                                           'min_size': s,\n",
    "                                           'max_size': s,\n",
    "                                           'avg_size': s}\n",
    "                    if i == 0:\n",
    "                        ds['coms'][com]['blocks'].append({'points': [p], 'c': com})\n",
    "                    else:\n",
    "                        ds['coms'][com]['blocks'][-1]['points'].insert(\n",
    "                            len(ds['coms'][com]['blocks'][-1]['points'])/2, p\n",
    "                        )\n",
    "                        ds['coms'][com]['avg_size'] += s\n",
    "                        ds['coms'][com]['avg_size'] /= 2.0\n",
    "                        if s > ds['coms'][com]['max_size']:\n",
    "                            ds['coms'][com]['max_size'] = s\n",
    "                        elif s < ds['coms'][com]['min_size']:\n",
    "                            ds['coms'][com]['min_size'] = s\n",
    "                \n",
    "                # SUCCES! Continue to next community\n",
    "                continue\n",
    "                \n",
    "        # If that fails, go ahead and place the block wherever it fits!\n",
    "        counter = 0\n",
    "        while True:\n",
    "            counter += 1\n",
    "            \n",
    "            if counter > 100000:\n",
    "                print \"Can't place block anywhere\"\n",
    "                print l\n",
    "                sys.exit()\n",
    "                \n",
    "            this_x = randrange(0,width-s)\n",
    "            block = {'x':this_x, 'w': s}\n",
    "            if is_valid_location(block, existing_blocks):\n",
    "                this_layer_blocks[com] = block\n",
    "                existing_blocks = this_layer_blocks.values()\n",
    "                \n",
    "                points = [[this_x,l-1], [this_x+s,l-1], [this_x,l], [this_x+s,l]]\n",
    "                for i, p in enumerate(points):\n",
    "                    if com not in ds['coms']:\n",
    "                        ds['coms'][com] = {'blocks': [], \n",
    "                                           't0': layers[l-1]['timestamp'].values[0], \n",
    "                                           'tt': layers[l-1]['timestamp'].values[-1],\n",
    "                                           'duration': thickness,\n",
    "                                           'abs_size': len(communities[c]),\n",
    "                                           'min_size': s,\n",
    "                                           'max_size': s,\n",
    "                                           'avg_size': s}\n",
    "                    if i == 0:\n",
    "                        ds['coms'][com]['blocks'].append({'points': [p], 'c': com})\n",
    "                    else:\n",
    "                        ds['coms'][com]['blocks'][-1]['points'].insert(\n",
    "                            len(ds['coms'][com]['blocks'][-1]['points'])/2, p\n",
    "                        )\n",
    "                        ds['coms'][com]['avg_size'] += s\n",
    "                        ds['coms'][com]['avg_size'] /= 2.0\n",
    "                        if s > ds['coms'][com]['max_size']:\n",
    "                            ds['coms'][com]['max_size'] = s\n",
    "                        elif s < ds['coms'][com]['min_size']:\n",
    "                            ds['coms'][com]['min_size'] = s\n",
    "                break\n",
    "            \n",
    "        preferred_locations[com] = block['x']+block['w']/2.0\n",
    "        \n",
    "        \n",
    "    prev_layer_blocks = this_layer_blocks\n",
    "    \n",
    "# Add colors\n",
    "# Deprecated: is still needed on script. Misleading that it is not namechanged or something.\n",
    "com_cols = {}\n",
    "for com in ds['coms'].keys():\n",
    "    r = np.average([mmh3.hash(n)%256 for n in communities[int(com[1:])]])\n",
    "    g = np.average([mmh3.hash(n)%255 for n in communities[int(com[1:])]])\n",
    "    b = np.average([mmh3.hash(n)%254 for n in communities[int(com[1:])]])\n",
    "\n",
    "    #ds['coms'][com]['color'] = 'rgb(%d,%d,%d)' % (r,g,b)\n",
    "    com_cols[com] = 'rgb(%d,%d,%d)' % (r,g,b)\n",
    "    \n",
    "# Add networks\n",
    "def layer_networks(filename, layer_communities):\n",
    "    \n",
    "    with open(filename, 'r') as infile:\n",
    "        rawstring = infile.read()\n",
    "        \n",
    "    rawstring_nodes, rawstring_edges = rawstring.split(\"#\")[:-1]\n",
    "    \n",
    "    nodes_map = dict((int(re.findall(r'\\d+', n)[0]), re.findall(r'[0-9a-f]{30}', n)[0]) for n in \n",
    "                      re.findall(r'\\d+ \"Node [0-9a-f]{30}\" \\d\\.\\d', rawstring_nodes))\n",
    "    nodes_map_reverse = dict((v,k) for k,v in nodes_map.items())\n",
    "    \n",
    "    ln = {'data': {}}\n",
    "    \n",
    "    e_li = [e_str.split() for e_str in rawstring_edges.split('\\n')[1:]] # [['1', '347', '1', '348', '1'],...]\n",
    "    \n",
    "    # Add edges, one at a time\n",
    "    for e in e_li:\n",
    "        layer, source, target, value = int(e[0]), int(e[1]), int(e[3]), int(e[4])\n",
    "        try:\n",
    "            group_source = [g for g,n in layer_communities[layer].items() if nodes_map[source] in n][0]\n",
    "            group_target = [g for g,n in layer_communities[layer].items() if nodes_map[target] in n][0]\n",
    "        except IndexError:\n",
    "            # This occurs if the source/target has no group, WHICH IT SHOULD. Why this happend (and it\n",
    "            # very rarely does) I don't know, maybe it's something with Infomap.\n",
    "            print \"node\", source, \"or\", target, \"has no group in layer\", layer, \"... skipping link!\"\n",
    "            continue\n",
    "            \n",
    "        if \"c\"+str(group_source) not in com_cols or \"c\"+str(group_target) not in com_cols:\n",
    "            continue\n",
    "    \n",
    "        edge = {'source': source, 'target': target, 'value': value/10.0}\n",
    "        try:\n",
    "            ln['data'][layer]['links'].append(edge)\n",
    "        except KeyError:\n",
    "            ln['data'][layer] = {'links': [edge]}\n",
    "        \n",
    "        for n1, n2 in [(source, target),(target, source)]:\n",
    "            try:\n",
    "                ln['data'][layer]['links_dict'][n1].append(n2)\n",
    "            except KeyError:\n",
    "                try:\n",
    "                    ln['data'][layer]['links_dict'][n1] = [n2]\n",
    "                except KeyError:\n",
    "                    ln['data'][layer]['links_dict'] = {n1: [n2]}\n",
    "                \n",
    "            \n",
    "    # Add nodes\n",
    "    for layer, edges_and_nodes in ln['data'].items():\n",
    "        edges = edges_and_nodes['links']\n",
    "        nodes_names = set()\n",
    "        for e in edges:\n",
    "            nodes_names.add(e['source'])\n",
    "            nodes_names.add(e['target'])\n",
    "        nodes = []\n",
    "        for nn in nodes_names:\n",
    "            group = [g for g,n in layer_communities[layer].items()\n",
    "                         if nodes_map[nn] in n][0]\n",
    "            try:\n",
    "                col = com_cols['c'+str(group)]\n",
    "            except KeyError:\n",
    "                #col = 'rgb(%d,%d,%d)' % (200,200,200)\n",
    "                continue\n",
    "            node = {'name': nn, 'id': nodes_map[nn], 'group': group}\n",
    "            try:\n",
    "                ln['data'][layer]['nodes'][nn] = node\n",
    "            except KeyError:\n",
    "                ln['data'][layer].update({'nodes': {nn: node}})\n",
    "                \n",
    "            try:\n",
    "                ln['data'][layer]['groups'][group].append(node['name'])\n",
    "            except KeyError:\n",
    "                try:\n",
    "                    ln['data'][layer]['groups'][group] = [node['name']]\n",
    "                except KeyError:\n",
    "                    ln['data'][layer]['groups'] = {group: [node['name']]}\n",
    "    \n",
    "    return ln\n",
    "\n",
    "layer_networks = layer_networks('input/multiplex-network.net', layer_communities)\n",
    "ds['layer_networks'] = layer_networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## -------------------------------- ##\n",
    "## Store D3-loadable data structure ##\n",
    "## -------------------------------- ##\n",
    "\n",
    "with open('Visualisation/data/datasetSun.json', 'w') as outfile:\n",
    "    json.dump(ds,outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-141-6f70a0cbdc74>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "[][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}