WikiSentiViewer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load related library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# import matplotlib as mpl\n",
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "from shapely.geometry import Point\n",
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "from geopandas import GeoSeries, GeoDataFrame\n",
    "\n",
    "import ipywidgets\n",
    "from ipywidgets import widgets \n",
    "from ipywidgets import *  \n",
    "from IPython.display import display,clear_output\n",
    "#https://github.com/jupyter-widgets/ipywidgets/issues/134 about ouput\n",
    "from ipywidgets import Layout\n",
    "import folium\n",
    "from datetime import datetime\n",
    "from datetime import date\n",
    "from dateutil import rrule\n",
    "\n",
    "from traitlets import directional_link\n",
    "\n",
    "from boto.s3.connection import S3Connection\n",
    "import requests\n",
    "from io import BytesIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print the version of libraries being used\n",
    "#pd.__version__, gpd.__version__, folium.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# side function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loadJson(filename):\n",
    "    import json\n",
    "    f = open(filename, \"r\", encoding = \"utf-8\")\n",
    "    output = json.load(f)\n",
    "    f.close()\n",
    "    return output\n",
    "def add_datetime_column(df,unit):\n",
    "    # unit: date,month,year\n",
    "    datetime_format = \"\"\n",
    "    if unit == \"date\":\n",
    "        datetime_format = \"%Y-%m-%d\"\n",
    "    elif unit == \"month\":\n",
    "        datetime_format = \"%Y-%m\"\n",
    "    elif unit == \"year\":\n",
    "        datetime_format = \"%Y\"\n",
    "    new_list = [datetime.strptime(one,datetime_format) for one in df[unit]]\n",
    "    if unit == \"date\":\n",
    "        df = df.assign(datetime_date=pd.Series(new_list).values)\n",
    "    elif unit == \"month\":\n",
    "        df = df.assign(datetime_month=pd.Series(new_list).values)\n",
    "    elif unit == \"year\":\n",
    "        df = df.assign(datetime_year=pd.Series(new_list).values)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load dataset\n",
    "# load remote dataset (when running in mybinder)\n",
    "#If you do it locally, check the commented one, replace all wiki package with pd.read_csv feeding the file path by yours."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data\n",
    "large_split1 = pd.read_csv('src/article_title_score_geo_country_split1.csv')\n",
    "large_split2 = pd.read_csv('src/article_title_score_geo_country_split2.csv')\n",
    "article_score_with_geo_csv = pd.concat([large_split1, large_split2])\n",
    "\n",
    "talk_score_with_geo_csv = pd.read_csv('src/talk_title_score_geo_country_continent.csv')\n",
    "person_birthdate_csv = pd.read_csv('src/person_birthdate_space.csv')\n",
    "event_date_csv = pd.read_csv('src/event_date_space.csv')\n",
    "person_list_csv = pd.read_csv('src/personList_space.csv')\n",
    "event_list_csv = pd.read_csv('src/eventList_space.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "### get score and geo information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading csv containing title, score*8, geometry, country, continent\n",
    "title_score_geo = {}\n",
    "#title_score_geo[\"Articles\"] = pd.read_csv(article_score_with_geo_csv)\n",
    "title_score_geo[\"Articles\"] = article_score_with_geo_csv\n",
    "# #if you do it locally\n",
    "# title_score_geo[\"Articles\"] = pd.read_csv(\"score_file/title_score_geo_country_continent.csv\")\n",
    "\n",
    "#title_score_geo[\"Talks\"] = pd.read_csv(talk_score_with_geo_csv)\n",
    "title_score_geo[\"Talks\"] = talk_score_with_geo_csv\n",
    "# #if you do it locally\n",
    "# title_score_geo[\"Talks\"] = pd.read_csv(\"talk_score_file/title_score_geo_country_continent.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "### get date information and exclude BC date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# load csv containing date for people and events\n",
    "# exclude BC\n",
    "# change dates to datetime type\n",
    "dates = {}\n",
    "temp_df = None\n",
    "# for people\n",
    "#temp_df = pd.read_csv(person_birthdate_csv,names=['title','date'])\n",
    "temp_df = person_birthdate_csv[['title','date']]\n",
    "\n",
    "\n",
    "#print (temp_df.head(3))\n",
    "#temp_df = pd.read_csv(\"person_birthdate_space.csv\")\n",
    "# exclude BC\n",
    "temp_df = temp_df[~temp_df['date'].str.startswith(\"-\")]\n",
    "# add datetime column\n",
    "temp_df = add_datetime_column(temp_df,'date')\n",
    "del temp_df['date']\n",
    "dates[\"People\"] = temp_df\n",
    "\n",
    "temp_df = None\n",
    "#temp_df = pd.read_csv(event_date_csv,names=['title','date'])\n",
    "temp_df = event_date_csv[['title','date']]\n",
    "#temp_df = pd.read_csv(\"event_date_space.csv\")\n",
    "# exclude BC\n",
    "temp_df = temp_df[~temp_df['date'].str.startswith(\"-\")]\n",
    "temp_df = add_datetime_column(temp_df,'date')\n",
    "del temp_df['date']\n",
    "dates[\"Events\"] = temp_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print (dates[\"People\"].head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "### get people list and events list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#people_list = pd.read_csv(person_list_csv,names=['title'])\n",
    "people_list = person_list_csv[['title']]\n",
    "#people_list = pd.read_csv(\"personList_space.csv\",names=[\"title\"])\n",
    "#events_list = pd.read_csv(event_list_csv,names=['title'])\n",
    "events_list = event_list_csv[['title']]\n",
    "#events_list = pd.read_csv(\"eventList_space.csv\",names=[\"title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "### merge score and date to get three dataframes separately for people, events, others"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split title_score_geo into people, events, others three groups, and add date to people, events\n",
    "df = {\"Articles\":{},\"Talks\":{}}\n",
    "temp_df = None\n",
    "for one in df:\n",
    "    # for people\n",
    "    temp_df = title_score_geo[one].merge(people_list,how='inner')\n",
    "    #print (\"length of People for %s with geolocation is %d\"%(one,len(temp_df)))\n",
    "    df[one][\"People\"] = temp_df.merge(dates[\"People\"],how='left')\n",
    "    \n",
    "    temp_df = None\n",
    "    # for events\n",
    "    temp_df = title_score_geo[one].merge(events_list,how='inner')\n",
    "    #print (\"length of Events for %s with geolocation is %d\"%(one,len(temp_df)))\n",
    "    df[one][\"Events\"] = temp_df.merge(dates[\"Events\"],how='left')\n",
    "\n",
    "    temp_df = None\n",
    "    # for others\n",
    "    temp_df = title_score_geo[one][~title_score_geo[one]['title'].isin(people_list['title'])]\n",
    "    df[one][\"Others\"] = temp_df[~temp_df['title'].isin(events_list['title'])]\n",
    "    #print (\"length of Others for %s with geolocation is %d\"%(one,len(df[one][\"Others\"])))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print (df[\"Articles\"][\"People\"].head(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get continent, country list from geopandas library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare data for dropdown: continent list and country list\n",
    "\n",
    "# get world info\n",
    "world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))\n",
    "# get continent list plus \"all\"\n",
    "continent_list = [\"all\"] + list(set(world['continent']))\n",
    "#continent_list.append(\"all\")\n",
    "\n",
    "continent_country_dict = {}\n",
    "for continent in continent_list:\n",
    "    continent_country_dict[continent] = [\"all\"] + list(world[world['continent']== continent]['name'])\n",
    "    #continent_country_dict[continent].append(\"all\")\n",
    "    \n",
    "#print (\"continent list for dropdown:\")\n",
    "#print (continent_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "744c87e3ce544d4f9c062d77670a8952",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(VBox(children=(VBox(children=(HTML(value='<h1>WikiSentiViewer</h1>\\n<p>Introduct…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# design the panel: map panel and side panel\n",
    "# for map panel:\n",
    "#     map\n",
    "#\n",
    "# for side panel:\n",
    "#     sentiment container: tell user the color to indicate negative and positive\n",
    "#         title label\n",
    "#         negative label\n",
    "#         positive label\n",
    "#     lexicon container: Which lexicon you want to choose? (OL, MPQA, LIWC)\n",
    "#         title label\n",
    "#         radio_button\n",
    "#     group container: Which group you want to show? (People, Events, Others)\n",
    "#         title label\n",
    "#         checkboxes container:\n",
    "#             checkbox\n",
    "#             checkbox\n",
    "#             checkbox\n",
    "#     area filter: Do you want to filter out the continent or country for entities?\n",
    "#         title label\n",
    "#         dropdown_continent\n",
    "#         dropdown_country\n",
    "#     time filter: Do you want to filter out time range?\n",
    "#         title label\n",
    "#         slider\n",
    "#     button to update\n",
    "\n",
    "\n",
    "# change label to HTML (for set style more flexible)\n",
    "    \n",
    "# set a general style and layout    \n",
    "#score_description_style = {'description_width':'50px'}\n",
    "with_border_layout = Layout(border='solid 1px')\n",
    "sp_items_layout = Layout(flex='1 1 auto',\n",
    "                         width='auto',\n",
    "                         border='solid 1px',)\n",
    "\n",
    "\n",
    "# container for all\n",
    "all_container = widgets.VBox(layout=with_border_layout)\n",
    "\n",
    "\n",
    "# preparing a container for side panel\n",
    "sp_container = widgets.VBox(layout=Layout(\n",
    "    display='flex',\n",
    "    flex_flow='column',\n",
    "    align_items='stretch',\n",
    "    border='solid 1px',\n",
    "    width='30%'\n",
    ")\n",
    ")\n",
    "\n",
    "\n",
    "# for sentiment lexicon\n",
    "# lexicon container for lexicons\n",
    "lexicon_container = widgets.VBox(layout=sp_items_layout)\n",
    "# add title\n",
    "html_lexicon = widgets.HTML(value=\"<b>Sentiment Lexicon</b>\")\n",
    "#label_lexicon = widgets.Label(value=\"sentiment lexicon\")\n",
    "# preparing a container to put in radio buttons\n",
    "radio_button_lexicon = widgets.RadioButtons(\n",
    "    options=['OL', 'MPQA', 'LIWC','ANEW'],\n",
    "    #description='sentiment lexicon',\n",
    "    #style=style,\n",
    "    disabled=False\n",
    ")\n",
    "# put text and button into lexicon container\n",
    "lexicon_container.children = [html_lexicon,radio_button_lexicon]\n",
    "\n",
    "\n",
    "# for domain\n",
    "domain_container = widgets.VBox(layout=sp_items_layout)\n",
    "# add title\n",
    "html_domain = widgets.HTML(value=\"<b>Domain</b>\")\n",
    "# preparing a container to put in radio buttons\n",
    "radio_button_domain = widgets.RadioButtons(\n",
    "    options=['Articles', 'Talks'],\n",
    "    disabled=False\n",
    ")\n",
    "# put text and button into domain container\n",
    "domain_container.children = [html_domain,radio_button_domain]\n",
    "\n",
    "\n",
    "# for target group\n",
    "# container for groups\n",
    "group_container = widgets.VBox(layout=sp_items_layout)\n",
    "html_group = widgets.HTML(value=\"<b>Target Group</b>\")\n",
    "# checkboxes container\n",
    "cb_container = widgets.VBox(layout=Layout(\n",
    ")) \n",
    "# preparing a container to put in created checkbox\n",
    "checkboxes = []  \n",
    "# create checkbox\n",
    "checkboxes.append(widgets.Checkbox(description = 'People', value=False))\n",
    "checkboxes.append(widgets.Checkbox(description = 'Events', value=False))\n",
    "checkbox_others = widgets.Checkbox(description = 'Others', value=False)\n",
    "checkboxes.append(checkbox_others)\n",
    "# put check box into checkboxes container\n",
    "cb_container.children=[i for i in checkboxes]\n",
    "group_container.children=[html_group,cb_container]\n",
    "\n",
    "\n",
    "# introduction part\n",
    "# how to indicate positive and negative\n",
    "# container for sentiment description\n",
    "header_container = widgets.VBox()\n",
    "header_text = \"\"\"<h1>WikiSentiViewer</h1>\n",
    "<p>Introduction<ul>\n",
    "<li>A widget to show sentiment distribution on geolocation for Wikipedia entities (concepts). </li>\n",
    "<li>Wikipedia entities contains entities in Wikipedia Article and Wikipedia Talk. Talk page is an area for editors to discuss about corresponding article, which can be visited from upper left side of article page.</li>\n",
    "</ul></p>\n",
    "<p>How to use:\n",
    "<ol>\n",
    "<li>Select value for Sentiment lexicon, Domain, Target group.</li>\n",
    "<li>Set Score range, Target area, minimum length setting, Time range. In which length indicates the number of tokens for Wikipedia texts.</li>\n",
    "<li>Click \"Update the graph\"</li>\n",
    "</ol>\n",
    "</p>\n",
    "<p>Results\n",
    "<ul>\n",
    "<li>For each run you will see a new map with circles on it. Each circle represents an entity.\n",
    "The size of the circle indicates the total score of its sentiment, the color indicates the balance of positive/negative sentiment,\n",
    "in which red represents negative whereas blue represents positive.</li>\n",
    "<li>In the bottom you will see the statistics information of the current run.</li>\n",
    "</ul>\n",
    "</p>\n",
    "<p>Notes: <ul><li>Running time is decided by the number of displayed entities. Since there are huge entities in group Others, please narrow down displayed entities while you pick up this group by \n",
    "using filters, otherwise it would take too much time to show the map and sometimes killing the kernel. </li></ul></p>\"\"\"\n",
    "#header_text = \"<h1>WikiSentiViewer</h1>\"\n",
    "html_header = widgets.HTML(value=header_text)\n",
    "\n",
    "header_container.children=[html_header]\n",
    "\n",
    "\n",
    "# for area filter\n",
    "#container for area\n",
    "area_container = widgets.VBox(layout=sp_items_layout)\n",
    "html_area = widgets.HTML(value=\"<b>Target Area</b>\")\n",
    "#dropdown_container = widgets.HBox()\n",
    "#dropdown\n",
    "dropdown_continent = widgets.Dropdown(\n",
    "    options = continent_list,\n",
    "    value = 'all',\n",
    "    description='Continent:',\n",
    "    layout=Layout(width='250px'),\n",
    "    disabled=False,\n",
    ")\n",
    "    \n",
    "dropdown_country = widgets.Dropdown(\n",
    "    #options = country_list,\n",
    "    description='Country:',\n",
    "    #options = [\"all\"],\n",
    "    #value='all',\n",
    "    layout=Layout(width='250px'),\n",
    "    \n",
    "    disabled = False,\n",
    ")\n",
    "\n",
    "def transform(case):\n",
    "    return continent_country_dict[case]\n",
    "\n",
    "directional_link((dropdown_continent,'value'),(dropdown_country,'options'),transform)\n",
    "\n",
    "\n",
    "#l = traitlets.link((dropdown_continent,'value'),(dropdown_country,'value'))\n",
    "#dropdown_container.children = [dropdown_continent,dropdown_country]\n",
    "area_container.children=[html_area,dropdown_continent,dropdown_country]\n",
    "\n",
    "\n",
    "# for length\n",
    "length_container = widgets.VBox(layout = sp_items_layout)\n",
    "length_checkbox = widgets.Checkbox(description = 'Minimum Length Setting', value=False,layout=Layout(left='-50px'))\n",
    "length_dropdown = widgets.Dropdown(options=[5,10,20,30],\n",
    "                                   #description='minimum length',\n",
    "                                   layout=Layout(width='100px',left='50px'),\n",
    "                                   disabled = True,)\n",
    "length_container.children = [length_checkbox,length_dropdown]\n",
    "# add link from length_checkbox to length_dropdown\n",
    "def transform_length(case):\n",
    "    return {True:False,False:True}[case]\n",
    "directional_link((length_checkbox,'value'),(length_dropdown,'disabled'),transform_length)\n",
    "\n",
    "\n",
    "\n",
    "# container for slider\n",
    "slider_container = widgets.VBox(layout=with_border_layout)\n",
    "# add a time slider: SelectionRangeSlider\n",
    "slider_dates = [day for day in rrule.rrule(rrule.MONTHLY, dtstart=date(1800,1,1), until=date(2016,4,1))]#date.today()\n",
    "options = [(i.strftime('%b%Y'),i) for i in slider_dates]\n",
    "#slider = widgets.SelectionRangeSlider\n",
    "slider = ipywidgets.SelectionRangeSlider(\n",
    "    options=options,\n",
    "    index=(0,len(slider_dates)-1),\n",
    "    #description='          ',\n",
    "    layout=Layout(width='950px',height='40px',left='10px'),\n",
    "    #,description_width='200px'\n",
    "    #description_width='20px',\n",
    "    #style = style,\n",
    "    disabled=True\n",
    ")\n",
    "html_slider = widgets.HTML(value=\"<b>Time Range</b>\")\n",
    "checkbox_slider = widgets.Checkbox(\n",
    "    layout=Layout(width='800px'),\n",
    "    description = 'Do not filter out date (if you tick it or select \"Others\" in \"Target Group\", the range of time will be ignored)', value=False)\n",
    "slider_container.children=[html_slider,checkbox_slider,slider]\n",
    "#print (slider.value[0].date())\n",
    "\n",
    "\n",
    "\n",
    "# add a link between others and checkbox_slider\n",
    "def transform_time_others_checkbox(case):\n",
    "    return {True:True,False:False}[case]\n",
    "directional_link((checkbox_others,'value'),(checkbox_slider,'value'),transform_time_others_checkbox)\n",
    "\n",
    "# add a link from others to check_slider\n",
    "def transform_time_others_checkbox2(case):\n",
    "    return {True:True,False:False}[case]\n",
    "directional_link((checkbox_others,'value'),(checkbox_slider,'disabled'),transform_time_others_checkbox2)\n",
    "\n",
    "# add a link between checkbox_slider and sliders (directly disabled slider)\n",
    "def transform_check_slider(case):\n",
    "    return {True:True,False:False}[case]\n",
    "\n",
    "directional_link((checkbox_slider,'value'),(slider,'disabled'),transform_check_slider)\n",
    "\n",
    "\n",
    "# for score range\n",
    "# container for score \n",
    "score_container = widgets.VBox(layout=sp_items_layout)\n",
    "# add a score slider: IntRangeSlider\n",
    "score_description_style = {'description_width': '20%'}\n",
    "\n",
    "slider_score = widgets.IntRangeSlider(value=[1,100],\n",
    "                                      min=0,\n",
    "                                      max=100,\n",
    "                                      step=1,\n",
    "                                      disabled=False,\n",
    "                                      continuous_update=False,\n",
    "                                      orientation='horizontal',\n",
    "                                      readout=True,\n",
    "                                      \n",
    "                                      layout=Layout(width='273px'),\n",
    "                                      readout_format='d',\n",
    "                                      description='Total',\n",
    "                                      style=score_description_style,)\n",
    "slider_score_positive = widgets.IntRangeSlider(value=[1,100],\n",
    "                                      min=0,\n",
    "                                      max=100,\n",
    "                                      step=1,\n",
    "                                      disabled=False,\n",
    "                                      continuous_update=False,\n",
    "                                      orientation='horizontal',\n",
    "                                      readout=True,\n",
    "                                      \n",
    "                                      layout=Layout(width='273px'),\n",
    "                                      readout_format='d',\n",
    "                                      description='Positive',\n",
    "                                      style=score_description_style,)\n",
    "\n",
    "slider_score_negative = widgets.IntRangeSlider(value=[1,100],\n",
    "                                      min=0,\n",
    "                                      max=100,\n",
    "                                      step=1,\n",
    "                                      disabled=False,\n",
    "                                      continuous_update=False,\n",
    "                                      orientation='horizontal',\n",
    "                                      readout=True,\n",
    "                                      \n",
    "                                      layout=Layout(width='273px'),\n",
    "                                      readout_format='d',\n",
    "                                      description='Negative',\n",
    "                                      style=score_description_style,)\n",
    "\n",
    "\n",
    "#html_score = widgets.HTML(value=\"<b>Score range</b><br>score=positive score + negative score<br>default range is 1 to 500.<br>\")\n",
    "# for labels\n",
    "html_score = widgets.HTML(value=\"<b>Score Range</b>\")  \n",
    "\n",
    "#score_container.children=[html_score,html_score_total,slider_score,html_score_positive,slider_score_positive,html_score_negative,slider_score_negative]\n",
    "score_container.children=[html_score,slider_score,slider_score_positive,slider_score_negative]\n",
    "#print (slider.value[0].date())\n",
    "\n",
    "\n",
    "\n",
    "# use an area for text output\n",
    "output_text_container = widgets.VBox(layout=Layout(width='100%',border='solid 1px'))\n",
    "output_label_str_container = widgets.VBox(layout=Layout(width='100%'))\n",
    "output_label = widgets.Label(value=\"Here is the summary...\",\n",
    "                               layout=Layout(width='100%'),\n",
    "                              disabled=False)\n",
    "text_out = widgets.Output()\n",
    "with text_out:\n",
    "    display(output_label_str_container)\n",
    "\n",
    "\n",
    "output_label_str_container.children=[output_label]\n",
    "output_text_container.children = [text_out]\n",
    "\n",
    "\n",
    "#for button\n",
    "update_container = widgets.VBox(layout=Layout(display='flex',\n",
    "                                              flex_flow='column',\n",
    "                                              align_items='center',\n",
    "                                              width='100%',\n",
    "                                              border='solid 1px'))\n",
    "#add button that updates the graph based on the checkboxes\n",
    "button = widgets.Button(description=\"Update the Graph\", button_style='primary')\n",
    "update_container.children=[button]\n",
    "\n",
    "\n",
    "sp_container.children=[lexicon_container, domain_container, group_container,score_container, area_container,length_container]\n",
    "\n",
    "\n",
    "# for map\n",
    "map_container = widgets.VBox(layout=Layout(width='80%',height='100%',border='solid 1px'))\n",
    "# preparing the plot \n",
    "myMap = folium.Map(location=[45.5236, -122.6750], zoom_start=1)\n",
    "map_out = widgets.Output()\n",
    "with map_out:\n",
    "    display(myMap)\n",
    "map_container.children = [header_container, map_out]\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "# container for upper part:map container + side panel\n",
    "upper_container = widgets.HBox(layout=with_border_layout\n",
    ")\n",
    "upper_container.children=[map_container,sp_container]\n",
    "\n",
    "\n",
    "# container for all\n",
    "#all_container = widgets.VBox(layout=with_border_layout)\n",
    "all_container.children = [upper_container,slider_container,update_container,output_text_container]\n",
    "display(all_container)\n",
    "dict_a_people = {\"total\":\"1,146,257\",\"geo\":\"1816\"}\n",
    "dict_a_events = {\"total\":\"54,071\",\"geo\":\"7477\"}\n",
    "dict_a_others = {\"total\":\"4,469,539\",\"geo\":\"913,577\"}\n",
    "dict_t_people = {\"total\":\"415,124\",\"geo\":\"1191\"}\n",
    "dict_t_events = {\"total\":\"21621\",\"geo\":\"4442\"}\n",
    "dict_t_others = {\"total\":\"1,469,630\",\"geo\":\"268,773\"}\n",
    "stat_dict = {\"Articles\":{\"People\":dict_a_people,\"Events\":dict_a_events,\"Others\":dict_a_others},\"Talks\":{\"People\":dict_t_people,\"Events\":dict_t_events,\"Others\":dict_t_others}}\n",
    "\n",
    "def on_button_clicked(b): \n",
    "#   # create a new map\n",
    "    myMap = folium.Map(location=[45.5236, -122.6750], zoom_start=1)\n",
    "    # create a new output container\n",
    "    output_label_str_container = widgets.VBox(layout=Layout(width='100%',border='solid 1px'))\n",
    "    # get continent and country info\n",
    "    continent = dropdown_continent.value\n",
    "    country = dropdown_country.value\n",
    "    # get score range info\n",
    "    score_start, score_end = slider_score.value\n",
    "    score_start_positive, score_end_positive = slider_score_positive.value\n",
    "    score_start_negative, score_end_negative = slider_score_negative.value\n",
    "    #print (score_start,score_end)\n",
    "    #time_start, time_end = slider.value\n",
    "    # get domain: Articles or Talks\n",
    "    domain = radio_button_domain.value\n",
    "    # get lexicon name: OL, MPQA, LIWC, ANEW\n",
    "    lexicon = radio_button_lexicon.value\n",
    "    \n",
    "    output_label_list = []\n",
    "    counter_showed = 0\n",
    "    for c in cb_container.children:\n",
    "        # deal with one target group\n",
    "        if c.value:\n",
    "            group = c.description\n",
    "            output_label_list.append(\"----There are %s entities for %s group in Wikipedia %s.\"%(stat_dict[domain][group][\"total\"],group,domain))\n",
    "\n",
    "            # area filter\n",
    "            target_data = None\n",
    "            if not continent == \"all\":\n",
    "                if not country == \"all\":\n",
    "                    if country == 'United States of America':\n",
    "                        country = 'United States'\n",
    "                    target_data = df[domain][group][df[domain][group]['country']==country]\n",
    "                else:\n",
    "                    # set continent but not country\n",
    "                    target_data = df[domain][group][df[domain][group]['continent']==continent]\n",
    "            else:\n",
    "                # include all continent\n",
    "                target_data = df[domain][group]\n",
    "            output_label_list.append(\"--------After filtering with geolocation, we got %d entities left\"%(len(target_data)))\n",
    "            \n",
    "            \n",
    "            # here we add doc length filter\n",
    "            if length_checkbox.value == True:\n",
    "                length_temp = len(target_data)\n",
    "                target_data = target_data[target_data['length']>=length_dropdown.value]\n",
    "                output_label_list.append(\"--------After filtering with doc length, we discard %d entities and got %d entities left.\"%(length_temp-len(target_data),len(target_data)))\n",
    "                \n",
    "            # here we do time filtering\n",
    "            #print (slider.disabled)\n",
    "            if slider.disabled == False:\n",
    "                # time start ,time end = slider.value\n",
    "                xaxis_value = [day for day in rrule.rrule(rrule.DAILY,dtstart=slider.value[0],until=slider.value[1])]\n",
    "                target_data = target_data[target_data['datetime_date'].isin(xaxis_value)]\n",
    "            output_label_list.append(\"--------After filtering with time, we got %d entities left.\"%(len(target_data)))\n",
    "            \n",
    "            # here we do score filtering\n",
    "            counter_0 = len(target_data)\n",
    "            target_data = target_data[target_data['pos_score_'+lexicon]>=score_start_positive]\n",
    "            counter_1 = len(target_data)\n",
    "            counter_less_pos = counter_0 - counter_1\n",
    "            target_data = target_data[target_data['pos_score_'+lexicon]<=score_end_positive]\n",
    "            counter_2 = len(target_data)\n",
    "            counter_larger_pos = counter_1-counter_2\n",
    "            target_data = target_data[target_data['neg_score_'+lexicon]>=score_start_negative]\n",
    "            counter_3 = len(target_data)\n",
    "            counter_less_neg = counter_2 - counter_3\n",
    "            target_data = target_data[target_data['neg_score_'+lexicon]<=score_end_negative]\n",
    "            counter_4 = len(target_data)\n",
    "            counter_larger_neg = counter_3 - counter_4\n",
    "            # add total score\n",
    "            target_data = target_data.assign(total=target_data['pos_score_'+lexicon]+target_data['neg_score_'+lexicon])\n",
    "            # filtering total score\n",
    "            target_data = target_data[target_data['total']>=score_start]\n",
    "            counter_5 = len(target_data)\n",
    "            counter_less_total = counter_4 - counter_5\n",
    "            target_data = target_data[target_data['total']<=score_end]\n",
    "            counter_6 = len(target_data)\n",
    "            counter_larger_total = counter_5 - counter_6\n",
    "            output_label_list.append(\"--------After filtering with score, we got %d entities left.\"%(len(target_data)))\n",
    "            output_label_list.append(\"------------for positive score: %d entities have a score higher than %d, %d entities have a score lower than %d.\"%(counter_larger_pos,score_start_positive,counter_less_pos,score_end_positive))\n",
    "            output_label_list.append(\"------------for negative score: %d entities have a score higher than %d, %d entities have a score higher than %d.\"%(counter_larger_neg,score_start_negative,counter_less_neg,score_end_negative))\n",
    "            output_label_list.append(\"------------for total score: %d entities have a score higher than %d, %d entities have a score lower than %d.\"%(counter_larger_total,score_start,counter_less_total,score_end))\n",
    "            counter_showed += len(target_data)\n",
    "            output_label_list.append(\"----Consequently, we got %d entities for group %s in Wikipedia %s showed in this run.\"%(len(target_data),group,domain))\n",
    "\n",
    "            for each in target_data.iterrows():\n",
    "                node_color = '#{:02x}00{:02x}'.format(int(255 * each[1]['neg_score_'+lexicon]/each[1]['total']),int(255 * each[1]['pos_score_'+lexicon]/each[1]['total']))\n",
    "                popup = folium.Popup(each[1]['title']+\". pos score:\"+str(each[1]['pos_score_'+lexicon])+\", neg score:\"+str(each[1]['neg_score_'+lexicon]),parse_html=True)\n",
    "                #print (node_color)\n",
    "\n",
    "                node_lon,node_lat = each[1]['geometry'][7:-1].split(\" \")\n",
    "                #print (title)\n",
    "                #print (float(node_lat),float(node_lon))\n",
    "\n",
    "                folium.CircleMarker([float(node_lat),float(node_lon)],\n",
    "                                radius= each[1]['total'],\n",
    "                                #radius = 5,     \n",
    "                                popup=popup,\n",
    "                                fill = True,\n",
    "                                color=node_color,\n",
    "                                weight = 1,\n",
    "                                fill_color=node_color,\n",
    "                                ).add_to(myMap)\n",
    "  \n",
    "    with map_out:\n",
    "        clear_output(wait=True)\n",
    "        display(myMap)\n",
    "    \n",
    "    output_label_list.append(\"In total, %d entities are displayed on the map.\"%(counter_showed))\n",
    "    output_labels = [widgets.HTML(value=i) for i in output_label_list]\n",
    "    output_label_str_container.children = [i for i in output_labels]\n",
    "    with text_out:\n",
    "        clear_output(wait=True)\n",
    "        display(output_label_str_container)\n",
    "    #print (\"finish\")\n",
    "\n",
    "button.on_click(on_button_clicked)  \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Link to [WikiSentiFlow](WikiSentiFlow.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}