2018-07-10 Digital Entrep POC - Web Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": "true"
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Notebook-setup\" data-toc-modified-id=\"Notebook-setup-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Notebook setup</a></span><ul class=\"toc-item\"><li><span><a href=\"#Import-required-libraries\" data-toc-modified-id=\"Import-required-libraries-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Import required libraries</a></span></li><li><span><a href=\"#Define-notebook-wide-variables\" data-toc-modified-id=\"Define-notebook-wide-variables-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Define notebook-wide variables</a></span></li><li><span><a href=\"#Create-output-directories\" data-toc-modified-id=\"Create-output-directories-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Create output directories</a></span></li><li><span><a href=\"#Define-helper-functions\" data-toc-modified-id=\"Define-helper-functions-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Define helper functions</a></span></li></ul></li><li><span><a href=\"#Extract-company-list-data\" data-toc-modified-id=\"Extract-company-list-data-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Extract company list data</a></span><ul class=\"toc-item\"><li><span><a href=\"#Afrilabs---map-of-African-accelerators-/-hubs\" data-toc-modified-id=\"Afrilabs---map-of-African-accelerators-/-hubs-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Afrilabs - map of African accelerators / hubs</a></span></li><li><span><a href=\"#Startups-List----global-country-specific-listings\" data-toc-modified-id=\"Startups-List----global-country-specific-listings-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Startups List -- global country-specific listings</a></span></li><li><span><a href=\"#Google-Places-API\" data-toc-modified-id=\"Google-Places-API-2.3\"><span class=\"toc-item-num\">2.3&nbsp;&nbsp;</span>Google Places API</a></span><ul class=\"toc-item\"><li><span><a href=\"#Nearby-places\" data-toc-modified-id=\"Nearby-places-2.3.1\"><span class=\"toc-item-num\">2.3.1&nbsp;&nbsp;</span>Nearby places</a></span></li><li><span><a href=\"#Text-search\" data-toc-modified-id=\"Text-search-2.3.2\"><span class=\"toc-item-num\">2.3.2&nbsp;&nbsp;</span>Text search</a></span></li></ul></li></ul></li><li><span><a href=\"#Extract-Company-Metadata\" data-toc-modified-id=\"Extract-Company-Metadata-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Extract Company Metadata</a></span><ul class=\"toc-item\"><li><span><a href=\"#Google-Place-API---Grab-Place-details-based-on-Place-IDs\" data-toc-modified-id=\"Google-Place-API---Grab-Place-details-based-on-Place-IDs-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Google Place API - Grab Place details based on Place IDs</a></span></li><li><span><a href=\"#Google-Search-Query-API\" data-toc-modified-id=\"Google-Search-Query-API-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Google Search Query API</a></span></li><li><span><a href=\"#Twitter-data\" data-toc-modified-id=\"Twitter-data-3.3\"><span class=\"toc-item-num\">3.3&nbsp;&nbsp;</span>Twitter data</a></span><ul class=\"toc-item\"><li><span><a href=\"#get-tweets-by-keyword\" data-toc-modified-id=\"get-tweets-by-keyword-3.3.1\"><span class=\"toc-item-num\">3.3.1&nbsp;&nbsp;</span>get tweets by keyword</a></span></li><li><span><a href=\"#get-tweets-from-user-timeline\" data-toc-modified-id=\"get-tweets-from-user-timeline-3.3.2\"><span class=\"toc-item-num\">3.3.2&nbsp;&nbsp;</span>get tweets from user timeline</a></span></li></ul></li><li><span><a href=\"#From-company-website\" data-toc-modified-id=\"From-company-website-3.4\"><span class=\"toc-item-num\">3.4&nbsp;&nbsp;</span>From company website</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-16T03:50:20.801726Z",
     "start_time": "2019-05-16T03:50:20.798734Z"
    }
   },
   "outputs": [],
   "source": [
    "# please import these required packages before running the code.\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import glob\n",
    "import os\n",
    "import datetime\n",
    "import time\n",
    "\n",
    "import requests\n",
    "import re\n",
    "import json\n",
    "import bs4\n",
    "\n",
    "import googlemaps\n",
    "from googleapiclient.discovery import build\n",
    "import pprint\n",
    "\n",
    "import twitter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define notebook-wide variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-06T20:25:36.534686Z",
     "start_time": "2018-09-06T20:25:36.531686Z"
    }
   },
   "outputs": [],
   "source": [
    "# generate timestamp for today for output files\n",
    "date_today = datetime.date.today().isoformat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input Places API key\n",
    "gmaps = googlemaps.Client(key='<please insert your own Gmaps key here>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 520,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:58:18.618142Z",
     "start_time": "2018-07-10T16:58:18.602665Z"
    }
   },
   "outputs": [],
   "source": [
    "# Google Search API keys here\n",
    "my_api_key = '<please input your own API key here>'\n",
    "my_cse_id = '<please input your CSE ID here>'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create output directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-16T03:49:51.082733Z",
     "start_time": "2019-05-16T03:49:51.077759Z"
    }
   },
   "outputs": [],
   "source": [
    "output_folder = \"outputs\"\n",
    "\n",
    "os.makedirs(output_folder, exist_ok=True)\n",
    "for subfolder in ['1 extract lists', '2 extract metadata']:\n",
    "    os.makedirs(os.path.join(output_folder, subfolder), exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:11:25.216015Z",
     "start_time": "2018-07-10T16:11:25.200390Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_dummy_type_cols(df_gmaps_all, col=\"types\"):\n",
    "    \n",
    "    for genre in set.union(*df_gmaps_all[col].apply(set)):\n",
    "        df_gmaps_all[genre] = df_gmaps_all.apply(lambda _: int(_[col].count(genre)), axis=1)\n",
    "    \n",
    "    return(df_gmaps_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:58:18.882961Z",
     "start_time": "2018-07-10T16:58:18.867432Z"
    }
   },
   "outputs": [],
   "source": [
    "def google_search(search_term, api_key, cse_id, **kwargs):\n",
    "    service = build(\"customsearch\", \"v1\", developerKey=api_key)\n",
    "    res = service.cse().list(q=search_term, cx=cse_id, **kwargs).execute()\n",
    "    return(res['items'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 635,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:42:31.001035Z",
     "start_time": "2018-07-10T17:42:30.938456Z"
    }
   },
   "outputs": [],
   "source": [
    "def extract_status_data(results):\n",
    "    status_cols = [\n",
    "        'created_at', \n",
    "    'favorited', \n",
    "    'id', \n",
    "    'text', \n",
    "    'location', \n",
    "    'in_reply_to_screen_name', \n",
    "    'in_reply_to_user_id', \n",
    "    'in_reply_to_status_id', \n",
    "    'truncated', \n",
    "    'retweeted', \n",
    "    'source', \n",
    "    'user', \n",
    "    'urls', \n",
    "    'user_mentions', \n",
    "    'hashtags', \n",
    "    'geo', \n",
    "    'place', \n",
    "    'coordinates', \n",
    "    'contributors', \n",
    "    'retweeted_status'\n",
    "    ]\n",
    "\n",
    "    twitter_data_all = []\n",
    "\n",
    "    for result in results:\n",
    "        created_at_res = result.created_at\n",
    "        favorited_res = result.favorited\n",
    "        id_res = result.id\n",
    "        text_res = result.text\n",
    "        location_res = result.location\n",
    "        in_reply_to_screen_name_res = result.in_reply_to_screen_name\n",
    "        in_reply_to_user_id_res = result.in_reply_to_user_id\n",
    "        in_reply_to_status_id_res = result.in_reply_to_status_id\n",
    "        truncated_res = result.truncated\n",
    "        retweeted_res = result.retweeted\n",
    "        source_res = result.source\n",
    "        user_res = result.user\n",
    "        urls_res = result.urls\n",
    "        user_mentions_res = result.user_mentions\n",
    "        hashtags_res = result.hashtags\n",
    "        geo_res = result.geo\n",
    "        place_res = result.place\n",
    "        coordinates_res = result.coordinates\n",
    "        contributors_res = result.contributors\n",
    "        retweeted_status_res = result.retweeted_status\n",
    "\n",
    "        twitter_data_all.append([created_at_res, \n",
    "            favorited_res, \n",
    "            id_res, \n",
    "            text_res, \n",
    "            location_res, \n",
    "            in_reply_to_screen_name_res, \n",
    "            in_reply_to_user_id_res, \n",
    "            in_reply_to_status_id_res, \n",
    "            truncated_res, \n",
    "            retweeted_res, \n",
    "            source_res, \n",
    "            user_res, \n",
    "            urls_res, \n",
    "            user_mentions_res, \n",
    "            hashtags_res, \n",
    "            geo_res, \n",
    "            place_res, \n",
    "            coordinates_res, \n",
    "            contributors_res, \n",
    "            retweeted_status_res, \n",
    "            ])\n",
    "        \n",
    "    df_twitter_data = pd.DataFrame(twitter_data_all, columns=status_cols)\n",
    "    return(df_twitter_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract company list data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Afrilabs - map of African accelerators / hubs\n",
    "Data source: http://www.afrilabs.com/afrilabs-passport/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T12:43:29.521231Z",
     "start_time": "2018-07-10T12:43:27.659691Z"
    }
   },
   "outputs": [],
   "source": [
    "# pull page source of Afrilabs page\n",
    "url = 'http://www.afrilabs.com/afrilabs-passport/'\n",
    "r = requests.get(url)\n",
    "page_source = r.content\n",
    "soup = bs4.BeautifulSoup(page_source, \"lxml\")\n",
    "\n",
    "# extract maps script\n",
    "script = soup.find('script', text=re.compile('\\.maps'))\n",
    "\n",
    "# extract the maps data\n",
    "json_text = re.search(r'.*\\.maps\\((.*)\\)\\.data\\(\"wpgmp_maps\"\\);}\\);$', script.string).group(1)\n",
    "data = json.loads(json_text)\n",
    "df_afrilabs = pd.DataFrame(data['places'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T13:51:35.976397Z",
     "start_time": "2018-07-10T13:51:35.960753Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'id': '1', 'name': 'Afri', 'type': 'category', 'extension_fields': False, 'icon': 'http://www.afrilabs.com/wp-content/plugins/wp-google-map-plugin//assets/images/icons/justice.png'}]    57\n",
      "Name: categories, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# don't mind categories column since contains redundant data\n",
    "print(df_afrilabs['categories'].astype(str).value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T13:51:40.712457Z",
     "start_time": "2018-07-10T13:51:40.634352Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mrpso\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "# extract nested data on location\n",
    "df_afrilabs_location = pd.DataFrame()\n",
    "for ix in df_afrilabs.index:\n",
    "    df_temp = pd.DataFrame.from_dict(df_afrilabs['location'].ix[ix], orient='index')\n",
    "    df_temp.columns = [ix]\n",
    "    df_temp = df_temp.T\n",
    "    df_afrilabs_location = df_afrilabs_location.append(df_temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T13:53:51.515853Z",
     "start_time": "2018-07-10T13:53:51.484708Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# merge scraped data\n",
    "df_afrilabs_full = df_afrilabs.join(df_afrilabs_location)[list(df_afrilabs.columns) + list(df_afrilabs_location.columns)]\n",
    "df_afrilabs_full.to_csv(\"outputs/1 extract lists/%s - Afrilabs List of Accelerators or Hubs.csv\" % date_today)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T13:51:42.711247Z",
     "start_time": "2018-07-10T13:51:42.680101Z"
    }
   },
   "source": [
    "## Startups List -- global country-specific listings\n",
    "Data source: http://nigeria.startups-list.com/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T14:19:08.697720Z",
     "start_time": "2018-07-10T14:19:05.382550Z"
    }
   },
   "outputs": [],
   "source": [
    "# grab page content\n",
    "url = 'http://nigeria.startups-list.com/'\n",
    "r = requests.get(url)\n",
    "page_source = r.content\n",
    "soup = bs4.BeautifulSoup(page_source, \"lxml\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T14:22:24.664214Z",
     "start_time": "2018-07-10T14:22:24.586071Z"
    }
   },
   "outputs": [],
   "source": [
    "# extract code with startups\n",
    "list_startups = soup.find_all(\"div\", {'class': 'startup'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract key elements per code chunk\n",
    "- startup name `<h1 property=\"name\">`\n",
    "- startup description `<p>`\n",
    "- startup link outer `<div data-href= attribute>`\n",
    "- startup keywords `<img alt attribute>`\n",
    "- startup logo `<img src= attribute />`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T14:49:35.032675Z",
     "start_time": "2018-07-10T14:49:34.720218Z"
    }
   },
   "outputs": [],
   "source": [
    "list_startups_data = []\n",
    "\n",
    "for startup in list_startups:\n",
    "    temp = startup.find(\"h1\")\n",
    "    startup_name = \" \".join(re.split(\"\\s+\", bs4.BeautifulSoup(' '.join([str(x) for x in temp.contents]), \"lxml\").text.strip(), flags=re.UNICODE))\n",
    "\n",
    "    temp = startup.find(\"p\")\n",
    "    startup_description = \" \".join(re.split(\"\\s+\", bs4.BeautifulSoup(' '.join([str(x) for x in temp.contents]), \"lxml\").text.strip(), flags=re.UNICODE))\n",
    "\n",
    "    startup_link = startup.get('data-href')\n",
    "\n",
    "    startup_keywords = startup.find('img').get('alt')\n",
    "\n",
    "    startup_logo_img_src = startup.find('img').get('src')\n",
    "    \n",
    "    if (startup_logo_img_src == \"/img/spacer.gif\") | (startup_logo_img_src == \"\"):\n",
    "        try:\n",
    "            startup_logo_img_src = startup.find('img').get('data-src')\n",
    "        except:\n",
    "            startup_logo_img_src = \"\"\n",
    "\n",
    "    list_startups_data.append([startup_name, startup_description, startup_link, startup_keywords, startup_logo_img_src])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T14:49:36.759284Z",
     "start_time": "2018-07-10T14:49:36.743636Z"
    }
   },
   "outputs": [],
   "source": [
    "df_startupslist = pd.DataFrame(list_startups_data, columns = ['startup_name', 'startup_description','startup_link','startup_keywords','startup_logo_img_src'])\n",
    "\n",
    "# clean invalid image logos\n",
    "df_startupslist = df_startupslist.replace({'/img/spacer.gif': np.nan, 'https://angel.co/images/shared/nopic_startup.png': np.nan})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T14:49:57.774999Z",
     "start_time": "2018-07-10T14:49:57.743767Z"
    }
   },
   "outputs": [],
   "source": [
    "df_startupslist.to_csv(\"outputs/1 extract lists/%s - Nigeria Startups List.csv\" % date_today)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Google Places API\n",
    "Documentation here: https://developers.google.com/places/web-service/search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nearby places"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T15:42:48.251265Z",
     "start_time": "2018-07-10T15:42:42.946873Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch 1 done\n",
      "batch 2 done\n",
      "(60, 13)\n"
     ]
    }
   ],
   "source": [
    "# get first N batches of 20 results\n",
    "batch_num = 50\n",
    "\n",
    "# use kenya capital (Abuja) geocoordinates. Approximate radius with 480km\n",
    "results = gmaps.places_nearby(radius=520000, location=(9.072264, 7.491302))\n",
    "df_gmaps_all = pd.DataFrame(results['results'])\n",
    "\n",
    "for batch in range(batch_num):\n",
    "    # get next 20 results\n",
    "    try:\n",
    "        page_token_next = results['next_page_token']\n",
    "    except:\n",
    "        break\n",
    "\n",
    "    # wait for 2 seconds before getting new data\n",
    "    time.sleep(2)\n",
    "    results = gmaps.places_nearby(page_token=page_token_next)\n",
    "    \n",
    "    df_gmaps_all = df_gmaps_all.append(pd.DataFrame(results['results']))\n",
    "    \n",
    "    print(\"batch %s done\" % str(batch + 1))\n",
    "    \n",
    "df_gmaps_all.reset_index(drop=True, inplace=True)\n",
    "print(df_gmaps_all.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T15:51:36.071598Z",
     "start_time": "2018-07-10T15:51:36.024701Z"
    }
   },
   "outputs": [],
   "source": [
    "for genre in set.union(*df_gmaps_all['types'].apply(set)):\n",
    "    df_gmaps_all[genre] = df_gmaps_all.apply(lambda _: int(_['types'].count(genre)), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T15:54:18.871770Z",
     "start_time": "2018-07-10T15:54:18.840496Z"
    }
   },
   "outputs": [],
   "source": [
    "df_gmaps_all.to_csv(\"outputs/1 extract lists/%s - Nigeria Google Places API - Nearby Places endpoint.csv\" % date_today)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:08:42.642491Z",
     "start_time": "2018-07-10T16:08:39.400775Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kenya\n",
      "accelerator (1, 14)\n",
      "hub (9, 14)\n",
      "startup (0, 2)\n",
      "business (20, 14)\n",
      "company (20, 14)\n",
      "incubator (3, 14)\n",
      "(53, 14)\n"
     ]
    }
   ],
   "source": [
    "keywords = [\"accelerator\", \"hub\", \"startup\", \"business\", \"company\", \"incubator\"]\n",
    "country = \"Kenya\"\n",
    "cou_id = \"ke\"\n",
    "\n",
    "df_gmaps_textsearch = pd.DataFrame()\n",
    "\n",
    "print(country)\n",
    "for keyword in keywords:\n",
    "    \n",
    "    results_textsearch = gmaps.places(keyword, region=cou_id)\n",
    "    df_temp = pd.DataFrame(results_textsearch['results'])\n",
    "\n",
    "    df_temp['keyword_for_text_search'] = keyword\n",
    "    df_temp['country_for_text_search'] = country\n",
    "    print(keyword, df_temp.shape)\n",
    "    df_gmaps_textsearch = df_gmaps_textsearch.append(df_temp)\n",
    "    \n",
    "df_gmaps_textsearch = df_gmaps_textsearch.reset_index(drop=True)\n",
    "df_gmaps_textsearch = get_dummy_type_cols(df_gmaps_textsearch)\n",
    "\n",
    "print(df_gmaps_textsearch.shape)\n",
    "\n",
    "df_gmaps_textsearch.to_csv(\"outputs/1 extract lists/%s - %s Google Places API - Text Search endpoint.csv\" % (date_today, country))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:17:23.941017Z",
     "start_time": "2018-07-10T16:17:23.909720Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_gmaps_textsearch_all = pd.concat([df_gmaps_textsearch, df_gmaps_textsearch_id])[df_gmaps_textsearch.columns]\n",
    "df_gmaps_textsearch_all.to_csv(\"outputs/1 extract lists/%s - Consolidated Google Places API - Text Search endpoint.csv\" % (date_today))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 660,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T20:56:56.514129Z",
     "start_time": "2018-07-10T20:56:56.482886Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((53, 28), (24, 21))"
      ]
     },
     "execution_count": 660,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gmaps_textsearch.shape, df_gmaps_textsearch_id.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Company Metadata\n",
    "We focus extracting data on these 3 Indonesian companies searched via Google Places API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 499,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:36:36.321912Z",
     "start_time": "2018-07-10T16:36:36.306265Z"
    }
   },
   "outputs": [],
   "source": [
    "df_extract = df_gmaps_textsearch_all[df_gmaps_textsearch_all['keyword_for_text_search'].isin([\"incubator\", \"accelerator\"]) & (df_gmaps_textsearch_all['country_for_text_search']== 'Indonesia')].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Google Place API - Grab Place details based on Place IDs\n",
    "https://developers.google.com/places/web-service/details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:36:01.695769Z",
     "start_time": "2018-07-10T16:36:01.680148Z"
    }
   },
   "outputs": [],
   "source": [
    "place_id_list = df_extract['place_id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:34:20.554162Z",
     "start_time": "2018-07-10T16:34:20.029230Z"
    }
   },
   "outputs": [],
   "source": [
    "df_place_details = pd.DataFrame()\n",
    "for place_id in place_id_list:\n",
    "    test_place_indonesia = gmaps.place(place_id)\n",
    "\n",
    "    consolidate = []\n",
    "    for key in test_place_indonesia['result'].keys():\n",
    "        consolidate.append(test_place_indonesia['result'][key])\n",
    "\n",
    "    df_temp = pd.DataFrame(consolidate[1:]).T\n",
    "    df_temp.columns = list(test_place_indonesia['result'].keys())[1:]\n",
    "    \n",
    "    df_place_details = pd.concat([df_place_details, df_temp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 490,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:34:20.897745Z",
     "start_time": "2018-07-10T16:34:20.882089Z"
    }
   },
   "outputs": [],
   "source": [
    "df_place_details = df_place_details.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T16:38:48.063011Z",
     "start_time": "2018-07-10T16:38:48.031904Z"
    }
   },
   "outputs": [],
   "source": [
    "df_place_details.to_csv(\"outputs/2 extract metadata/%s - Indonesia Google Places API - Place Details endpoint.csv\" % (date_today))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Google Search Query API\n",
    "Top Google search results for the company such as relevant articles, etc.\n",
    "\n",
    "Parameter info here: https://developers.google.com/custom-search/json-api/v1/reference/cse/list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 642,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:44:19.013028Z",
     "start_time": "2018-07-10T17:44:17.586391Z"
    }
   },
   "outputs": [],
   "source": [
    "cou_id = 'id'\n",
    "results = google_search('GnB Accelerator', my_api_key, my_cse_id, num=10, gl=cou_id, cr = cou_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:44:19.028648Z",
     "start_time": "2018-07-10T17:44:19.013028Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_google_search = pd.DataFrame(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 645,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:44:41.529201Z",
     "start_time": "2018-07-10T17:44:41.498079Z"
    }
   },
   "outputs": [],
   "source": [
    "df_google_search.to_csv(\"outputs/2 extract metadata/%s - Indonesia GnB Accelerator Google Search API.csv\" % (date_today))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Twitter data\n",
    "Top tweets resuls for the company such as relevant articles, etc.\n",
    "\n",
    "Use Twitter API to get public tweets or search relevant tweets \n",
    "\n",
    "Python client library: https://github.com/bear/python-twitter\n",
    "\n",
    "Twitter Standard (FREE) tier is LIMITED -- This search API searches against a sampling of recent Tweets published in the past 7 days. Part of the 'public' set of APIs.\n",
    "\n",
    "Other endpoints have max requests per 15-min window\n",
    "\n",
    "More info here:\n",
    "- https://apps.twitter.com\n",
    "- https://developer.twitter.com/en/docs/tweets/search/api-reference/get-search-tweets.html\n",
    "- https://developer.twitter.com/en/docs/tweets/timelines/api-reference/get-statuses-user_timeline\n",
    "- https://developer.twitter.com/en/docs/basics/authentication/overview/application-only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-06T20:25:39.084730Z",
     "start_time": "2018-09-06T20:25:39.069107Z"
    }
   },
   "outputs": [],
   "source": [
    "api = twitter.Api(consumer_key='<add consumer key here>',\n",
    "                      consumer_secret='<add consumer secret here>',\n",
    "                      access_token_key= '<add access token key here>',\n",
    "                      access_token_secret='<add access token secret here>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-16T03:43:39.546920Z",
     "start_time": "2019-05-16T03:43:39.535975Z"
    }
   },
   "outputs": [],
   "source": [
    "print(api.VerifyCredentials())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-16T03:43:57.417996Z",
     "start_time": "2019-05-16T03:43:57.414005Z"
    }
   },
   "outputs": [],
   "source": [
    "api.GetUserTimeline(screen_name='<add screenname here>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-16T03:44:22.241515Z",
     "start_time": "2019-05-16T03:44:22.238524Z"
    }
   },
   "outputs": [],
   "source": [
    "api.GetFollowersPaged(screen_name='<add screenname here>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-16T03:44:17.979907Z",
     "start_time": "2019-05-16T03:44:17.975918Z"
    }
   },
   "outputs": [],
   "source": [
    "api.ShowFriendship(source_screen_name='<add screenname here>', target_screen_name='<add another screenname here>')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get tweets by keyword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 626,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:39:27.850732Z",
     "start_time": "2018-07-10T17:39:27.467611Z"
    }
   },
   "outputs": [],
   "source": [
    "results = api.GetSearch(raw_query=\"q=gnb%20accelerator&count=100\", include_entities=True)\n",
    "# results = api.GetSearch(raw_query='q=%22mad%20incubator%22')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 637,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:42:54.012715Z",
     "start_time": "2018-07-10T17:42:53.981472Z"
    }
   },
   "outputs": [],
   "source": [
    "df_twitter_data = extract_status_data(results)\n",
    "df_twitter_data['TwitterAPIsource'] = 'keyword_search'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get tweets from user timeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 633,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:41:42.172068Z",
     "start_time": "2018-07-10T17:41:41.812659Z"
    }
   },
   "outputs": [],
   "source": [
    "statuses = api.GetUserTimeline(screen_name='GnBAccelerator')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 663,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T21:47:48.937175Z",
     "start_time": "2018-07-10T21:47:48.906058Z"
    }
   },
   "outputs": [],
   "source": [
    "df_twitter_usertimeline = extract_status_data(statuses)\n",
    "df_twitter_usertimeline['TwitterAPIsource'] = 'user_timeline'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 665,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T21:48:15.459039Z",
     "start_time": "2018-07-10T21:48:15.443442Z"
    }
   },
   "outputs": [],
   "source": [
    "df_twitter_data_all = pd.concat([df_twitter_data,df_twitter_usertimeline])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 666,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T21:48:24.072887Z",
     "start_time": "2018-07-10T21:48:24.057262Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "keyword_search    10\n",
       "user_timeline      5\n",
       "Name: TwitterAPIsource, dtype: int64"
      ]
     },
     "execution_count": 666,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_twitter_data_all['TwitterAPIsource'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 667,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T21:49:11.595599Z",
     "start_time": "2018-07-10T21:49:11.564353Z"
    }
   },
   "outputs": [],
   "source": [
    "df_twitter_data_all.to_csv(\"outputs/2 Extract Company Metadata/%s - Indonesia GnB Accelerator Twitter API.csv\" % (date_today))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## From company website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 648,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:46:30.718517Z",
     "start_time": "2018-07-10T17:46:30.702896Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'GnB Accelerator': 'https://gnb.ac/',\n",
       " 'Mad Incubator': 'http://www.incubator.com.my/',\n",
       " 'The Accelerator': 'http://www.accelerator.co.id/'}"
      ]
     },
     "execution_count": 648,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "website_list = df_place_details[['name', 'website']].set_index('name').to_dict()['website']\n",
    "website_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 654,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:49:24.700000Z",
     "start_time": "2018-07-10T17:49:24.684374Z"
    }
   },
   "outputs": [],
   "source": [
    "company = 'GnB Accelerator'\n",
    "url = website_list[company]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 655,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:49:28.513860Z",
     "start_time": "2018-07-10T17:49:25.013580Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\mrpso\\Anaconda3\\lib\\site-packages\\urllib3\\connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  \n",
      "C:\\Users\\mrpso\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file C:\\Users\\mrpso\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "# grab page content\n",
    "r = requests.get(url, verify=False)\n",
    "page_source = r.content\n",
    "soup = bs4.BeautifulSoup(page_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 657,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:50:50.492117Z",
     "start_time": "2018-07-10T17:50:50.460879Z"
    }
   },
   "outputs": [],
   "source": [
    "html = soup.prettify(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 659,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-10T17:51:25.920577Z",
     "start_time": "2018-07-10T17:51:25.873713Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "html = soup.prettify(\"utf-8\")\n",
    "with open(\"outputs/2 extract metadata/%s - Indonesia GnB Accelerator Website data.html\" % (date_today), \"wb\") as file:\n",
    "    file.write(html)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "274px",
    "width": "252px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "798px",
    "left": "0px",
    "right": "1690px",
    "top": "111px",
    "width": "230px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}