instagram_scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from IPython.core.display import display, HTML\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import os\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = pd.read_csv(\"celebrity_handles.csv\")\n",
    "df = pd.read_csv(\"realitytv_handles.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = \"https://www.instagram.com/\"\n",
    "urls = []\n",
    "for i, row in df.iterrows():\n",
    "    url = base_url + row[\"handle\"].replace(\"@\",\"\")\n",
    "    urls.append(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up chromedriver\n",
    "chromedriver = \"/Users/katherinepully/chromedriver\"\n",
    "os.environ[\"webdriver.chrome.driver\"] = chromedriver\n",
    "driver = webdriver.Chrome(chromedriver)\n",
    "actions = ActionChains(driver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=0\n",
    "\n",
    "driver.get(urls[i])\n",
    "\n",
    "acct_handle = urls[i].replace(base_url, \"\")\n",
    "\n",
    "images = driver.find_elements_by_class_name(\"_bz0w\")\n",
    "images_count = len(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_curr = images[1].find_element_by_tag_name(\"a\").get_attribute(\"href\")\n",
    "driver.get(image_curr) #go to first picture\n",
    "\n",
    "img_id = driver.current_url.replace(base_url + \"p/\", \"\")[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 pages loaded\n",
      "10 pages loaded\n",
      "15 pages loaded\n",
      "20 pages loaded\n",
      "25 pages loaded\n",
      "30 pages loaded\n",
      "35 pages loaded\n",
      "40 pages loaded\n",
      "45 pages loaded\n",
      "50 pages loaded\n",
      "final click count: 50; should yield roughly 600 comments\n"
     ]
    }
   ],
   "source": [
    "# approximately 12 comments per page\n",
    "click_count=0\n",
    "\n",
    "while (click_count<50):\n",
    "    try:\n",
    "        load_more = driver.find_element_by_class_name(\"glyphsSpriteCircle_add__outline__24__grey_9.u-__7\")\n",
    "        actions.move_to_element(load_more).perform()\n",
    "        load_more.click()\n",
    "        click_count+=1\n",
    "        if click_count%5==0:\n",
    "            print(str(click_count) + \" pages loaded\")\n",
    "        time.sleep(1)\n",
    "    except Exception as e:\n",
    "        time.sleep(2)\n",
    "        try:\n",
    "            load_more = driver.find_element_by_class_name(\"glyphsSpriteCircle_add__outline__24__grey_9.u-__7\")\n",
    "            actions.move_to_element(load_more).perform()\n",
    "            load_more.click()\n",
    "            click_count+=1\n",
    "            if click_count%5==0:\n",
    "                print(str(click_count) + \" pages loaded\")\n",
    "        except:\n",
    "            print(e)\n",
    "            break\n",
    "        \n",
    "print(\"final click count: \" + str(click_count) + \"; should yield roughly \" + str(click_count*12) + \" comments\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total comments: 514\n",
      "\n"
     ]
    }
   ],
   "source": [
    "comments = driver.find_elements_by_class_name(\"C4VMK\")\n",
    "print(\"Total comments: \" + str(len(comments)) + \"\\n\")\n",
    "\n",
    "comments_list = []\n",
    "users_list = []\n",
    "\n",
    "for c in comments:\n",
    "    comment = c.find_element_by_css_selector('span').get_attribute(\"textContent\")\n",
    "    user = c.find_element_by_class_name(\"TlrDj\").get_attribute(\"textContent\")\n",
    "    #     print(\"\" + user + \": \" + str(comment))\n",
    "    comments_list.append(comment)\n",
    "    users_list.append(user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ = pd.DataFrame({\"user\": users_list, \"comment\": comments_list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>comment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>laurenbushnell</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>efergie13</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>colympios</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>andreeacristina</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>amberlancaster</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              user   comment\n",
       "0   laurenbushnell  Verified\n",
       "1        efergie13  Verified\n",
       "2        colympios  Verified\n",
       "3  andreeacristina  Verified\n",
       "4   amberlancaster  Verified"
      ]
     },
     "execution_count": 338,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"data/\"+acct_handle + \"/\"\n",
    "if not os.path.exists(path):\n",
    "    os.makedirs(path)\n",
    "    \n",
    "file_name = str(img_id)+\".csv\"\n",
    "    \n",
    "df_.to_csv(path+file_name, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "# misc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}