Scraper.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Installing Dependencies needed for Scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bs4 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (0.0.1)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from bs4) (4.9.3)\n",
      "Requirement already satisfied: soupsieve>1.2; python_version >= \"3.0\" in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from beautifulsoup4->bs4) (2.0.1)\n",
      "Requirement already satisfied: requests in c:\\users\\ekami\\anaconda3\\lib\\site-packages (2.24.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from requests) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from requests) (2020.6.20)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from requests) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from requests) (1.25.11)\n",
      "Requirement already satisfied: pandas in c:\\users\\ekami\\anaconda3\\lib\\site-packages (1.1.3)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: numpy>=1.15.4 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from pandas) (1.19.2)\n",
      "Requirement already satisfied: pytz>=2017.2 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from pandas) (2020.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
      "Requirement already satisfied: datetime in c:\\users\\ekami\\anaconda3\\lib\\site-packages (4.3)\n",
      "Requirement already satisfied: zope.interface in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from datetime) (5.1.2)\n",
      "Requirement already satisfied: pytz in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from datetime) (2020.1)\n",
      "Requirement already satisfied: setuptools in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from zope.interface->datetime) (50.3.1.post20201107)\n",
      "Requirement already satisfied: openpyxl in c:\\users\\ekami\\anaconda3\\lib\\site-packages (3.0.5)\n",
      "Requirement already satisfied: jdcal in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from openpyxl) (1.4.1)\n",
      "Requirement already satisfied: et-xmlfile in c:\\users\\ekami\\anaconda3\\lib\\site-packages (from openpyxl) (1.0.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install bs4\n",
    "!pip install requests\n",
    "!pip install pandas\n",
    "!pip install datetime\n",
    "!pip install openpyxl\n",
    "from bs4 import BeautifulSoup\n",
    "import requests as rq\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import sys\n",
    "import random\n",
    "import time as t\n",
    "import re\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This is where the method used to scrap from Upworks is created. To avoid timeouts from the website, a limit of 1 page scraped per 15 seconds is made. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape(pages):\n",
    "\n",
    "    job_list = []\n",
    "    descr_list = []\n",
    "    time_stamp = []\n",
    "    payment_type = []\n",
    "    duration = []\n",
    "    workload = []\n",
    "    experience = []\n",
    "    budget = []\n",
    "    link_list = []\n",
    "\n",
    "    for r in range(0,pages):\n",
    "        #getting html soup\n",
    "        url = \"https://www.upwork.com/ab/jobs/search/?category2_uid=531770282580668418&page=\"+str(r)+\"&sort=recency&user_location_match=1\"\n",
    "        agent = {\"User-Agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36\"}\n",
    "        source = rq.get(url, headers=agent).text\n",
    "        soup = BeautifulSoup(source, 'html.parser')\n",
    "        \n",
    "\n",
    "        if \"Access may\" in soup.text:\n",
    "            sys.exit(\"Access Denied \\n Either User-Agent is incorrect or too many requests were made\")\n",
    "\n",
    "        #finding each individual job post\n",
    "        jobs = soup.findAll('section',{'class':\"air-card air-card-hover job-tile-responsive\"})\n",
    "\n",
    "        for x in jobs:\n",
    "            \n",
    "            ########\n",
    "            link = x.find('a',href=True)\n",
    "            link = \"https://www.upwork.com\"+str(link['href'])\n",
    "            \n",
    "            link_list.append(link)\n",
    "            job_list.append(x.find('up-c-line-clamp',{'lines':\"2\"}).text)\n",
    "            descr_list.append(x.find('span',{\"class\":\"js-description-text\"}).text)\n",
    "\n",
    "\n",
    "            time = x.time['datetime']\n",
    "            time = datetime.fromisoformat(time)\n",
    "            diff = datetime.utcnow()-time.replace(tzinfo=None)\n",
    "            time_stamp.append(round(diff.total_seconds()/3600,2))\n",
    "\n",
    "            #Project payment type, either hourly or fixed. If hourly rate separated\n",
    "            payment = x.find('strong',{'class':'js-type text-muted'}).text\n",
    "            if \"Hourly\" in payment:\n",
    "                if \"$\" in payment:\n",
    "                    ind = payment.rfind('$')\n",
    "                    budget.append(float(payment[ind+1:].strip().strip('\\n')))\n",
    "                else:\n",
    "                    budget.append(None)\n",
    "                payment_type.append(\"Hourly\")\n",
    "            else:\n",
    "                payment_type.append(\"Fixed\")\n",
    "\n",
    "            #Total number of months expected to complete project, only for hourly\n",
    "            dur = x.find('strong',{'class':'js-duration'},text = True)\n",
    "            if dur != None:\n",
    "                duration.append(dur.text)\n",
    "            else:\n",
    "                duration.append(None)\n",
    "\n",
    "            #Weekly hours need to commit, only for hourly projects\n",
    "            wkload = x.find('strong',{'class':'js-workload'})\n",
    "            if wkload != None:\n",
    "                workload.append(wkload.text)\n",
    "            else:\n",
    "                workload.append(None)\n",
    "\n",
    "            #Difficutly of project\n",
    "            experience.append(x.find('strong',{'class':'js-contractor-tier'}).text)\n",
    "\n",
    "            #Budget of fixed projects\n",
    "            bug = x.find('strong',{'class':'js-budget'})\n",
    "            if bug != None:\n",
    "                bug = bug.find('span').text.strip()[1:].strip('\\n')\n",
    "                if ',' in str(bug):\n",
    "                    budget.append(float(bug.replace(',','')))\n",
    "                else:\n",
    "                    budget.append(float(bug))\n",
    "                    \n",
    "        print(\"Time: \"+ str(.25*(r+1)) +\" mins out of \"+str(pages/4))\n",
    "        t.sleep(15)\n",
    "\n",
    "    df = pd.DataFrame({\n",
    "        'Url':link_list,\n",
    "        'Job Name':job_list,\n",
    "        'Job Description':descr_list,\n",
    "        'Payment Type':payment_type,\n",
    "        'Budget':budget,\n",
    "        'Workload':workload,\n",
    "        'Level':experience,\n",
    "        'Est. Time':duration,\n",
    "        'Time Created':time_stamp\n",
    "    })\n",
    "    name = \"Jobs.xlsx\"\n",
    "    try:\n",
    "        os.remove(name)\n",
    "        try:\n",
    "            df = df.dropna(axis=0, subset=['Budget'])\n",
    "            df.to_excel(name)\n",
    "        except PermissionError:\n",
    "            name = \"Jobs\"+str(random.randrange(100))+\".xlsx\"\n",
    "            df = df.dropna(axis=0, subset=['Budget'])\n",
    "            df.to_excel(name)\n",
    "\n",
    "    except OSError:\n",
    "        df = df.dropna(axis=0, subset=['Budget'])\n",
    "        df.to_excel(name)\n",
    "    # try:\n",
    "    #     os.remove('Jobs.xlsx')\n",
    "    #\n",
    "    # except OSError:\n",
    "    #     pass\n",
    "    # df = df.dropna(axis=0, subset=['Budget'])\n",
    "    # df.to_excel(\"Jobs.xlsx\")\n",
    "    return name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This is where the resulting job posts are filtered and Sorted based on relevancy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter(name):\n",
    "    df = pd.read_excel(name)\n",
    "    \n",
    "    ################################# This is where to add words to filters or change limits ##################################\n",
    "    banned_words = ['drupal','wordpress','woocommerce','shopify','squarespace']\n",
    "    react = ['React Native','react','React']\n",
    "    hardware = ['Hardware','embedded','microcontroller']\n",
    "    scraping = ['Scrape','scraping']\n",
    "    sports = ['sport','basketball','football','soccer','baseball']\n",
    "    fixed_limit = 1000.00\n",
    "    hourly_limit = 40.00\n",
    "\n",
    "    drops = []\n",
    "    c_react = []\n",
    "    c_hardware = []\n",
    "    c_scraping = []\n",
    "    c_sports = []\n",
    "    for index,row in df.iterrows():\n",
    "\n",
    "        val = False\n",
    "        for x in react:\n",
    "            if (x.upper() in row['Job Description'].upper()) | (x.upper() in row['Job Name'].upper()):\n",
    "                val = val | True\n",
    "            else:\n",
    "                val = val\n",
    "        c_react.append(val)\n",
    "\n",
    "        val = False\n",
    "        for x in hardware:\n",
    "            if (x.upper() in row['Job Description'].upper()) | (x.upper() in row['Job Name'].upper()):\n",
    "                val = val | True\n",
    "            else:\n",
    "                val = val\n",
    "        c_hardware.append(val)\n",
    "\n",
    "        val = False\n",
    "        for x in scraping:\n",
    "            if (x.upper() in row['Job Description'].upper()) | (x.upper() in row['Job Name'].upper()):\n",
    "                val = val | True\n",
    "            else:\n",
    "                val = val\n",
    "        c_scraping.append(val)\n",
    "\n",
    "        val = False\n",
    "        for x in sports:\n",
    "            if (x.upper() in row['Job Description'].upper()) | (x.upper() in row['Job Name'].upper()):\n",
    "                val = val | True\n",
    "            else:\n",
    "                val = val\n",
    "        c_sports.append(val)\n",
    "\n",
    "        for x in banned_words:\n",
    "            if (x.upper() in row['Job Description'].upper()) | (x.upper() in row['Job Name'].upper()):\n",
    "                drops.append(index)\n",
    "\n",
    "        if (row['Payment Type'] == \"Fixed\") & (float(row['Budget']) < fixed_limit):\n",
    "            if index not in drops:\n",
    "                drops.append(index)\n",
    "\n",
    "        elif (row['Payment Type'] == \"Hourly\") & (float(row['Budget']) < hourly_limit):\n",
    "            if index not in drops:\n",
    "                drops.append(index)\n",
    "\n",
    "    df['React Native'] = c_react\n",
    "    df['Hardware'] = c_hardware\n",
    "    df['Scrape'] = c_scraping\n",
    "    df['Sports'] = c_sports\n",
    "    df.drop(drops, inplace=True)\n",
    "    df = df.reset_index(drop=True)\n",
    "    df = df.drop_duplicates(ignore_index=True)\n",
    "    df = df.rename(columns={'Time Created':'Hours Since Posting'})\n",
    "    df.sort_values(['React Native', 'Hardware','Scrape','Sports'], ascending=[False, False, False, False], inplace=True)\n",
    "    name = \"Jobs_cleaned.xlsx\"\n",
    "    try:\n",
    "        os.remove(name)\n",
    "        df.to_excel(name,\n",
    "                    columns=['Url','Job Name', 'Job Description', 'Payment Type', 'Budget', 'Workload', 'Level',\n",
    "                             'Est. Time', 'Hours Since Posting', 'React Native', 'Hardware', 'Scrape', 'Sports'])\n",
    "    except PermissionError:\n",
    "        name = \"Jobs_cleaned\"+str(random.randrange(100))+\".xlsx\"\n",
    "        df.to_excel(name,\n",
    "                        columns=['Url','Job Name', 'Job Description', 'Payment Type', 'Budget', 'Workload', 'Level',\n",
    "                                 'Est. Time', 'Hours Since Posting', 'React Native', 'Hardware', 'Scrape', 'Sports'])\n",
    "\n",
    "    except OSError:\n",
    "        df.to_excel(\"Jobs_cleaned.xlsx\",columns=['Url','Job Name', 'Job Description','Payment Type','Budget','Workload','Level',\n",
    "                                             'Est. Time','Hours Since Posting','React Native','Hardware','Scrape','Sports'])\n",
    "    return name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This is where to run the program\n",
    "\n",
    "## The number in scrape() represents how many pages to scrape. I recommend 20 pages to get between 2-3 days worth of posts. After Scraping, the excel file will be located in the temp directory the notebook is currently in."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Jobs_cleaned.xlsx'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = scrape(20)\n",
    "filter(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}