sprint-3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import psycopg2\n",
    "import pandas as pd \n",
    "import sqlalchemy as salc\n",
    "import numpy as np\n",
    "import warnings\n",
    "import datetime\n",
    "import json\n",
    "from credentials import * \n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "with open(\"config.json\") as config_file:\n",
    "    config = json.load(config_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Database Connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "database_connection_string = 'postgres+psycopg2://{}:{}@{}:{}/{}'.format(config['user'], config['password'], config['host'], config['port'], config['database'])\n",
    "\n",
    "dbschema='augur_data'\n",
    "engine = salc.create_engine(\n",
    "    database_connection_string,\n",
    "    connect_args={'options': '-csearch_path={}'.format(dbschema)})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retreiving repos given GitHub ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "GitHub ID dpose\n"
     ]
    }
   ],
   "source": [
    "github_id = input(\"GitHub ID\")\n",
    "\n",
    "QUERY = \"\"\"\n",
    "select distinct repo_id from \n",
    "(\n",
    "select contributors.cntrb_id, pull_requests.pr_augur_contributor_id, pull_requests.repo_id \n",
    "from contributors, pull_requests\n",
    "where contributors.cntrb_id=pull_requests.pr_augur_contributor_id\n",
    "and gh_login='%s') a \n",
    "\n",
    "\"\"\" % github_id\n",
    "\n",
    "repo_id = pd.read_sql(QUERY, con=engine)\n",
    "repo_list = repo_id['repo_id'].tolist()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['JavaScript', 'TypeScript']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if len(repo_list) == 0:\n",
    "    display(\"There is no repo_id for the provided GitHub User\")\n",
    "else:\n",
    "    placeholders= ', '.join(['%s']*len(repo_list))  # \"%s, %s, %s, ... %s\"\n",
    "    SQL = \"\"\"\n",
    "    SELECT\n",
    "        programming_language,\n",
    "        COUNT ( * ) AS language_files \n",
    "    FROM\n",
    "        repo_labor \n",
    "    WHERE\n",
    "        repo_id IN ({}) \n",
    "    GROUP BY\n",
    "        repo_id,\n",
    "        programming_language \n",
    "    ORDER BY\n",
    "        language_files DESC;\n",
    "    \"\"\".format(placeholders)\n",
    "\n",
    "    SQL = SQL % tuple(repo_list)\n",
    "\n",
    "    language_list = pd.read_sql(SQL, con=engine, params=repo_list)\n",
    "    languages = language_list.head(2)['programming_language'].tolist()\n",
    "    display(languages)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# API HEADERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Go to https://rapidapi.com/lattice-data-lattice-data-default/api/job-search4 and register to get credentials for the API \n",
    "url = \"https://job-search4.p.rapidapi.com/simplyhired/search\"\n",
    "headers = {\n",
    "    'x-rapidapi-key': credentials['x-rapidapi-key'],\n",
    "    'x-rapidapi-host': credentials['x-rapidapi-host']\n",
    "}\n",
    "trend_dict = dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# API REQUEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The search returned 19 results\n"
     ]
    }
   ],
   "source": [
    "job_listing_search = languages[0] + \" Language\"\n",
    "\n",
    "if job_listing_search in trend_dict:\n",
    "    trend_dict[job_listing_search] += 1\n",
    "else:\n",
    "    trend_dict[job_listing_search] = 1\n",
    "\n",
    "querystring = {\"query\": job_listing_search, \"page\": \"1\"}\n",
    "\n",
    "response = requests.request(\"GET\", url, headers=headers, params=querystring)\n",
    "result = json.loads(response.text)\n",
    "\n",
    "\n",
    "job_listing_results = result['jobs']\n",
    "print(\"The search returned \" + str(len(job_listing_results))  + \" results\") \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Job Title: Onsite Web Developer with working SEO knowledge\n",
      "Company Name: The American Fence Company\n",
      "Location: Omaha, NE\n",
      "Description: Experience working with coding languages such as HTML, JQuery, JSON, and Ajax. You will be assisting with website builds, implementing new design features, and…\n",
      "Date posted: 2021-04-21T00:00:00\n",
      "url : https://www.simplyhired.com/job/78K7CAiJAmEKv35pHx5y75sxf5PMw5HHXUW9bC7GHO6S6p_lSdtG_g?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Baseball Systems Developer\n",
      "Company Name: Arizona Diamondbacks\n",
      "Location: Phoenix, AZ\n",
      "Description: Minimum of one year(s) of professional experience developing enterprise web applications in Java or another object-oriented language.\n",
      "Date posted: 2021-04-30T00:00:00\n",
      "url : https://www.simplyhired.com/job/Uol78wC15s-zgAP_oklXlMdITY-03PdqDS-3eChhTn2V5L5kET9UaQ?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Model Developer\n",
      "Company Name: Boston Region Metropolitan Planning Organization\n",
      "Location: Boston, MA\n",
      "Description: Demonstrated ability to rapidly learn new tools and technologies, such as new scripting languages and analytics platforms.\n",
      "Date posted: 2021-04-12T00:00:00\n",
      "url : https://www.simplyhired.com/job/JkHWzFsr1v6CPKVwfyL4Qb4WcP0Q8UbjD-3o_XTS8LvcLdEVWT1W9g?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Backend Engineer\n",
      "Company Name: Pie Insurance\n",
      "Location: Denver, CO\n",
      "Description: Deep knowledge of procedural programming languages such as C# or Java, network protocols, specifically HTTP, and JSON. Take-as-you-need PTO – Salaried.\n",
      "Date posted: 2020-09-23T00:00:00\n",
      "url : https://www.simplyhired.com/job/Is637tQlDP0zwk1qNY2SRqZDecWmrJBBSbgXXziFb1CrcFLZzci0sQ?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: QA Automation Engineer\n",
      "Company Name: ABOTTS\n",
      "Location: Saint Paul, MN\n",
      "Description: Job duties: Working on Automation testing with WebDriverIO similar to Protractor, NightWatch ( which sits on top of Selenium) Identify the elements on the UI…\n",
      "Date posted: 2020-11-26T00:00:00\n",
      "url : https://www.simplyhired.com/job/4t8US67s_ZbL0hCBIiIHC46s1vB_2zK5FBUAMH-2MuH_st3B3ZIH8Q?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Junior Frontend Engineer\n",
      "Company Name: CLIENT GIANT\n",
      "Location: Costa Mesa, CA\n",
      "Description: Comfortable working with web development tools and languages such as HTML / CSS / SASS and Javascript. The Junior Frontend Engineer is required to maintain and…\n",
      "Date posted: 2021-01-28T00:00:00\n",
      "url : https://www.simplyhired.com/job/EFG0usMa36FkVu8R8Z_IWvdeTJ3XqXzisifcms6uTOmq_JZOksXaIg?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Product Support Manager, Amazon Gift Cards\n",
      "Company Name: Amazon.com Services LLC\n",
      "Location: Seattle, WA\n",
      "Description: 5+ years of relevant professional experience as a Support Analyst, Technical Support, Operations Support or related role.\n",
      "Date posted: 2021-02-27T00:00:00\n",
      "url : https://www.simplyhired.com/job/VCXPj0yYpebSGRAB7Fqz_lRCIrnox-4KjqAIZVnVUrl1t_uQIC-AFg?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Software Developer\n",
      "Company Name: Leidos\n",
      "Location: Gaithersburg, MD\n",
      "Description: Should have programming skills in several of the following programming and markup languages: Python, Bash, C++/Rust, JSON/XML. Experience in RDBMS is a plus.\n",
      "Date posted: 2021-04-29T00:00:00\n",
      "url : https://www.simplyhired.com/job/1Q2jlj1m74C-Pmyc-_8qM9wQ4NJ3xXi5jtCMGdiSn6dAj1Y0o5VSjg?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: SOAR Developer\n",
      "Company Name: Leidos\n",
      "Location: Washington, DC\n",
      "Description: Department of Homeland Security (DHS), Security Operations Center (SOC) Support Services is a US Government program responsible to monitor, detect, analyze,…\n",
      "Date posted: 2021-04-15T00:00:00\n",
      "url : https://www.simplyhired.com/job/sCekpiPO_yO7762u9D9jQ9r8pkMD1E_sJlt4n3ZznikakCfgX_WbiA?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Core Development Program\n",
      "Company Name: InterSystems\n",
      "Location: Cambridge, MA\n",
      "Description: Proven prior coding experience in any language, including: Java, C++, C#, JavaScript, AngularJS, JSON, Python, or similar.\n",
      "Date posted: 2020-09-03T00:00:00\n",
      "url : https://www.simplyhired.com/job/oK__PHFpcw6ncYFrWiNZ22oQzvru8wvJyLusWTlHYRIIUiYFKHp2jQ?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n",
      "Job Title: Natural Language Processing Modeling Engineer\n",
      "Company Name: Interactions\n",
      "Location: Franklin, MA\n",
      "Description: Previous experience in machine learning and natural language processing and speech recognition. Masters Degree in computer science with specialization in data…\n",
      "Date posted: 2021-04-22T00:00:00\n",
      "url : https://www.simplyhired.com/job/uk27WCJtB5TVEpTLiDnghV720nU5mLoBxDbFBvAz_x1kWbQ3OeznWw?q=JSON+Language\n",
      "\n",
      "---------------------------------------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for each_job in job_listing_results:\n",
    "    print(\"Job Title: \"  + each_job[\"title\"])\n",
    "    print(\"Company Name: \"  + each_job[\"company_name\"])\n",
    "    print(\"Location: \"  + each_job[\"location\"])\n",
    "    print(\"Description: \"  + each_job[\"description\"])\n",
    "    print(\"Date posted: \"  + each_job[\"date_posted\"])\n",
    "    print(\"url : \"  + each_job[\"detail_url\"])\n",
    "    print('\\n---------------------------------------------------------------------------------------------------\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'JSON Language': 1}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trend_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}