Light Github index generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Github Repositories Index Generator\n",
    "This notebook generates two ElasticSearch (ES) indexes with information about:\n",
    "* git (commits, files, lines added, lines removed, commit authors)\n",
    "* github (issues, pull requests, time to close in hours)\n",
    "\n",
    "Let's start by importing the `utils` python script, setting up the connection to the ES server and defining some variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import utils\n",
    "utils.logging.basicConfig(level=utils.logging.INFO)\n",
    "settings = utils.read_config_file('settings.yml')\n",
    "es = utils.establish_connection(settings['es_host'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's give a name to the indexes and create them in the ES server. Take care `utils.create_ES_index()` deletes any existing index with the same name before creating it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "index_name_git = 'github-git'\n",
    "utils.create_ES_index(es, index_name_git, utils.MAPPING_GITHUB_GIT)\n",
    "index_name_github_issues = 'github-issues'\n",
    "utils.create_ES_index(es, index_name_github_issues, utils.MAPPING_GITHUB_ISSUES)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's import needed backends from [Perceval](http://github.com/grimoirelab/perceval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from perceval.backends.core.git import Git\n",
    "from perceval.backends.core.github import GitHub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each repository in the settings file, get git related info and upload it to defined git ES index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for repo_url in settings['github-repo']:\n",
    "    \n",
    "    repo_owner = repo_url.split('/')[-2]\n",
    "    repo_name = repo_url.split('/')[-1]\n",
    "    repo_git_url = repo_url + '.git'\n",
    "    \n",
    "    git_repo = Git(uri=repo_git_url, gitpath='/tmp/'+repo_name)\n",
    "    \n",
    "    utils.logging.info('Parsing log from {}'.format(repo_name))\n",
    "    \n",
    "    items = []\n",
    "    bulk_size = 10000\n",
    "    \n",
    "    for commit in git_repo.fetch():\n",
    "        \n",
    "        contributor_name = commit['data']['Author'].split('<')[0][:-1]\n",
    "        contributor_email_domain = commit['data']['Author'].split('@')[-1][:-1]\n",
    "        \n",
    "        for file in commit['data']['files']:\n",
    "            if 'added' not in file.keys() or file['added'] == '-': \n",
    "                file['added'] = 0\n",
    "            if 'removed' not in file.keys() or file['removed'] == '-':\n",
    "                file['removed'] = 0\n",
    "            \n",
    "            summary = {\n",
    "                'date': commit['data']['AuthorDate'], \n",
    "                'commit_id': commit['data']['commit'],\n",
    "                'contributor_name': contributor_name, \n",
    "                'contributor_email_domain': contributor_email_domain,\n",
    "                'file': file['file'],\n",
    "                'lines_added': file['added'], \n",
    "                'lines_removed': file['removed'],\n",
    "                'github_owner': repo_owner, 'github_repository': repo_name\n",
    "            }\n",
    "            \n",
    "            items.append({'_index': index_name_git, '_type': 'item', '_source': summary})\n",
    "            \n",
    "            if len(items) > bulk_size:\n",
    "                utils.helpers.bulk(es, items)\n",
    "                items = []\n",
    "                utils.logging.info('{} items uploaded'.format(bulk_size))\n",
    "        \n",
    "    if len(items) != 0:\n",
    "        utils.helpers.bulk(es, items)\n",
    "        utils.logging.info('Remaining {} items uploaded'.format(len(items)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each repository in the settings file, get github issues related info and upload it to defined github issues ES index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import datetime as datetime\n",
    "\n",
    "for repo_url in settings['github-repo']:\n",
    "    \n",
    "    repo_owner = repo_url.split('/')[-2]\n",
    "    repo_name = repo_url.split('/')[-1]\n",
    "    repo_git_url = repo_url + '.git'\n",
    "    \n",
    "    github_repo = GitHub(owner=repo_owner, repository=repo_name, api_token=settings['github_token'])\n",
    "    \n",
    "    utils.logging.info('Parsing issues from {}'.format(repo_name))\n",
    "    \n",
    "    items = []\n",
    "    \n",
    "    for issue in github_repo.fetch():\n",
    "        created_at = issue['data']['created_at']\n",
    "        \n",
    "        #If the issue/pull-request is closed, we get the time to close it\n",
    "        if issue['data']['state'] == 'closed':\n",
    "            closed_at = issue['data']['closed_at']\n",
    "            creation_date = datetime.datetime.strptime(created_at, \"%Y-%m-%dT%H:%M:%SZ\")\n",
    "            closing_date = datetime.datetime.strptime(closed_at, \"%Y-%m-%dT%H:%M:%SZ\")\n",
    "            delta_time = (closing_date - creation_date).total_seconds()\n",
    "        else:\n",
    "            delta_time = None\n",
    "        \n",
    "        summary = {\n",
    "            'date': created_at, \n",
    "            'title': issue['data']['title'],\n",
    "            'state': issue['data']['state'],\n",
    "            'url': issue['data']['html_url'],\n",
    "            'comments': issue['data']['comments'],\n",
    "            'closed_at': issue['data']['closed_at'],\n",
    "            'time_to_solve': delta_time,\n",
    "            'github_owner': repo_owner,\n",
    "            'github_repository': repo_name\n",
    "        }\n",
    "        \n",
    "        \"\"\" If there is submitter name, we use it as contributor_name\n",
    "            If not, we use the github username as contributor_name\n",
    "        \"\"\"\n",
    "        if issue['data']['user_data']['name'] != None:\n",
    "            summary['contributor_name'] = issue['data']['user_data']['name']\n",
    "        else:\n",
    "            summary['contributor_name'] = issue['data']['user_data']['login']\n",
    "        \n",
    "        \"\"\" If there is someone assigned, we try to get the name as assignee_name\n",
    "            If there is no name, we use the github username\n",
    "        \"\"\"\n",
    "        try:\n",
    "            summary['assignee_name'] = issue['data']['assignee_data']['name']\n",
    "        except:\n",
    "            try:\n",
    "                summary['assignee_name'] = issue['data']['assignee']['login']\n",
    "            except:\n",
    "                summary['assignee_name'] = None\n",
    "                \n",
    "        # We check if the item is an issue or pull request\n",
    "        if 'pull_request' in issue['data'].keys():\n",
    "            summary['issue_type'] = 'pull-request'\n",
    "        else:\n",
    "            summary['issue_type'] = 'issue'\n",
    "        \n",
    "        items.append({'_index': index_name_github_issues, '_type': 'item', '_source': summary})\n",
    "            \n",
    "        if len(items) > bulk_size:\n",
    "            utils.helpers.bulk(es, items)\n",
    "            items = []\n",
    "            utils.logging.info('{} items uploaded'.format(bulk_size))\n",
    "    \n",
    "    if len(items) != 0:\n",
    "        utils.helpers.bulk(es, items)\n",
    "        utils.logging.info('Remaining {} items uploaded'.format(len(items)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}