Light Git index generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Git Index Generator\n",
    "\n",
    "This notebook generates a ElasticSearch (ES) index with information about git (commits, files, lines added, lines removed, commit authors) for a given list of git repositories defined in a `settings.yml` file.\n",
    "\n",
    "Let's start by importing the utils python script, setting up the connection to the ES server and defining some variables\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import utils\n",
    "utils.logging.basicConfig(level=utils.logging.INFO)\n",
    "\"\"\" You can comment previous line if you don't want logging information\n",
    "\"\"\"\n",
    "\n",
    "settings = utils.read_config_file('settings.yml')\n",
    "es = utils.establish_connection(settings['es_host'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "Let's define an ES index mapping for the data that will be uploaded to the ES server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAPPING_GIT = {\n",
    "    \"mappings\": {\n",
    "        \"item\": {\n",
    "            \"properties\": {\n",
    "                \"date\": {\n",
    "                    \"type\": \"date\",\n",
    "                    \"format\" : \"E MMM d HH:mm:ss yyyy Z\",\n",
    "                    \"locale\" : \"US\"\n",
    "                },\n",
    "                \"commit\": {\"type\": \"keyword\"},\n",
    "                \"author\": {\"type\": \"keyword\"},\n",
    "                \"domain\": {\"type\": \"keyword\"},\n",
    "                \"file\": {\"type\": \"keyword\"},\n",
    "                \"added\": {\"type\": \"integer\"},\n",
    "                \"removed\": {\"type\": \"integer\"},\n",
    "                \"repository\": {\"type\": \"keyword\"}\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's give a name to the index to be created, and create it.\n",
    "\n",
    "**Note**: `utils.create_ES_index()` removes any existing index with the given name before creating it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "index_name = 'git'\n",
    "utils.create_ES_index(es, index_name, MAPPING_GIT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's import the git backend from [Perceval](http://github.com/grimoirelab/perceval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from perceval.backends.core.git import Git"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For each repository in the settings file, let's get its data, create a `summary` object with the desired information and upload data to the ES server using ES `bulk` API."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for repo_url in settings['git']:\n",
    "    \n",
    "    repo_name = repo_url.split('/')[-1]\n",
    "    repo = Git(uri=repo_url, gitpath='/tmp/'+repo_name)\n",
    "    \n",
    "    utils.logging.info('Go for {}'.format(repo_name))\n",
    "    \n",
    "    items = []\n",
    "    bulk_size = 10000\n",
    "    \n",
    "    for commit in repo.fetch():\n",
    "        \n",
    "        author_name = commit['data']['Author'].split('<')[0][:-1]\n",
    "        author_domain = commit['data']['Author'].split('@')[-1][:-1]\n",
    "        \n",
    "        for file in commit['data']['files']:\n",
    "            if 'added' not in file.keys() or file['added'] == '-':\n",
    "                file['added'] = 0\n",
    "            if 'removed' not in file.keys() or file['removed'] == '-':\n",
    "                file['removed'] = 0\n",
    "\n",
    "            summary = {\n",
    "                'date': commit['data']['AuthorDate'],\n",
    "                'commit': commit['data']['commit'],\n",
    "                'author': author_name,\n",
    "                'domain': author_domain,\n",
    "                'file': file['file'],\n",
    "                'added': file['added'],\n",
    "                'removed': file['removed'],\n",
    "                'repository': repo_name\n",
    "            }\n",
    "            \n",
    "            items.append({'_index': index_name, '_type': 'item', '_source': summary})\n",
    "            \n",
    "            if len(items) > bulk_size:\n",
    "                utils.helpers.bulk(es, items)\n",
    "                items = []\n",
    "                utils.logging.info('{} items uploaded'.format(bulk_size))\n",
    "            \n",
    "    if len(items) != 0:\n",
    "        utils.helpers.bulk(es, items)\n",
    "        utils.logging.info('Remaining {} items uploaded'.format(len(items)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}