git-rdm

#!/usr/bin/env python

# Git-RDM is released under the MIT license.

# The MIT License (MIT)

# Copyright (c) 2016 Christian T. Jacobs, Alexandros Avdis

# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:

# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import os, os.path, sys
import itertools
import argparse
import git
import logging
import sqlite3 as sqlite
import datetime
import glob
import json
import subprocess

_LOG = logging.getLogger(__name__)
_HANDLER = logging.StreamHandler()
_LOG.addHandler(_HANDLER)
_HANDLER.setFormatter(logging.Formatter('%(module)s %(levelname)s: %(message)s'))
del(_HANDLER)
_LOG.setLevel(logging.INFO)

try:
    from pyrdm.publisher import Publisher
except ImportError:
    _LOG.exception("Could not import the PyRDM library necessary for research data management.")
    sys.exit(1)

PUBLICATIONS_TABLE = "publications"

class GitRDM:

    """ The Git-RDM plugin for the Git version control system. """

    def __init__(self):
        """ Open the Git repository. """
        
        try:
            self.repo = git.Repo(".", search_parent_directories=True)
            _LOG.debug("The Git working directory is: %s" % self.repo.working_dir)
        except git.InvalidGitRepositoryError:
            _LOG.exception("Not in a Git version controlled repository.")
            sys.exit(1)

        return

    def initialise(self):
        """ Initialise the RDM control directory and set up the SQL database of published files (and files staged for publication). """
        
        # Create the RDM control directory if it doesn't already exist.
        rdm_control_directory = self.repo.working_dir + "/.rdm"
        if not os.path.exists(rdm_control_directory):
            os.makedirs(rdm_control_directory)
        
        # Set up the SQLite database.
        self.db_connect()
        
        # If this file already exists, then skip this step.
        if self.db_exists():
            response = raw_input("The publications database already exists. Do you want to overwrite it? (y/n)\n")
            if response == "y" or response == "Y":
                _LOG.info("Overwriting...")
                with self.connection:
                    c = self.connection.cursor()
                    query = "DROP TABLE %s" % PUBLICATIONS_TABLE
                    c.execute(query)
            elif response == "n" or response == "N":
                _LOG.info("Not overwriting.")
                return
            else:
                _LOG.error("Unknown response '%s'. Not overwriting." % response)
                return

        # Set up publication table columns.
        with self.connection:
            c = self.connection.cursor()
            query = "CREATE TABLE %s (id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT, date TEXT, time TEXT, sha TEXT, pid TEXT, doi TEXT)" % PUBLICATIONS_TABLE
            c.execute(query)

        # Disconnect.
        self.db_disconnect()

        return

    def add(self, paths):
        """ Add a desired file to the 'publication' staging area.
        
        :arg paths: A string, or list of strings, of absolute or relative paths to files to be added.
        """
        
        self.db_connect()

        # Expand and get the absolute paths
        expanded_paths = self.expand_paths(paths)
        _LOG.debug("Expanded paths: %s", str(expanded_paths))
        
        # Check that the file/files being added is/are actually under version control.
        skipped = []
        for f in expanded_paths:
            found = False
            for b in self.repo.tree().traverse():
                if b.abspath == f:
                    found = True
                    break
            if not found:
                _LOG.error("Could not add file '%s' for publishing because it is not under Git version control. Skipping..." % f)
                skipped.append(f)
                continue
        
        # Remove the skipped files
        expanded_paths = list(set(expanded_paths).difference(skipped))
        _LOG.debug("Expanded paths after skipped files removed: %s", str(expanded_paths))
        
        # Check that the file has not been added for publication already. If not, then add it to the database.
        for f in expanded_paths:
            with self.connection:
                query = "SELECT * FROM %s WHERE path=? AND doi IS NULL" % PUBLICATIONS_TABLE
                c = self.connection.cursor()
                c.execute(query, [f])
                result = c.fetchall()
                if len(result) > 0:
                    _LOG.warning("File '%s' has been added already. Skipping..." % os.path.abspath(f))
                else:
                    query = "INSERT INTO %s VALUES (NULL, ?, NULL, NULL, NULL, NULL, NULL)" % PUBLICATIONS_TABLE
                    c = self.connection.cursor()
                    c.execute(query, [f])
            
        self.db_disconnect()
        return

    def rm(self, paths):
        """ Remove a desired file from the 'publication' staging area.
        
        :arg paths: A string, or list of strings, of absolute or relative paths to files to be removed.
        """
        
        self.db_connect()

        # Expand and get the absolute paths
        expanded_paths = self.expand_paths(paths)
        
        query = "DELETE FROM %s WHERE path=? AND doi IS NULL" % PUBLICATIONS_TABLE
        with self.connection:
            c = self.connection.cursor()
            for f in expanded_paths:
                c.execute(query, [f])

        self.db_disconnect()
        
        return

    def ls(self, path=None, by_doi=True, raw=False):
        """ List all published files, and files to be published. If a file has multiple DOIs associated with it then they are all listed together.
        
        :arg path: The relative/absolute path to the file to be listed.
        :arg bool by_doi: List the files by DOI first (i.e. each DOI will be listed at the top level, with names of the files associated with that DOI underneath).
        :arg bool raw: Print out the entire SQL database table without any formatting.
        """

        self.db_connect()

        # Raw database dump of all the rows
        if raw:
            _LOG.info("Database dump:")

            query = "SELECT * FROM %s" % PUBLICATIONS_TABLE

            with self.connection:
                c = self.connection.cursor()
                c.execute(query)
                result = c.fetchall()
                if len(result) > 0:
                    # Column names
                    names = list(map(lambda x: x[0], c.description))
                    _LOG.info(", ".join(names))

                    # Rows
                    for r in result:
                        values = [str(r[name]) for name in names]
                        _LOG.info(", ".join(values))

            return

        # If a path is provided, then specify all the publications/DOIs associated with that path.
        if path:
            path = os.path.abspath(path)  # Get the absolute path
            query = "SELECT * FROM %s WHERE path=? AND doi IS NOT NULL" % PUBLICATIONS_TABLE
            with self.connection:
                c = self.connection.cursor()
                c.execute(query, [path])
                result = c.fetchall()
                if len(result) > 0:
                    _LOG.info(path)
                    for r in result:
                        _LOG.info("\t" + str(r["doi"]) + " (" + str(r["date"]) + " @ " + str(r["time"]).split(".")[0] + ", revision '%s')" % r["sha"])
            return

        # List all published files.
        if by_doi:
            query = "SELECT * FROM %s WHERE doi IS NOT NULL ORDER BY doi" % PUBLICATIONS_TABLE
        else:
            query = "SELECT * FROM %s WHERE doi IS NOT NULL ORDER BY path" % PUBLICATIONS_TABLE
            
        with self.connection:
            c = self.connection.cursor()
            c.execute(query)
            result = c.fetchall()
            
            if len(result) > 0:
                _LOG.info("Published files:")
                
                if by_doi:
                    for doi, publication_iter in itertools.groupby(result, key=lambda r: r[len(r)-1]):
                        publication_list = list(publication_iter)
                        _LOG.info("\t" + doi)
                        for p in publication_list:
                            _LOG.info("\t\t" + str(p[1]) + " (" + str(p[2]) + " @ " + str(p[3]).split(".")[0] + ", revision '%s')" % p[4])
                else:
                    for path, publication_iter in itertools.groupby(result, key=lambda r: r[1]):
                        publication_list = list(publication_iter)
                        _LOG.info("\t" + path)
                        for p in publication_list:
                            _LOG.info("\t\t" + str(p[len(p)-1]) + " (" + str(p[2]) + " @ " + str(p[3]).split(".")[0] + ", revision '%s')" % p[4])
                
        # List all files staged for publishing.
        query = "SELECT * FROM %s WHERE doi IS NULL" % PUBLICATIONS_TABLE
        with self.connection:
            c = self.connection.cursor()
            c.execute(query)
            result = c.fetchall()
            if len(result) > 0:
                _LOG.info("Files staged for publishing:")
                for r in result:
                    _LOG.info("\t"+r["path"])
                
        self.db_disconnect()
        
        return
    
    def show(self, service, pid):
        """ Show details of a particular publication.
        
        :arg str service: The service with which the repository is hosted.
        :arg pid: The publication ID.
        """
        
        publisher = Publisher(service=service)
        
        if service == "figshare":
            try:
                publication_details = publisher.figshare.get_article_details(article_id=pid)
            except:
                publication_details = publisher.figshare.get_article_details(article_id=pid, private=True)
        elif service == "zenodo":
            publication_details = publisher.zenodo.retrieve_deposition(deposition_id=int(pid))
        else:
            _LOG.error("Unknown service '%s'" % service)
            return
        
        # Pretty print
        try:
            _LOG.info(json.dumps(publication_details, sort_keys=True, indent=4, separators=(',', ': ')))
        except:
            _LOG.info(publication_details)
            
        return        
        
    def publish(self, service, pid=None):
        """ Publish the desired files.
        
        :arg str service: The repository hosting service with which to publish the files.
        :arg pid: An (optional) existing publication ID. If this is not None, then a new version of the repository will be created.
        """

        self.db_connect()

        # Find all files without a DOI (and assume these are in the publication staging area).
        with self.connection:
            query = "SELECT * FROM %s WHERE doi IS NULL" % PUBLICATIONS_TABLE
            c = self.connection.cursor()
            c.execute(query)
            to_publish = c.fetchall()

        if not to_publish:
            _LOG.warning("No files selected for publication.")
            return

        # Does the user needs to commit any modified files first?
        modified_files = subprocess.check_output(['git', 'diff', '--name-only']).split()
        for i in range(len(modified_files)):
            # Get the absolute path
            modified_files[i] = self.repo.working_dir + "/" + modified_files[i]
        _LOG.debug("Modified files: %s" % str(modified_files))
        
        # We only care if the uncommitted changes apply to files in the 'publishing staging area'.
        overlap = False
        for f in to_publish:
            if f["path"] in modified_files:
                overlap = True
        if self.repo.is_dirty() and overlap:
            _LOG.error("Uncomitted changes exist in the repository. Please commit these changes before trying to publish any files.")
            return
        
        # Get the minimal amount of metadata needed to publish from the user.
        response = raw_input("Private publication? (y/n): ")
        if response == "y" or response == "Y":
            _LOG.info("Publishing as a private repository...")
            private = True
        elif response == "n" or response == "N":
            _LOG.info("Publishing as a public repository...")
            private = False
        else:
            _LOG.error("Unknown response '%s'. Not publishing." % response)
            return

        parameters = self.get_publication_parameters()
                
        # Publish to the repository hosting service.
        publisher = Publisher(service=service)
        pid, doi = publisher.publish_data(parameters, pid=pid, private=private)
        
        # Update the publications database by adding the DOIs and publication IDs to the previously-staged files.
        with self.connection:
            c = self.connection.cursor()
            query = "UPDATE %s SET doi=? WHERE doi IS NULL" % (PUBLICATIONS_TABLE)
            c.execute(query, [doi])
            query = "UPDATE %s SET pid=? WHERE pid IS NULL" % (PUBLICATIONS_TABLE)
            c.execute(query, [pid])
            query = "UPDATE %s SET date=? WHERE date IS NULL" % (PUBLICATIONS_TABLE)
            c.execute(query, [str(datetime.datetime.now().date())])
            query = "UPDATE %s SET time=? WHERE time IS NULL" % (PUBLICATIONS_TABLE)
            c.execute(query, [str(datetime.datetime.now().time())])
            query = "UPDATE %s SET sha=? WHERE sha IS NULL" % (PUBLICATIONS_TABLE)
            c.execute(query, [str(self.repo.head.object.hexsha)])
          
        self.db_disconnect()
        
        return

    def get_publication_parameters(self):
        """ Return the parameters required for publication from the user.
        
        :rtype: dict
        :returns: The parameters required for publication.
        """
    
        parameters = {}
        
        title = raw_input("Title: ")
        description = raw_input("Description: ")
        tags = raw_input("Tags/keywords (in list format [\"a\", \"b\", \"c\"]): ")
        if tags:
            tags = eval(tags)
        else:
            tags = []
        
        # Get the list of file paths.
        files = self.db_select_unpublished()
        
        parameters = {"title":title, "description":description, "category":"Uncategorized", "tag_name":tags, "files":files}
        
        return parameters
        
    def db_connect(self):
        """ Create a connection to the publications database. """
    
        _LOG.debug("Attempting to connect to publication database...")
        path = self.repo.working_dir + "/.rdm/publications.db"
        try:
            self.connection = sqlite.connect(path)
            self.connection.row_factory = sqlite.Row
            _LOG.debug("Connected successfully!")
        except sqlite.Error as e:
            _LOG.exception("Could not connect to the publication database. Check read permissions? Check that the .rdm directory exists? If it doesn't, run the 'git rdm init' command.")
            sys.exit(1)
        return

    def db_disconnect(self):
        """ Distroy the existing connection to the publications database. """
        
        if(self.connection):
            self.connection.close()
        return

    def db_search_by_path(self, path):
        """ Create a connection to the publications database. """
        
        try:
            with self.connection:
                c = self.connection.cursor()
                query = "SELECT * FROM %s WHERE path=?" % PUBLICATIONS_TABLE
                c.execute(query, [path])
                return c.fetchone()  # This path is a unique absolute path.
        except sqlite.Error as e:
            logging.exception(e)
            return None
    
    def db_exists(self):
        """ Return True if the publications table exists in the database, otherwise return False. """
        
        with self.connection:
            c = self.connection.cursor()
            c.execute("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE name=?)", [PUBLICATIONS_TABLE])
            exists = c.fetchone()
            if(exists[0] == 1):
                return True
            else:
                return False

    def db_select_all(self):
        """ Return all rows of the database.
        
        :rtype: list
        :returns: All rows of the database.
        """
        
        query = "SELECT * FROM %s" % PUBLICATIONS_TABLE
        with self.connection:
            c = self.connection.cursor()
            c.execute(query)
            return c.fetchall()

    def db_select_unpublished(self):
        """ Return all rows of the database in which the DOI field is NULL.
        
        :rtype: list
        :returns: All rows of the database in which the DOI field is NULL.
        """
        
        query = "SELECT * FROM %s WHERE doi IS NULL" % PUBLICATIONS_TABLE
        with self.connection:
            c = self.connection.cursor()
            c.execute(query)
            result = c.fetchall()
            
        paths = []
        for r in result:
            paths.append(str(r["path"]))
        return paths

    def expand_paths(self, paths):
        """ Expand and get the absolute paths.
        
        :arg paths: A string, or list of strings, of absolute or relative paths to files.
        :rtype: list
        :returns: A list of expanded paths.
        """
        
        expanded_paths = []
        if isinstance(paths, str):  # A single path
            expanded = glob.glob(paths)
            for e in expanded:
                expanded_paths.append(os.path.abspath(e))
        elif isinstance(paths, list):  # Multiple path
            for p in paths:
                expanded = glob.glob(p)
                for e in expanded:
                    expanded_paths.append(os.path.abspath(e))
        else:
            _LOG.exception("Unknown input for the 'add' function.")
        return expanded_paths
        
if(__name__ == "__main__"):
    # Command line arguments
    parser = argparse.ArgumentParser(prog="git-rdm")
    parser.add_argument("-d", "--debug", action="store_true", default=False, help="Enable debugging.")
    
    # Subparsers
    subparsers = parser.add_subparsers(help="The subcommand of 'git rdm'.", dest='subcommand')
    
    # 'git rdm init'
    init_parser = subparsers.add_parser("init", help="Initialise the .rdm control directory and publication database.")
    
    # 'git rdm add'
    add_parser = subparsers.add_parser("add", help="Add a file to the publishing staging area.")
    add_parser.add_argument("path", nargs='+', help="The path(s) to the file(s) to be added.", action="store", type=str)

    # 'git rdm rm'
    rm_parser = subparsers.add_parser("rm", help="Remove a file from the publishing staging area.")
    rm_parser.add_argument("path", nargs='+', help="The path(s) to the file(s) to be removed.", action="store", type=str)

    # 'git rdm ls'
    ls_parser = subparsers.add_parser("ls", help="List the published files, and files staged for publishing.")
    ls_parser.add_argument("--path", required=False, help="The path to the file to list.", action="store", type=str, default=None)
    ls_parser.add_argument("--by-doi", help="Group files by DOI.", action="store_true", default=False, dest="by_doi")
    ls_parser.add_argument("--raw", help="Print out the entire SQL database table without any formatting.", action="store_true", default=False, dest="raw")
    
    # 'git rdm publish'
    publish_parser = subparsers.add_parser("publish", help="Publish the files in the publishing staging area.")
    publish_parser.add_argument("service", help="The service with which to publish.", action="store", type=str)
    publish_parser.add_argument("--pid", required=False, help="The ID of an existing publication. This will result in a new version of the publication being created. The DOI will stay the same.", action="store", default=None, type=str)

    # 'git rdm show'
    show_parser = subparsers.add_parser("show", help="Show details about a particular publication.")
    show_parser.add_argument("service", help="The hosting service of the publication.", action="store", type=str)
    show_parser.add_argument("pid", help="The ID of the publication.", action="store", type=str)
    
    # Parse all arguments
    args = parser.parse_args()

    # Output debugging messages to a file
    if(args.debug):
        logger = logging.getLogger(__name__)
        logger.setLevel(logging.DEBUG)

    # Execute the desired subcommand
    rdm = GitRDM()
    if args.subcommand == "init":
        rdm.initialise()
    elif args.subcommand == "add":
        rdm.add(args.path)
    elif args.subcommand == "rm":
        rdm.rm(args.path)
    elif args.subcommand == "ls":
        rdm.ls(path=args.path, by_doi=args.by_doi, raw=args.raw)
    elif args.subcommand == "publish":
        rdm.publish(service=args.service, pid=args.pid)
    elif args.subcommand == "show":
        rdm.show(service=args.service, pid=args.pid)
    else:
        _LOG.error("Unknown git-rdm subcommand '%s'" % args.subcommand)