clang-tags

#! /usr/bin/python -O

"""
C and C++ source code base indexer using libclang.
"""

import os
import sys
import json
import shlex, subprocess
import time
import re
import types
import tempfile

### Common constants
logPath    = ".ct.log"
socketPath = ".ct.sock"
pidPath    = ".ct.pid"

extensions = [".c", ".cxx", ".cc", ".C", ".cpp"]
compilers  = ["gcc", "g++", "c++", "clang", "clang++"]


### Helper functions
def endsWithOne (string, suffixes):
    "Return True iff STRING ends with one of SUFFIXES."
    for suffix in suffixes:
        if string.endswith(suffix):
            return True
    return False

def sendRequest (request, processOutput=sys.stdout.write):
    "Send a JSON request to the clang-tags daemon."
    #TODO: implement this in python rather than calling `socat`
    request = json.dumps (request)

    if os.getenv ("CLANG_TAGS_TEST") is None:
        cmd = ["socat", "-", "UNIX-CONNECT:%s" % socketPath]
    else:
        cmd = ["clang-tags-server", "--stdin"]

    process = subprocess.Popen (cmd,
                                stdin  = subprocess.PIPE,
                                stdout = subprocess.PIPE)
    process.stdin.write (request + "\n\n")

    while 1:
        # Use readline() to avoid buffering problems
        line = process.stdout.readline()
        if line == "":
            break
        processOutput (line)

    process.wait()
    return process.returncode



### Manage the server
def start (args):
    "Start the clang-tags server."

    if os.path.exists(socketPath):
        sys.stderr.write ("ERROR: socket already exists!\n")
        sys.exit (1)

    print "Starting server..."
    command = ["sh", "-c", "clang-tags-server --cachesize %d >%s 2>&1 &" %
        (args.cachesize, logPath)]
    sys.exit (subprocess.call (command))


def stop (args):
    "Stop the clang-tags server."

    request = {"command": "exit"}
    return sendRequest (request)


def clean (args):
    "Clean server leftovers after a non-nice exit."

    command = ["rm", socketPath, pidPath]
    sys.exit (subprocess.call (command))


def kill (args):
    "Kill the clang-tags server."

    f = open (pidPath, "r")
    pid = f.readline().strip()
    f.close()

    command = ["kill", pid]
    sys.exit (subprocess.call (command))


### Generate the compilation database

def getSourceFile (command):
    "Find the source file name within a compile command"
    for arg in command:
        if endsWithOne (arg, extensions) and os.path.exists (arg):
            return arg


def fakeCompiler (args):
    """Generate a compilation database by creating a fake compiler proxy."""
    tmpdir = tempfile.mkdtemp ()

    # Heuristic: `fake-compiler` can be found in the same directory
    #  as `clang-tags-server`
    which = subprocess.Popen (["which", "clang-tags-server"],
                              stdout = subprocess.PIPE)
    line = which.stdout.readline()
    fakeCompilersDir = "%s/fake-compiler" % os.path.dirname(line)
    for line in which.stdout:
        pass

    # Create a new environment in which compilers are faked
    os.environ["PATH"] = "%s:%s" % (fakeCompilersDir, os.environ["PATH"])
    return subprocess.call (args.command)


def trace (args):
    """Generate a compilation database by tracing a build command."""
    command  = ["strace", "-f", "-s256", "-etrace=execve,chdir", "-q"]
    command += args.command

    p = subprocess.Popen (command,
                          stderr = subprocess.PIPE)

    sourceFiles = {}
    cwd = os.getcwd()

    chdir_re = re.compile ("\[pid\s+\d+\] chdir\((.*)\) = 0")
    exec_re  = re.compile ("\[pid\s+\d+\] execve(\(.*\)) = ")

    for line in p.stderr:

        m = chdir_re.match (line)
        if m is not None:
            cwd = eval(m.group(1))
            continue

        m = exec_re.match (line)
        if m is not None:
            command = None
            sourceFile = None

            line = m.group(1)
            line = line.replace('"...', '"')
            line = re.sub(", \[/\* [^*]+ \*/\]", "", line)

            (programName, command) = eval(line)
            if endsWithOne(programName, [ "/" + c for c in compilers]):
                sourceFile = getSourceFile (command)
                if sourceFile is not None:
                    sourceFiles[sourceFile] = (cwd, command)
    p.wait()

    compilationDb = []
    for sourceFile in sourceFiles:
        (directory, command) = sourceFiles[sourceFile]
        compilationDb.append ({"file": os.path.realpath(sourceFile),
                               "directory": directory,
                               "command": subprocess.list2cmdline(command)})
    f = open ("compile_commands.json", "w")
    json.dump(compilationDb, f,
              indent=4)
    f.close()

    return p.returncode


def scan (args):
    """Generate a compilation database by scanning a sources directory."""
    srcdir = args.srcdir
    clangArgs = args.clangArgs
    compiler = args.compiler

    compilationDb = []
    for (root, _, files) in os.walk (srcdir):
        for fileName in files:
            if endsWithOne (fileName, [".c", ".cxx", ".cc", ".C"]):
                sourceFile = os.path.realpath (os.path.join (root, fileName))
                command  = "%s %s" % (compiler, sourceFile)
                command += " " + subprocess.list2cmdline(clangArgs)
                compilationDb.append({"file": sourceFile,
                                      "directory": ".",
                                      "command": command})
    f = open ("compile_commands.json", "w")
    json.dump(compilationDb, f,
              indent=4)
    f.close()


### Manage the source files index
def add (args):
    """Load a single compilation command in the server"""
    if args.command[0] == '--':
        args.command = args.command[1:]

    sourceFile = getSourceFile (args.command)
    if sourceFile is None:
        return 1

    database = [{"directory": os.getcwd(),
                 "command": subprocess.list2cmdline(args.command),
                 "file": os.path.realpath(sourceFile)}]

    (fd, tmpFile) = tempfile.mkstemp()

    f = open (tmpFile, "w")
    json.dump (database, f, indent=4)
    f.close()
    request = {"command": "load",
               "database": tmpFile}
    ret = sendRequest (request)
    os.remove (tmpFile)

    return ret


def load (args):
    """Read a compilation database."""

    request = {"command": "load",
               "database": args.compilationDB}
    ret = sendRequest (request)

    if args.emacs_conf is not None:
        f = open (os.path.join (args.emacs_conf, ".dir-locals.el"), "w")
        f.write (
            """;; WARNING: this file has been automatically generated by clang-args\n"""
            """;; any modification you do might be overwritten.\n"""
            """((c++-mode\n"""
            """  (eval . (clang-tags-mode 1))\n"""
            """  (ct/default-directory . "%s")))\n"""
            % os.getcwd())
        f.close()

    return ret


def index (args):
    """Index the source code base."""

    exclude = [os.path.realpath(d) for d in args.exclude]

    request = {"command": "index",
               "exclude": exclude}
    return sendRequest (request)


def update (args):
    """Update the source code base index."""

    request = {"command": "update"}
    return sendRequest (request)



### IDE-like features
def findDefinition (args):
    """Find the definition of an identifier."""
    fileName = os.path.realpath (args.fileName)

    request = {"command":   "find",
               "file":      fileName,
               "offset":    args.offset,
               "fromIndex": args.fromIndex}

    def processOutput (line):
        try:
            refDef = json.loads (line)
            refDef["ref"]["file"] = os.path.relpath (refDef["ref"]["file"])
            refDef["def"]["file"] = os.path.relpath (refDef["def"]["file"])
            refDef["def"]["col2"] -= 1

            sys.stdout.write ("-- %(substring)s -- %(kind)s %(spelling)s\n" % refDef["ref"])
            sys.stdout.write ("   %(file)s:%(line1)d-%(line2)d:%(col1)d-%(col2)d:"
                              " %(spelling)s\n"
                              % refDef["def"])
            sys.stdout.write ("   USR: %(usr)s\n" % refDef["def"])
            sys.stdout.write ("\n")
        except:
            sys.stdout.write (line)

    return sendRequest (request, processOutput)


def grep (args):
    """Find all references to a symbol."""

    request = {"command": "grep",
               "usr": args.usr}

    def processOutput (line):
        try:
            ref = json.loads (line)
            ref["file"] = os.path.relpath (ref["file"])

            sys.stdout.write ("%(file)s:%(line1)s:%(lineContents)s\n" % ref)
        except:
            sys.stdout.write (line)

    return sendRequest (request, processOutput)


def complete (args):
    """Automatic completion."""

    request = {"command": "complete",
               "file": os.path.realpath (args.fileName),
               "line": args.line,
               "column": args.column}
    return sendRequest (request)



### Process command-line arguments
def main_argparse ():
    "Entry point for the application"

    parser = argparse.ArgumentParser (
        description = "Index a source code base using libclang.")

    parser.add_argument (
        "--debug", "-d",
        action = 'store_true',
        help = "print debugging information")

    subparsers = parser.add_subparsers (metavar = "SUBCOMMAND")


    # Manage the server
    s = subparsers.add_parser (
        "start",
        help = "start the clang-tags server",
        description = "Start the clang-tags server.")
    s.add_argument (
        "--cachesize",
        metavar = "CACHESIZE",
        type = int,
        help = "Specify the maximum size of the translation unit cache (in MB)")
    s.set_defaults (cachesize = 1000000)
    s.set_defaults (fun = start)

    s = subparsers.add_parser (
        "stop",
        help = "shutdown the clang-tags server",
        description = "Shutdown the clang-tags server.")
    s.set_defaults (fun = stop)

    s = subparsers.add_parser (
        "kill",
        help = "kill the clang-tags server",
        description = "Kill the clang-tags server.")
    s.set_defaults (fun = kill)

    s = subparsers.add_parser (
        "clean",
        help = "clean up after a non-nice server shutdown",
        description = "Clean up after a non-nice server shutdown:"
        " pid file and server socket are wiped.")
    s.set_defaults (fun = clean)


    # Create the compilation database
    s = subparsers.add_parser (
        "trace",
        help = "trace a build command",
        description = "Create a compilation database by tracing a build command.")
    s.add_argument (
        "command",
        metavar = "COMMAND",
        nargs = argparse.REMAINDER,
        help = "build command line")
    s.set_defaults (sourceType = "trace")
    s.set_defaults (fun = trace)


    s = subparsers.add_parser (
        "scan",
        help = "scan a source directory",
        description = "Create a compilation database by scanning a source directory.")
    s.add_argument (
        "--compiler", "-c",
        default = "gcc",
        help = "compiler name (default: gcc)")
    s.add_argument (
        "srcdir",
        help = "top sources directory")
    s.add_argument (
        "clangArgs",
        metavar = "CLANG_ARGS",
        nargs = argparse.REMAINDER,
        help = "additional clang command-line arguments")
    s.set_defaults (sourceType = "scan")
    s.set_defaults (fun = scan)


    # Load compilation commands
    s = subparsers.add_parser (
        "load",
        help = "load a compilation database",
        description = "Read the compilation commands needed to build the"
        " project from a compilation database.")
    s.add_argument (
        "compilationDB",
        metavar = "FILEPATH",
        help = "path to the compilation database",
        nargs = "?",
        default = "compile_commands.json")
    s.add_argument(
        "--emacs-conf",
        metavar = "SRC_DIR",
        default = None,
        help = "generate an emacs configuration file in SRC_DIR")
    s.set_defaults (fun = load)


    s = subparsers.add_parser (
        "add",
        help = "manually load a compilation command",
        description = "Manually load a single compilation command into the server.")
    s.add_argument (
        "command",
        metavar = "COMMAND",
        nargs = argparse.REMAINDER,
        help = "compilation command line")
    s.set_defaults (fun = add)


    s = subparsers.add_parser (
        "fake-compiler",
        help = "fake compilers for a build command",
        description = "Load compilation commands by interposing fake proxy compilers"
        " during the build process.")
    s.add_argument (
        "command",
        metavar = "COMMAND",
        nargs = argparse.REMAINDER,
        help = "build command line")
    s.set_defaults (fun = fakeCompiler)


    # Manage the source files index
    s = subparsers.add_parser (
        "index",
        help = "index source code base",
        description = "Create an index of all tags in the source code base."
        " Source files and compilation commands are taken from a clang"
        " \"compilation database\" in JSON format, previously read using"
        " the \"load\" subcommand.")
    s.add_argument (
        "--exclude", "-e",
        metavar = "DIR",
        action = "append",
        help = "do not index files under DIR")
    s.add_argument (
        "--exclude-clear", "-E",
        dest = "exclude",
        action = "store_const", const = [],
        help = "reset exclude list")
    s.set_defaults (exclude = ["/usr"])
    s.set_defaults (fun = index)


    s = subparsers.add_parser (
        "update",
        help = "update index",
        description = "Update the source code base index, using the same"
        " arguments as previous call to `index'")
    s.set_defaults (fun = update)


    # IDE-like features
    s = subparsers.add_parser (
        "find-def",
        help = "find the definition of an identifier in a source file",
        description = "Find the definition location of an identifier in a"
        " source file.")
    s.add_argument (
        "fileName",
        metavar = "FILE_NAME",
        help = "source file name")
    s.add_argument (
        "offset",
        metavar = "OFFSET",
        help = "offset in bytes")
    s.add_argument (
        "--index", "-i",
        dest = "fromIndex",
        action = "store_true",
        help = "look for the definition in the index")
    s.add_argument (
        "--recompile", "-r",
        dest = "fromIndex",
        action = "store_false",
        help = "recompile the file to find the definition")
    s.set_defaults (fromIndex = True)
    s.set_defaults (fun = findDefinition)


    s = subparsers.add_parser (
        "grep",
        help = "find all uses of a definition",
        description = "Find all uses of a definition, identified by its USR"
        " (Unified Symbol Resolution). Outputs results in a grep-like format.")
    s.add_argument (
        "usr",
        metavar = "USR",
        help = "USR for the definition")
    s.set_defaults (fun = grep)


    s = subparsers.add_parser (
        "complete",
        help = "find completions at point",
        description =
        "Find possible code completions at a given position in the source code.")
    s.add_argument (
        "fileName",
        metavar = "FILE_NAME",
        help = "source file name")
    s.add_argument (
        "line",
        metavar = "LINE",
        help = "Line number")
    s.add_argument (
        "column",
        metavar = "COLUMN",
        help = "Column number")
    s.set_defaults (fun = complete)


    args = parser.parse_args ()
    return args.fun (args)



if __name__ == "__main__":
    try:
        import argparse
    except ImportError:
        import arg2opt as argparse
    sys.exit (main_argparse())