git-results

#! /usr/bin/env python3

"""Created by Walt Woods, 2014"""

from __future__ import print_function

import argparse
import collections.abc
import datetime
import fnmatch
import inspect
import os
import pickle
import random
import re
import reprconf
import shlex
import shutil
import subprocess
import sys
# Note on tempfile - only used for configuration, not for any execution.  This
# is so that if any of the git-results-* scripts invoke network file system
# activity, those files are accessible to all nodes with access to the NFS.
import tempfile
import textwrap
import threading
import time
import traceback

# For pickle, we're using git_results.* for class names.  So, shenanigans
__oname__ = __name__
class __dummy(object):
    pass
sys.modules['git_results'] = inspect.getmodule(__dummy)
__name__ = 'git_results'

if sys.version_info[0] > 2:
    raw_input = input

ABORT_SUFFIX = '-abrt'
FAIL_SUFFIX = '-fail'
RUN_SUFFIX = '-run'
MANUAL_SUFFIX = '-manual-retry'
SUFFIXES = [ ABORT_SUFFIX, FAIL_SUFFIX, RUN_SUFFIX, MANUAL_SUFFIX, '' ]
# Overridden by tests to suppress raw_input
IS_TEST = (os.environ.get('GIT_RESULTS_TEST', '').strip() != '')
IS_TEST_FAIL_MANUAL = False

class HelpfulParser(argparse.ArgumentParser):
    """Prints help before an error.

    https://stackoverflow.com/a/4042861/160205
    """

    def error(self, message):
        self.print_help()
        sys.stderr.write('\n\nError: {}\n'.format(message))
        self.exit(2)



class IndexStates(object):
    ABORT = 'abrt'
    FAIL = 'fail'
    # GONE means BUILD FAILURE
    GONE = 'gone'
    OK = '  ok'
    MOVE = 'move'
    RUN = ' run'
    MANUAL = 'rtry'
for key, val in IndexStates.__dict__.items():
    if key[0] == '_':
        continue
    if len(val) != 4:
        raise ValueError("IndexState {} has len(val) != 4".format(key))


class FolderState(object):
    """An object that tracks which files exist at init and at another time,
    and can copy changes to another folder."""

    def __init__(self, dir, resultsDir, args):
        """If dir is None, then this is an in-place operation, and we should
        move (rather than copy) the results."""
        self._args = args
        self._dir = dir or os.path.abspath('.')
        self._resultsDirRel = os.path.relpath(resultsDir, self._dir)
        if not self._resultsDirRel.startswith(".."):
            while True:
                rdr = os.path.split(self._resultsDirRel)[0]
                if not rdr:
                    break
                self._resultsDirRel = rdr
        self._files = set()
        self._scan(self._dir)


    def moveResultsTo(self, dir, trimCommonPaths = False):
        """Copy changes from self._dir to dir.  Use most common path."""
        allChanges = set()
        self._scanChanges(self._dir, allChanges)

        prefix = self._dir
        if trimCommonPaths:
            prefix = None
            for c in allChanges:
                if prefix is None:
                    prefix = c
                else:
                    while not c.startswith(prefix):
                        prefix = os.path.dirname(prefix)

            if prefix is None:
                prefix = self._dir
            elif not os.path.isdir(prefix):
                # Only one file remains, the common path is its folder
                prefix = os.path.dirname(prefix)

        allErrors = []
        for c in allChanges:
            # Add 1 to len(prefix) so that we don't include the preceding slash,
            # which would make os.path.join treat it as an absolute
            target = os.path.join(dir, os.path.relpath(c, prefix))
            safeMake(os.path.dirname(target))
            nRetries = 4
            for _retry in range(nRetries):
                try:
                    os.rename(c, target)
                except OSError as e:
                    if e.errno == 16 and _retry != nRetries - 1:
                        # Device busy, will retry
                        pass
                    elif e.errno == 2:
                        # 2 - No such file; was temporary
                        break
                    else:
                        allErrors.append("{0}: {1}".format(c, ''.join(
                                traceback.format_exception_only(
                                    *sys.exc_info()[:2]))))
                        break
                else:
                    break

                # Will retry
                time.sleep(1.)

        if allErrors:
            raise Exception("Errors during file move:\n\n{0}".format(
                    '\n\n'.join(allErrors)))


    def forgetPath(self, path):
        """Forget a path and all subpaths."""
        f = set()
        for p in self._files:
            if p.startswith(path):
                f.add(p)
        for p in f:
            self._files.remove(p)


    def _isBanned(self, path):
        """Gets a folder; returns True if it is banned (is our results dir)"""
        if path == os.path.join(self._dir, self._resultsDirRel):
            return True
        return False


    def _isIgnored(self, path):
        """Gets a file; returns True if it should be ignored."""
        for ext in self._args.ignoreExt:
            if path.endswith('.' + ext):
                return True

        # Ignore logic
        relPath = os.path.relpath(path, self._dir)
        excluded = False
        for ext in self._args.ignore:
            negate = False
            if ext.startswith('!'):
                negate = True
                ext = ext[1:]

            if '/' in ext:
                if ext.startswith('/'):
                    ext = '^' + ext[1:]
                r = re.compile(ext
                        .replace('\\*', '\\\\<STAR>')
                        .replace('**', '.\\<STAR>')
                        .replace('*', '[^/]\\<STAR>')
                        .replace('\\<STAR>', '*')
                )
            else:
                r = re.compile(fnmatch.translate(ext))

            if r.search(relPath) is not None:
                excluded = not negate
        return excluded


    def _scan(self, path):
        if self._isBanned(path):
            return
        for f in os.listdir(path):
            ff = os.path.join(path, f)
            if os.path.isfile(ff) and not self._isIgnored(ff):
                self._files.add(ff)
            if os.path.isdir(ff) and not os.path.islink(ff):
                self._scan(ff)


    def _scanChanges(self, path, changeset):
        if self._isBanned(path):
            return
        for f in os.listdir(path):
            ff = os.path.join(path, f)
            if (os.path.isfile(ff) and ff not in self._files
                    and not self._isIgnored(ff)):
                changeset.add(ff)
            elif os.path.isdir(ff) and not os.path.islink(ff):
                self._scanChanges(ff, changeset)



def checked(cmd, nonZeroOk = False, shell = False):
    if not isinstance(cmd, (tuple, list)):
        raise ValueError("Must always be a list or tuple so that quotes "
                "aren't messed up")

    p = subprocess.Popen(cmd, stdout = subprocess.PIPE,
            stderr = subprocess.PIPE, shell = shell)
    stdout, stderr = p.communicate()
    r = p.poll()

    if r != 0:
        if (isinstance(nonZeroOk, (list, tuple)) and r in nonZeroOk
                or nonZeroOk == True):
            pass
        else:
            raise Exception("Command '{0}' failed with {1}".format(cmd, r))
    return stdout.decode('utf-8')


def ensureGitignore(repoBase, resultsRoot):
    output = checked([ "git", "status", "{0}/{1}/.gitignore".format(repoBase,
            resultsRoot), "--porcelain", "--ignored" ]).strip()
    if output and not output.startswith("!!"):
        with open(os.path.join(repoBase, '.gitignore'), 'a') as f:
            f.write("\n/{0}".format(resultsRoot))


def getPathForResumeKey(resumeKey, filename = None):
    base = os.path.expanduser("~/.gitresults")
    if resumeKey:
        base = os.path.join(base, resumeKey)
        if filename:
            base = os.path.join(base, filename)
    elif filename:
        raise Exception("filename but no resumeKey?")
    return base


def shellOpen(cmd):
    """Returns a subprocess.Popen object with stdout and stderr redirected to
    pipes, using the correct environment.

    This is done so that the supervisor has consistency with what the user
    runs.
    """
    osEnv = os.environ
    env = {}
    for k in [ 'HOME', 'LOGNAME', 'LANG' ]:
        if k in osEnv:
            env[k] = osEnv[k]

    # For python applications, ensure that output is streamed constantly.
    env['PYTHONUNBUFFERED'] = '1'

    return subprocess.Popen(cmd, stdout=subprocess.PIPE,
            stderr=subprocess.PIPE, shell=True, env=env)


def tee(infile, *files):
    """Thanks to http://stackoverflow.com/a/4985080/160205, tee lines from the
    given file to one or more other files.
    """
    def fanout():
        for line in iter(infile.readline, b''):
            try:
                assert isinstance(line, bytes)
                line = line.decode('utf-8')
            except:
                line = (">>>>>>> git-results begin\n"
                        "{}\n"
                        ">>>>>>> git-results end\n").format(
                        traceback.format_exc())

            for f in files:
                try:
                    f.write(line)
                except:
                    # We can use below to log the exception.  However, often
                    # this situation is caused by e.g. stderr being closed by
                    # the OS when launched via cron (because the cron process
                    # closes).  Unless evidence is found that the file versions
                    # of stdout and stderr are lacking text, it's probably best
                    # to silently accept a closed stream and move on.

                    # Can't do much about a closed stderr or stdout
                    if f is sys.stderr or f is sys.stdout:
                        continue

                    tb = (">>>>>>> git-results begin\n"
                            "{}\n"
                            "\nWhile writing to {}: {}"
                            ">>>>>>> git-results-end\n").format(
                            traceback.format_exc(), f, line)
                    with open(os.path.expanduser("~/git-results.log"), "a") as f2:
                        f2.write(tb)
        infile.close()
    t = threading.Thread(target = fanout)
    t.daemon = True
    t.start()
    return t


def touch(fname):
    """Thanks to http://stackoverflow.com/questions/1158076/implement-touch-using-python"""
    with open(fname, 'a'):
        os.utime(fname, None)


def index_splitTag(repoBase, commitTag):
    """Splits (and validates) a commit tag into corresponding INDEX file and
    the experiment index within the file (as a string)."""
    if commitTag.startswith('/'):
        raise ValueError("Should not be an absolute path! {}".format(commitTag))
    indexDir, exp = os.path.split(os.path.join(repoBase, commitTag))
    exp = exp.rsplit('-', 1)[0]
    try:
        int(exp)
    except ValueError:
        raise ValueError("Bad experiment number? {}".format(commitTag))

    return (os.path.join(repoBase, indexDir, 'INDEX'), exp)


def indexExpunge(repoBase, commitTag):
    """Marks the given commitTag as GONE, for e.g. build failures.  Never
    deletes INDEX, since that is where we save the commit message.  Which will
    be used for the next default.
    """
    # Update the index
    indexUpdate(repoBase, commitTag, IndexStates.GONE)


class NotInIndexError(Exception):
    pass


def indexRead(repoBase, commitTag):
    """Returns (index, state, message) for given commitTag's entry in INDEX
    file.  Raises NotInIndexError if commitTag not found in INDEX."""
    indexFile, exp = index_splitTag(repoBase, commitTag)
    if not os.path.lexists(indexFile):
        raise NotInIndexError("Tag {} not found in INDEX (no INDEX)".format(
                commitTag))

    with open(indexFile, 'r') as f:
        contents = f.read()

    mState = None
    mStart = None
    mEnd = None
    for m in re.finditer(r"^(\d+) \((....)\) - ", contents, re.MULTILINE):
        if m.group(1) == exp:
            mState = m.group(2)
            mStart = m.end()
        elif mStart is not None:
            mEnd = m.start()
            break

    if mStart is None:
        raise NotInIndexError("Tag {} not found in INDEX".format(commitTag))

    if mEnd is None:
        mEnd = len(contents)

    return (exp, mState, contents[mStart:mEnd].strip())


def indexWrite(repoBase, commitTag, state, message):
    """Overwrites (or appends) the record for commitTag to the corresponding
    INDEX file."""
    indexFile, exp = index_splitTag(repoBase, commitTag)
    safeMake(os.path.dirname(indexFile))

    # Generate our record.
    header = "{} ({}) - ".format(exp, state)
    wrapper = textwrap.TextWrapper(width = 79, initial_indent = '',
            subsequent_indent = '  ', replace_whitespace=True,
            max_lines=3)
    record = wrapper.fill(header + message.strip()) + "\n"

    with open(indexFile, 'a+') as f:
        f.seek(0)
        contents = f.read()
        fStart = None
        fEnd = None
        for m in re.finditer(r"^(\d+) \(....\) - ", contents, re.MULTILINE):
            if m.group(1) == exp:
                fStart = m.start()
            elif fStart is not None:
                fEnd = m.start()
                break

        if fStart is None:
            # Append to end
            f.seek(len(contents))
            f.write(record)
        else:
            if fEnd is None:
                fEnd = len(contents)

            # Overwrite interval [fStart, fEnd)
            f.seek(0)
            f.truncate()
            f.write(''.join([ contents[:fStart], record, contents[fEnd:] ]))


def indexUpdate(repoBase, commitTag, state):
    """Given a commitTag from git (with or without suffix) and a new index
    state, update the (xxxx) portion of the INDEX file to reflect the given
    state.  Preserves the message.
    """
    indexFile, exp = index_splitTag(repoBase, commitTag)
    with open(indexFile, 'r+') as f:
        contents = f.read()
        lastMatch = None
        for m in re.finditer(r"^{} \(....\) - ".format(int(exp)),
                contents, re.MULTILINE):
            lastMatch = m
        if lastMatch is not None:
            f.seek(lastMatch.start())
            f.write("{} ({}) - ".format(int(exp), state))
        else:
            raise ValueError("{} not found in INDEX".format(commitTag))


def runExperiment(args, dir, workingDir, extraFiles, resultsDir, commitTag,
        trimCommonPaths):
    """Given a directory to initialize and run our experiment in 'dir', run it
    and put results in 'resultsDir'

    workingDir - Relative to dir, where to execute from.

    Populates args.isGoingToRetry if args.retry_until_stall is set.

    dir can be None for current dir (not a temporary directory!)
    """

    # If False, this is a retry that has already been built.
    shouldBuild = True
    # Folder state post-build; used to identify run results vs build
    # intermediates
    fs = None
    if args.retry_until_stall:
        # Set this attribute so that we have access to it.
        args.isGoingToRetry = False
        if os.path.lexists(getPathForResumeKey(args.retryKey, "build-state")):
            shouldBuild = False

        def writeRetryStats():
            """Write a bunch of stats from our closure"""
            open(getPathForResumeKey(args.retryKey, "build-state.new"),
                    'wb').write(pickle.dumps({
                        'buildTime': preRun - preBuild,
                        'runStart': preRun,
                        'fs': fs,
                        'progress': lastProgress,
                        'retry': retryIndex }))
            os.rename(getPathForResumeKey(args.retryKey, "build-state.new"),
                    getPathForResumeKey(args.retryKey, "build-state"))

    if dir is None:
        dirRelative = '.'
    else:
        dirRelative = dir
        dir = os.path.abspath(dir)
    resultsDirRelative = resultsDir
    resultsDir = os.path.abspath(resultsDir)
    odir = os.getcwd()
    if dir is not None:
        os.chdir(dir)
    # Output files, will be opened later
    output = None
    error = None
    try:
        # Timers
        preBuild = preRun = None
        # Keeps track of last known completed progress _at the start_ of an
        # execution.
        # Used to determine stalls (intransient errors).  Updated at end of
        # execution (on failure) if there was a change in progress since the
        # last update.
        lastProgress = -1e300
        # The current retry index.
        retryIndex = 0
        if shouldBuild:
            buildHeartbeatStop = False
            buildHeartbeat = None
            if args.retry_until_stall:
                def hbUpdate():
                    tsleep = max(1e-2, args.retry_delay / 3.0)
                    touch(getPathForResumeKey(args.retryKey, "heartbeat"))
                    while not buildHeartbeatStop:
                        time.sleep(tsleep)
                        touch(getPathForResumeKey(args.retryKey, "heartbeat"))
                buildHeartbeat = threading.Thread(target = hbUpdate)
                buildHeartbeat.daemon = True
                buildHeartbeat.start()

            # Build not previously completed, rebuild project
            if dir is not None:
                checked([ "git", "init" ])
                checked([ "git", "remote", "add", "origin",
                        "file://" + args.base ])
                checked([ "git", "fetch", "origin", commitTag ])
                checked([ "git", "reset", "--hard", "FETCH_HEAD" ])

            # Copy supplementary (extra) files over to our tree before build
            cfgLeaf = os.path.dirname(args.tag_root)
            for f in extraFiles:
                fFrm, fTo = f.split(':')
                shutil.copy2(os.path.join(args.cwd, fFrm),
                        os.path.join(dir, cfgLeaf, fTo))

            # At this point, we have the directory with the right git-results-*.
            os.chdir(workingDir or os.path.curdir)

            preBuild = time.time()

            if args.build:
                print("Building {0} in {1}".format(commitTag, dirRelative))
                # stderr redirection benefits nosetests, mainly.
                p = shellOpen(args.build)
                thread = tee(p.stdout, sys.stdout)
                thread2 = tee(p.stderr, sys.stderr)
                thread.join()
                thread2.join()
                r = p.wait()
                if r != 0:
                    print("== BUILD FAILED ==")
                    sys.exit(1)
            else:
                print("Skipping build, none specified for {0}".format(
                        commitTag))

            # Now that we've built it, snapshot the folder so we can copy whatever
            # changed as results.  We'll be a little smart and take off the
            # greatest common path too...
            changeRoot = os.path.join(dir or '.',
                    os.path.dirname(args.tag_root))
            fs = FolderState(changeRoot, resultsDir, args)
            for f in extraFiles:
                _, fTo = f.split(':')
                fs.forgetPath(os.path.join(os.path.join(dir, cfgLeaf, fTo)))

            # Actual run starts now..ish.  Yes, this doesn't include scanning
            # the folder state.  However, for a retry, we want to remember both
            # how long the build took and the cumulative runtime.
            preRun = time.time()

            if args.retry_until_stall:
                buildHeartbeatStop = True
                buildHeartbeat.join()

                # Save fs so that we know that we've already built
                writeRetryStats()
                touch(getPathForResumeKey(args.retryKey, "heartbeat"))
        else:
            # Previously built OK, use old values
            try:
                d = pickle.loads(open(getPathForResumeKey(args.retryKey,
                        "build-state"), 'rb').read(), encoding='bytes')
            except pickle.UnpicklingError:
                # Corrupt test; mark as failed, append to stderr!
                with open('{0}/stderr'.format(resultsDir), 'a') as f:
                    f.write("\n\ngit-results detected bad formatting for "
                            "pickle file build-state; copied to result "
                            "directory and marking experiment failed")
                    return 1, False, False
            fs = d['fs']
            preRun = time.time() - (os.path.getmtime(getPathForResumeKey(
                    args.retryKey, "heartbeat")) - d['runStart'])
            preBuild = preRun - d['buildTime']
            lastProgress = d['progress']
            retryIndex = d['retry']
            touch(getPathForResumeKey(args.retryKey, "heartbeat"))

            # Already have the directory with the right git-results-*.
            os.chdir(workingDir or os.path.curdir)


        hasProgress = args.retry_until_stall
        print("Running {0} in {1}".format(commitTag, dirRelative))
        print("=" * 79)
        print("=" * 79)

        threadsShouldDie = False
        threads = []

        if hasProgress:
            def checkProgress():
                """Returns the monotonically increasing, floating-point number
                that denotes progress.  If retry_until_stall is set, also
                updates the progress file associated with the retry."""
                p = shellOpen(args.progress)
                stdout, stderr = p.communicate()
                stdout = stdout.decode('utf-8')
                stderr = stderr.decode('utf-8')
                r = p.wait()
                try:
                    value = float(stdout.strip().split('\n')[-1])
                except ValueError:
                    raise ValueError(("Last non-blank line must be a "
                            + "floating-point number; was: {}").format(
                                stdout.strip().split('\n')[-1]))
                return value
        if args.retry_until_stall:
            # Update heartbeats
            def hbUpdate():
                tsleep = max(1e-2, args.retry_delay / 3.0)
                touch(getPathForResumeKey(args.retryKey, "heartbeat"))
                while not threadsShouldDie:
                    time.sleep(tsleep)
                    touch(getPathForResumeKey(args.retryKey, "heartbeat"))
            hbThread = threading.Thread(target = hbUpdate)
            hbThread.daemon = True
            hbThread.start()
            threads.append(hbThread)

        iothreads = []
        # Name of abort type, if applicable, or False for no abort type
        didAbort = False
        # Special handling for our runner script
        line_buffered = 1
        output = open('{0}/stdout'.format(resultsDir), 'a', line_buffered)
        error = open('{0}/stderr'.format(resultsDir), 'a', line_buffered)
        if not args.internal_retry_abort:
            # Set up exit precautions...
            def inner_check():
                """Function responsible for handling KeyboardInterrupt
                behavior.  Recurses as much as needed.
                """
                nonlocal r, didAbort
                try:
                    [t.join() for t in iothreads]
                    r = p.wait()
                    didAbort = "abort"
                except KeyboardInterrupt:
                    inner_check()

            # For whatever reason, KeyboardInterrupt() may manifest multiple
            # times.  Make sure that doesn't happen.
            try:
                try:
                    p = shellOpen(args.run)
                    iothreads = [ tee(p.stdout, output, sys.stdout),
                            tee(p.stderr, error, sys.stderr) ]
                    [ t.join() for t in iothreads ]
                    r = p.wait()
                except KeyboardInterrupt:
                    # Child got Ctrl+C,
                    print("== CTRL+C caught by git-results; child should exit as per "
                            "CTRL+C semantics")
                    inner_check()
                except:
                    traceback.print_exc()
                    inner_check()
            except KeyboardInterrupt:
                print("== CTRL+C caught by git-results; child should exit as per "
                        "CTRL+C semantics (2)")
                inner_check()
        else:
            r = 1
            didAbort = "abort"
        # output, error are both closed in outer finally

        allDone = time.time()
        print("=" * 79)
        print("=" * 79)

        # Any exceptions raised cause the whole experiment to be deleted.
        # So, catch them, redirect them to error, and
        wasMoveFailure = False
        try:
            if r != 0 or didAbort:
                error.write(("\n\ngit-results detected abnormal program "
                        "termination: {0}\n").format(r))
                print("Non-zero exit status after {0}: {1}".format(
                        allDone - preRun, r))

                if args.retry_until_stall and not didAbort:
                    # Chance of supervisor retry...
                    if not hasProgress:
                        raise Exception("No git-results-progress but "
                                "retry_until_stall?")
                    newProgress = checkProgress()
                    if lastProgress > newProgress:
                        raise Exception("Progress MUST be a non-decreasing "
                                + "function")
                    elif lastProgress == newProgress:
                        # Stall; note that the FIRST potentially transient
                        # failure will happen with lastProgress != newProgress,
                        # so if retryIndex was 0, this is technically the first
                        # retry
                        retryIndex += 1
                        if args.retry_minimum > retryIndex:
                            args.isGoingToRetry = True
                            writeRetryStats()
                        else:
                            # Cannot retry!  Switch to manual mode
                            args.isGoingToRetry = True
                            writeRetryStats()
                            didAbort = "manual"
                    else:
                        # Progress!
                        lastProgress = newProgress
                        retryIndex = 0
                        args.isGoingToRetry = True
                        writeRetryStats()
            else:
                print("OK after {0}".format(allDone - preRun))

            if not args.retry_until_stall or not args.isGoingToRetry:
                with open(os.path.join(resultsDir, 'git-results-message'),
                        'a') as f:
                    f.write("\n\n")
                    f.write("{0} after {1}s\n".format(
                            "OK" if r == 0 else "FAIL", allDone - preRun))
                    f.write("Build took {0}s\n".format(preRun - preBuild))

                # If we reach here, everything ran OK, so copy files
                print("Copying results to {0}".format(resultsDirRelative))
                try:
                    fs.moveResultsTo(resultsDir, trimCommonPaths)
                except:
                    wasMoveFailure = True
                    raise
        except:
            # Only thing we change is that our result is no longer zero.
            tbErr = traceback.format_exc()
            sys.stderr.write(tbErr)
            error.write("\n\ngit-results error: {0}\n".format(tbErr))

            if r == 0 and not wasMoveFailure:
                # It's over 9000
                r = 9001

        # Terminate e.g. heartbeat thread
        threadsShouldDie = True
        [ t.join() for t in threads ]

        return r, didAbort, wasMoveFailure
    finally:
        if output:
            output.close()
        if error:
            error.close()
        os.chdir(odir)


def safeMake(path):
    try:
        os.makedirs(path)
    except OSError as e:
        # exists
        if e.errno != 17:
            raise


def safeRemove(f):
    """Removes a single file, safely"""
    try:
        os.remove(f)
    except OSError as e:
        # No such file or directory
        if e.errno != 2:
            raise


def safeRemoveDir(d):
    try:
        shutil.rmtree(d)
    except OSError as e:
        # No such file or directory
        if e.errno != 2:
            raise


def safeRollback(d):
    """Delete d and all parent directories (up to most common directory with
    current directory)
    """
    try:
        os.removedirs(d)
    except OSError as e:
        # Directory not empty OR no such file or directory (66 is mac os x)
        if e.errno != 39 and e.errno != 2 and e.errno != 66:
            raise


def setupExperiment(args, repoBase, resultsRoot, resultsLeaf, message):
    """Sets up the experiment skeleton and commits the git repo to an acceptable
    state.

    resultsRoot is the directory containing results.

    resultsLeaf is the rest of the experiment path, within resultsRoot.

    If message is unspecified, will prompt for a suitable message.

    Returns the directory for results (absolute path), dated symlink path (for
    rollback / updated), latest symlink path, and the commit tag to run.
    """
    resultsDir = os.path.abspath(os.path.join(repoBase, resultsRoot))
    experimentDir = os.path.join(resultsDir, resultsLeaf)
    if not os.path.lexists(os.path.join(resultsDir, ".gitignore")):
        if not IS_TEST:
            r = raw_input("To do this, git-results needs to make a new "
                    "results directory:\n\n{0}\n\nin repository {1}.\nIt will "
                    "also be added to the repository's .gitignore.  Is "
                    "this ok? (y/n) ".format(resultsRoot, repoBase))
            r = r.strip().lower()
            if r != 'y':
                raise SystemExit("y not entered")
        safeMake(resultsDir)
        # If we don't make a file, it looks like results is always ignored even
        # though it isn't.  So, make a sensible .gitignore to start if people
        # want to revision their results.
        with open("{0}/.gitignore".format(resultsDir), 'w') as f:
            f.write("/.tmp\n")

    # Now that it's made, make sure it's ignored
    ensureGitignore(repoBase, resultsRoot)

    # Allow / encourage running same tag several times.  We'll use /1, /2, etc
    n = 1
    if os.path.lexists(experimentDir):
        # Folder containing results under this tag exists.  Check both extant
        # folders and INDEX file to get next index.
        for run in os.listdir(experimentDir):
            # Remove any suffixes
            run = run.split('-', 1)[0]
            try:
                n = max(n, int(run) + 1)
            except ValueError:
                # Not an integer, probably a file someone bundled in with the
                # results.
                pass

        indexFile = os.path.join(experimentDir, 'INDEX')
        if os.path.lexists(indexFile):
            for nid in re.finditer(r'^(\d+) \((....)\)',
                    open(indexFile).read(), re.M):
                # GONE experiments are no longer registered, so their numbers
                # can be reused.  Trade-off is that the record will be
                # overwritten.
                if nid.group(2) != IndexStates.GONE:
                    n = max(n, int(nid.group(1)) + 1)

    # the tag for this instance of the experiment
    tag = "{0}/{1}/{2}".format(resultsRoot, resultsLeaf, n)
    tagDir = os.path.join(experimentDir, "{0}".format(n))

    ## It's too easy to include files that shouldn't be included, like images
    # or temporary files, without seeing git status.  Therefore, check if we
    # have local changes.  If we do, inquire about that commit first.

    cleanMessage = '' if not message else message
    if not cleanMessage:
        try:
            _tag, _state, oldMessage = indexRead(repoBase, tag)
        except NotInIndexError:
            pass
        else:
            cleanMessage = oldMessage.strip()
    if checked([ "git", "status", "--porcelain" ]).strip():
        # Need to make a commit.  Open up our message in git-results
        branch = "git-results"
        # Include the path to git-results.cfg as an annotation for the branch
        pathToCur = os.path.relpath(os.path.dirname(resultsDir), repoBase)
        if pathToCur != os.path.curdir:
            branch += "-{}".format(pathToCur)

        # If this fails, the branch didn't exist or we're already on it, both
        # of which are fine
        checked([ "git", "checkout", "-B", branch ])
        checked([ "git", "add", "-A", repoBase ])
        # Note that we run this outside of checked(), since we must share all
        # input/output
        cmd = [ "git", "commit", "-m", cleanMessage ]
        if not IS_TEST:
            # -e means edit the message in the git editor.
            cmd.append("-e")
            p = subprocess.Popen(cmd)
            r = p.wait()
        else:
            # Bypass user-input sections for testing.
            for s in ['echo', 'cat']:
                if os.environ.get('EDITOR', '').startswith(s):
                    cmd.append("-e")
                    break
            print(os.environ.get('EDITOR'))
            checked(cmd)
            print("DONE")
            r = 0
        if r != 0:
            raise Exception("Command '{0}' failed with {1}".format(cmd, r))

        # Read back commit message, use for git-results
        curCommit = checked([ "git", "rev-parse", "HEAD" ]).strip()
        cleanMessage = checked([ "git", "log", "--format=%B", "-n", "1",
                curCommit ]).strip()
    else:
        # No commit necessary.  Do we need a message anyway?
        # NOTE: deliberately use "message" here, not "cleanMessage".
        # cleanMessage was previously overwritten with the prior index state,
        # so it would not be empty, but we'd still want to prompt for changes.
        if not message:
            cleanMessage = _getStringViaEditor("git-results needs an "
                    "experiment message", default=cleanMessage)
            if len(cleanMessage) < 5 or cleanMessage.startswith("Please replace"):
                raise ValueError("Message must be at least 5 characters; "
                        "got: '" + cleanMessage + "'")

    curCommit = checked([ "git", "rev-parse", "HEAD" ]).strip()

    # Actually make our folder
    tagDirRun = tagDir + RUN_SUFFIX
    safeMake(tagDirRun)

    linkAs = None
    latestLinkAs = None
    try:
        # Ensure INDEX file
        indexWrite(repoBase, tag, IndexStates.RUN, cleanMessage)

        # Dated linkage
        now = datetime.datetime.now()
        dated = os.path.join(resultsDir, 'dated', now.strftime("%Y"),
                now.strftime("%m"))
        safeMake(dated)

        linkAs = os.path.join(dated, "{0}-{1}/{2}{3}".format(now.strftime("%d"),
                resultsLeaf, n, RUN_SUFFIX))
        safeMake(os.path.dirname(linkAs))
        os.symlink(os.path.relpath(tagDirRun, os.path.dirname(linkAs)), linkAs)

        latestLinkAs = os.path.join(resultsDir, 'latest', resultsLeaf + RUN_SUFFIX)
        safeMake(os.path.dirname(latestLinkAs))
        for s in SUFFIXES:
            linkPath = os.path.join(resultsDir, 'latest', resultsLeaf + s)
            if os.path.lexists(linkPath):
                os.unlink(linkPath)
        os.symlink(os.path.relpath(tagDirRun, os.path.dirname(latestLinkAs)),
                latestLinkAs)

        # Add our tag to the git repo
        checked([ "git", "tag", "-a", "-m", cleanMessage, tag ])

        # Write our message file
        with open(os.path.join(tagDirRun, "git-results-message"), "w") as f:
            f.write("{0}\n{1}\n".format(tag, "=" * min(79, len(tag))))
            if cleanMessage:
                f.write("{0}\n".format(cleanMessage))
            f.write("\nCommit: {0}".format(curCommit))
            f.write("\n\ngit-results\n-----------\n")
            f.write("run: {}\nbuild: {}\n".format(args.run, args.build))
            if args.progress:
                f.write("progress: {}\n".format(args.progress))
            f.write("\nStarted {0}".format(now.strftime("%Y-%m-%dT%H:%M:%S")))
    except:
        typ, err, tb = sys.exc_info()
        try:
            safeRemoveDir(tagDirRun)
            safeRollback(os.path.dirname(tagDirRun))
            if linkAs is not None:
                safeRemove(linkAs)
                safeRollback(os.path.dirname(linkAs))
            if latestLinkAs is not None:
                safeRemove(latestLinkAs)
                safeRollback(os.path.dirname(latestLinkAs))
            checked([ "git", "tag", "-d", tag ], nonZeroOk = [ 1 ])
            indexExpunge(repoBase, tag)
        except:
            traceback.print_exc()
        raise err

    return tagDirRun, linkAs, latestLinkAs, tag


def checkTag(s, allowExperimentInstance = None):
    """Takes a string s and returns it, if it is a valid tag.  Validity is
    assessed only AFTER the results root - that is, s should not be a full tag,
    but only the part after the root.

    allowExperimentInstances - If specified, must be list with one element.  Element
        will be set to True if tag specifies experiment, or False if it is
        a category.
    """
    if re.search("^[a-zA-Z0-9_/\.-]+$", s) is None:
        raise ValueError("Tag must match regex [a-zA-Z0-9_/\.-]+")
    if s.startswith('dated'):
        raise ValueError("Tag may not start with 'dated'")
    if s.startswith('latest'):
        raise ValueError("Tag may not start with 'latest'")
    if s.strip('/') != s:
        raise ValueError("Tag may not begin or end with '/'")
    isInt = False
    try:
        int(s.split('/')[-1])
        isInt = True
    except ValueError:
        pass

    if allowExperimentInstance is not None:
        allowExperimentInstance[0] = isInt
    elif isInt:
        raise ValueError("Tag may not end in a number")
    return s


def _getStringViaEditor(prompt = "", default = ""):
    """Gets a string by popping up an editor in the CLI; optionally presents
    a reason."""
    baseEditor = os.environ.get("VISUAL") or os.environ.get("EDITOR") or "vi"
    # Note that git config returns 1 if key not found
    gitEditor = checked([ "git", "config", "--get", "core.editor" ],
            nonZeroOk=[1]).strip()
    editor = gitEditor or baseEditor

    (fd, name) = tempfile.mkstemp(prefix="git-results-", suffix=".txt",
            text = True)
    try:
        f = os.fdopen(fd, "wt")
        f.write(default)
        prompt = prompt.split('\n')
        prompt = ['# ' + p for p in prompt]
        f.write("\n\n" + '\n'.join(prompt))
        f.close()

        r = os.system('{} "{}"'.format(editor, name))
        if r != 0:
            raise Exception('Editor failed: {} "{}"'.format(editor, name))

        lines = []
        with open(name) as f:
            for line in f:
                if line.strip().startswith('#'):
                    continue
                lines.append(line.rstrip())
        return '\n'.join(lines).strip()
    finally:
        os.unlink(name)


def _processTagArgs(args, *tagArgs, **kwargs):
    """Sort out an  arbitrary number of tags.  Sets attribute tag_root with
    the root results directory for the tag (e.g., the directory lower than
    the one containing git-results.cfg), and the tag attribute itself to the
    rest of the tag (the leaf).  The whole tag may be assembled as
    '/'.join(tag_root, tag).

    Also sets args.base with the abs path of the parent folder of current
    working directory containing .git, which is used for the base of each
    argument in tags.

    If there is exactly one tagArgs, then git-results.cfg will be parsed and
    loaded, populating more fields in args.

    Kwargs:
    allowExperimentInstances - If True, allow individual numbered experiments if ALL
            tag args are numbered.  Also sets "tagsAreInstances" on args object.
    """
    allowExperimentInstances = kwargs.pop('allowExperimentInstances', False)
    if kwargs:
        raise ValueError("Bad kwargs: {}".format(kwargs))

    # Find base
    args.base = os.getcwd()
    extraTagPath = []
    while True:
        if os.path.lexists(os.path.join(args.base, '.git')):
            break
        extraTagPath.insert(0, os.path.basename(args.base))
        nbase = os.path.dirname(args.base)
        if nbase == args.base:
            raise ValueError("git-results must be executed from a git "
                    "repository!")
        args.base = nbase

    # Sanitize tags, find roots
    tagIsExp = [ None ]
    if not allowExperimentInstances:
        tagIsExp = None
    lastWasExperiment = None
    for tagArg in tagArgs:
        tag = getattr(args, tagArg).rstrip('/')
        if tag.startswith('/'):
            raise ValueError("Tag may not be absolute path: {}".format(tag))
        elif tag.startswith('-'):
            raise ValueError("Path used by git-results cannot start with "
                    "'-': {}".format(tag))
        elif '//' in tag:
            raise ValueError("Tag may not have empty element (//): {}".format(
                    tag))

        tagRoot = '/'.join(extraTagPath + [ tag ])
        tagLeaf = []
        while tagRoot:
            if os.path.lexists(os.path.join(args.base, os.path.dirname(tagRoot),
                    'git-results.cfg')):
                break

            if '/' not in tagRoot:
                tagRoot = None
                break

            tagRoot, nextLeaf = tagRoot.rsplit('/', 1)
            tagLeaf.insert(0, nextLeaf)

        if not tagRoot:
            raise ValueError("git-results.cfg not found in tag path: {}".format(
                    tag))

        tagRootFull = os.path.join(args.base, tagRoot)
        if allowExperimentInstances and not os.path.lexists(tagRootFull):
            raise ValueError("Results folder '{}' not found".format(
                    tagRoot))

        if not tagLeaf:
            raise ValueError("Experiments must have at least two folder names, "
                    "e.g. /results/experiment1.  The first folder is used as a "
                    "results root.  Here, only the results root '{}' was "
                    "specified.".format(tag))

        checkTag('/'.join(tagLeaf), allowExperimentInstance = tagIsExp)
        if tagIsExp is not None:
            if lastWasExperiment is None:
                lastWasExperiment = tagIsExp[0]
            elif lastWasExperiment != tagIsExp[0]:
                raise ValueError("Non-homogenous tags.  Must either all be "
                        "instances of experiments, or all whole experiments.")
        setattr(args, tagArg, '/'.join(tagLeaf))
        setattr(args, '{}_root'.format(tagArg), tagRoot)

    # Flag all tags as either experiments (individual) or folders
    if tagIsExp is not None:
        args.tagsAreInstances = lastWasExperiment

    # Populate config entries if needed
    if len(tagArgs) == 1:
        _parseConfig(args, getattr(args, '{}_root'.format(tagArgs[0])),
                getattr(args, tagArgs[0]))


def _parseConfig(args, tagRoot, tagLeaf):
    """For the given tag (with results directory), populate args from the
    git-results.cfg at args.base.
    """
    tag = '{}/{}'.format(tagRoot, tagLeaf)
    tagMatch = '{}/{}'.format(os.path.basename(tagRoot), tagLeaf)
    cfg = reprconf.Config(os.path.join(args.base, os.path.dirname(tagRoot),
            'git-results.cfg'))

    parms = {
            'build': None,
            'ignore': [],
            'ignoreExt': [ "pyc", "pyo", "swp" ],
            'progress': None,
            'progressTries': 3,
            'progressDelay': 30,
            'run': None,
            'trim': False,
    }

    applicable = []
    for sec in cfg:
        if sec.startswith("/"):
            if (
                    # Root matches all
                    len(sec) == 1
                    # Otherwise want a prefix match at folder boundaries
                    or tagMatch == sec[1:]
                    or tagMatch.startswith(sec[1:] + '/')
                    or tagMatch.startswith(sec[1:] + '\\')
                    ):
                applicable.append(sec)
    if not applicable:
        raise SystemExit("At least one section in git-results.cfg must match "
                "the provided path: {0}".format(tagMatch))
    applicable.sort(key=lambda m: len(m))

    fmtKwargs = cfg.get('vars', {})
    if 'tag' in fmtKwargs:
        raise ValueError("'tag' is a reserved [vars] member.")
    fmtKwargs['tag'] = tag
    for sec in applicable:
        if "vars" in cfg[sec]:
            fmtKwargs.update(cfg[sec]["vars"])
        for k, v in cfg[sec].items():
            if k == "vars":
                # Already applied
                continue

            if k not in parms:
                raise ValueError("Unrecognized configuration key '{}'".format(
                        k))
            parms[k] = v

    # Sort formatting according to requirements so that each argument only
    # needs format called on it once.
    reqs = {}
    reqRe = re.compile(r"{([^}]*)}")
    for k, v in fmtKwargs.items():
        r = set([ m.group(1) for m in reqRe.finditer(v) ])
        reqs[k] = r

        if k in r:
            raise ValueError("Cannot self-reference: {} was '{}'"
                    .format(k, fmtKwargs[k]))
    # If we propagate requirement sets N times, we're guaranteed that all
    # of the requirements will be propagated
    for _ in range(len(reqs)):
        for k, r in reqs.items():
            for k2 in r.copy():
                r.update(reqs[k2])

    def fmtV(v):
        if isinstance(v, str):
            return v.format(**fmtKwargs)
        if isinstance(v, list):
            return [ fmtV(q) for q in v ]
        if isinstance(v, tuple):
            return ( fmtV(q) for q in v )
        if isinstance(v, dict):
            return { k: fmtV(q) for k, q in v.items() }
        return v

    # The size of each set is now the number of ancestors it has
    reqs = sorted([ (k, r) for k, r in reqs.items() ],
            key=lambda m: len(m[1]))
    reqsDone = set()
    for k, r in reqs:
        d = r.difference(reqsDone)
        if len(d) != 0:
            raise ValueError("'{}' seems cyclical on {}: {}".format(k, d,
                    fmtKwargs[k]))
        fmtKwargs[k] = fmtV(fmtKwargs[k])
        reqsDone.add(k)
    parms = fmtV(parms)

    # Finally apply those kwargs onto args
    args.build = parms['build']
    args.ignore = parms['ignore']
    if not isinstance(args.ignore, collections.abc.Iterable):
        raise ValueError("ignore must be iterable: {}".format(args.ignore))
    args.ignoreExt = parms['ignoreExt']
    if not isinstance(args.ignoreExt, collections.abc.Iterable):
        raise ValueError("ignoreExt must be iterable: {}".format(
                args.ignoreExt))
    args.progress = parms['progress']
    args.run = parms['run']
    args.trim = parms['trim']

    # Legacy names
    args.retry_until_stall = bool(parms['progress'])
    args.retry_minimum = parms['progressTries']
    args.retry_delay = parms['progressDelay']


def _auditMove(base, pathFrom, pathTo):
    """Audits that the given matchingTags can be moved from pathFrom to pathTo
    without collisions.

    Returns matchingTags - tagSrc, tagDst, dirSrc, dirDest, suffix.  Suffix
    includes hyphen.  dirSrc and dirDest are highest ancestor and may be
    redundant between several results.  tagSrc and tagDest are unique, and are
    the actual numbered runs.
    """
    gitTags = set([ t.strip()
            for t in checked([ "git", "tag", "-l", pathFrom,
                "{0}/*".format(pathFrom) ]).strip().split('\n')
            if t.strip() ])

    # Sometimes tags don't exist, but the results folders do.  This is post-bug
    # behavior, and git-results should handle it
    fsTags = _auditMoveFindFolderMatches(base, pathFrom)

    diff = gitTags.difference(fsTags)
    if diff:
        raise ValueError("Tags found in git but not on filesystem: {}".format(
                diff))
    diff = fsTags.difference(gitTags)
    if diff:
        # On filesystem, not in git
        gitTags.update(diff)
        print("!!! Tags found on filesystem but not in git: {}".format(diff))

    if not gitTags:
        raise ValueError("No result found under '{}'".format(pathFrom))

    matchingTags = []
    for src in gitTags:
        if not src.startswith(pathFrom):
            raise ValueError("Tag '{0}' doesn't start with '{1}'".format(src,
                    pathFrom))
        dest = pathTo + src[len(pathFrom):]
        # Check that the source folder exists, not just the tag in git
        suffix = None
        for s in SUFFIXES:
            if os.path.lexists(os.path.join(base, src + s)):
                if s == "-run":
                    sys.stderr.write(
                            "Cannot move/link tag that is running: {0}\n"
                            .format(src))
                    sys.exit(1)
                suffix = s
                break

        if suffix is None:
            sys.stderr.write("Cannot find data for {0}\n".format(src))
            sys.exit(1)
        matchingTags.append(( src, dest, os.path.join(base, pathFrom),
                os.path.join(base, pathTo), suffix ))

        # Check that the destination doesn't exist
        existingForms = []
        if checked([ "git", "tag", "-l", dest ]).strip():
            existingForms.append("tag")
        for s in SUFFIXES:
            if os.path.lexists(dest + s):
                existingForms.append("filesystem" + s)
        if existingForms:
            sys.stderr.write("Destination tag {0} exists ({1})\n".format(
                    dest, ', '.join(existingForms)))
            sys.exit(1)

    return matchingTags


def _auditMoveFindFolderMatches(base, pathFrom):
    """Part of _auditMove.  Find files that match the requested path, even if
    they are untagged.

    Experiments are assumed to be folders under pathFrom that have a
    git-results-message file.

    Return: set() of git tags that are indicated by the file system.
    """
    r = set()
    tagGlob = [ pathFrom ]
    while tagGlob:
        tag = tagGlob.pop(0)
        fsPath = os.path.join(base, tag)
        if not os.path.lexists(fsPath):
            continue

        pathList = os.listdir(fsPath)
        if 'git-results-message' not in pathList:
            for p in pathList:
                nTag = '{}/{}'.format(tag, p)
                fsTag = os.path.join(base, nTag)
                if not os.path.isdir(fsTag):
                    continue
                tagGlob.append(nTag)
            continue

        # Is an experiment... do suffix detection
        for s in SUFFIXES:
            if not s:
                continue
            if tag.endswith(s):
                tag = tag[:-len(s)]
        r.add(tag)
    return r


class LatestTracker(object):
    def __init__(self):
        """Keeps track of moved stuff, sorts out which should be the new latest
        link, does the update."""
        self._entries = {}


    def addTagDir(self, srcDir, tagDir, suffix):
        results, path = tagDir.split('/', 1)
        tagLatest, tagNumber = (results + '/latest/' + path).rsplit('/', 1)

        results, path = srcDir.split('/', 1)
        srcLatest, srcNumber = (results + '/latest/' + path).rsplit('/', 1)

        number = int(tagNumber.split('-')[0])
        if tagLatest in self._entries:
            if number > self._entries[tagLatest][0]:
                self._entries[tagLatest] = (number, suffix, tagDir, srcLatest)
        else:
            self._entries[tagLatest] = (number, suffix, tagDir, srcLatest)


    def commit(self, deleteOldLatest = False):
        """Make new latest symlinks"""
        # Update latest; each matchingTag's last part is the trial number, and
        # we should link the highest
        for path, (number, suffix, target, srcLatest
                ) in self._entries.items():
            safeMake(os.path.dirname(path + suffix))
            os.symlink(os.path.relpath(target, os.path.dirname(path)),
                    path + suffix)
            if deleteOldLatest:
                # srcLatest + suffix must be it, but it may not exist due to
                # e.g. old version of code.  Best be safe
                for s in SUFFIXES:
                    if os.path.lexists(srcLatest + s):
                        os.unlink(srcLatest + s)


def _runLink(args):
    ap = HelpfulParser(description = "Link one or more tagged "
            "results to a new location (which refers to the original).")
    ap.add_argument("tag_from")
    ap.add_argument("tag_to")
    args = ap.parse_args(args)
    _processTagArgs(args, "tag_from", "tag_to", allowExperimentInstances = True)
    pathFrom = '/'.join([ args.tag_from_root, args.tag_from ])
    pathTo = '/'.join([ args.tag_to_root, args.tag_to ])

    matchingTags = _auditMove(args.base, pathFrom, pathTo)

    # Do the links
    latestTracker = LatestTracker()
    for tagSrc, tagDest, _dirSrc, _dirDest, suffix in matchingTags:
        dirSrc = tagSrc + suffix
        dirDest = tagDest + suffix
        safeMake(os.path.dirname(dirDest))
        os.symlink(os.path.relpath(dirSrc, os.path.dirname(dirDest)), dirDest)
        tagMsg = checked([ "git", "tag", "-l", "-n1000", tagSrc ]).strip()[
                len(tagSrc):].strip()
        checked([ "git", "tag", "-a", "-m", tagMsg, tagDest, tagSrc ])
        latestTracker.addTagDir(dirSrc, dirDest, suffix)
    latestTracker.commit()


def _runMove(args):
    ap = HelpfulParser(description = "Move one or more tagged "
            "results to a different tag.")
    ap.add_argument("tag_from")
    ap.add_argument("tag_to")
    args = ap.parse_args(args)
    # Fills args.tagsAreInstances
    _processTagArgs(args, "tag_from", "tag_to",
            allowExperimentInstances = True)
    pathFrom = '/'.join([ args.tag_from_root, args.tag_from ])
    pathTo = '/'.join([ args.tag_to_root, args.tag_to ])

    matchingTags = _auditMove(args.base, pathFrom, pathTo)

    # { tag path : [ Y, M, D ] }
    tagDates = {}
    def walkDateDir(path, tag, dateInfo):
        if os.path.islink(path):
            # Strip off the fail / success / whatever status
            if '-' in tag.rsplit('/', 1)[1]:
                tag = tag.rsplit('-', 1)[0]
            tagDates[tag] = dateInfo
        elif os.path.isdir(path):
            for p in os.listdir(path):
                walkDateDir(os.path.join(path, p), '/'.join([ tag, p ]),
                        dateInfo)

    dateBase = os.path.join(args.base, args.tag_from_root, 'dated')
    for year in os.listdir(dateBase):
        yearBase = os.path.join(dateBase, year)
        for month in os.listdir(yearBase):
            monthBase = os.path.join(yearBase, month)
            for dayHyphen in os.listdir(monthBase):
                day, tagStart = dayHyphen.split('-', 1)
                walkDateDir(os.path.join(monthBase, dayHyphen),
                        '/'.join([ args.tag_from_root, tagStart ]),
                        [ year, month, day ])

    # Everything seems OK, rename it all.  Start with all of the filesystem
    # changes since we can run git results resync (once implemented) afterwards.
    latestTracker = LatestTracker()
    for tagSrc, tagDest, dirSrc, dirDest, suffix in matchingTags:
        if os.path.lexists(dirSrc):
            # This one hasn't been moved yet.  Note that if
            # args.tagsAreInstances is False, then this will ALSO move the
            # INDEX!
            safeMake(os.path.dirname(dirDest))
            os.rename(dirSrc, dirDest)
            safeRollback(os.path.dirname(dirSrc))
        elif not os.path.lexists(dirDest):
            raise ValueError("Neither source nor destination exists: {} -> {}"
                    .format(dirSrc, dirDest))

        # Does the source tag actually exist?
        if checked([ "git", "tag", "-l", tagSrc ]).strip():
            tagMsg = checked([ "git", "tag", "-l", "-n1000", tagSrc ]).strip()[
                    len(tagSrc):].strip()
            checked([ "git", "tag", "-a", "-m", tagMsg, tagDest, tagSrc ])
            checked([ "git", "tag", "-d", tagSrc ])
        elif checked([ "git", "tag", "-l", tagDest ]).strip():
            # destination tag exists, assume it is correct (this is essentially
            # a retry operation)
            pass
        else:
            # No previously existing tag...  try to infer commit from
            # git-results-message
            with open('{}/{}{}/git-results-message'.format(args.base, tagDest,
                    suffix)) as f:
                m = re.search(r"^Commit: ([a-zA-Z0-9]+)\n\ngit-results.*\n"
                        r"------------*\n",
                        f.read(),
                        re.M)
                if m is None:
                    raise ValueError("Could not get commit from filesystem "
                            "for {}".format(tagSrc))
                msg = checked([ "git", "log", "--format=%B", "-n", "1", m.group(1) ]).strip()
                checked([ "git", "tag", "-m", msg, tagDest, m.group(1) ])
        latestTracker.addTagDir(tagSrc + suffix, tagDest + suffix, suffix)

        if args.tagsAreInstances:
            # We need to update the INDEX file
            _tag, _state, message = indexRead(args.base, tagSrc)
            indexWrite(args.base, tagSrc, IndexStates.MOVE,
                    "(moved to {}) {}".format(tagDest, message))
            indexWrite(args.base, tagDest, _state, message)

        # We have to update any dated links...
        dateInfo = tagDates.get(tagSrc)
        if dateInfo:
            # Note that dated directories do not include the results root
            tagDirDest = tagDest + suffix
            def getTagSansResults(d):
                if not d.startswith(args.tag_from_root + "/"):
                    raise ValueError("Bad path? {0}".format(d))
                return d[len(args.tag_from_root)+1:]
            sansResultDir = getTagSansResults(tagSrc + suffix)
            oldLink = os.path.join(args.base, args.tag_from_root, 'dated',
                    dateInfo[0], dateInfo[1],
                    dateInfo[2] + '-' + sansResultDir)
            os.unlink(oldLink)
            safeRollback(os.path.dirname(oldLink))
            targLink = os.path.join(args.tag_to_root, 'dated', dateInfo[0],
                    dateInfo[1], dateInfo[2] + '-'
                    + getTagSansResults(tagDirDest))
            safeMake(os.path.dirname(targLink))
            os.symlink(os.path.relpath(tagDirDest, os.path.dirname(targLink)),
                    targLink)
    latestTracker.commit(True)


def _runSupervisor(args):
    """Runs the supervisor functionality.  That is, restarts any processes that
    have not updated their heartbeats in awhile."""
    ap = HelpfulParser("Designed to be put in crontab (or some other "
            "scheduler).  Retries experiments started with --retry-until-stall "
            "or -r if they are no longer running.")
    ap.add_argument("-v", "--verbose", action = 'store_true')
    ap.add_argument("--manual", action='store_true')
    args = ap.parse_args(args)
    retryDir = os.path.join(os.path.expanduser("~"), ".gitresults")

    exps = os.path.lexists(retryDir) and os.listdir(retryDir) or []

    def log(m):
        print("{} {}".format(
                datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
                m))

    now = time.time()
    startedCount = 0
    # Used for tests
    allStarted = []
    for retryKey in exps:
        isTestKey = retryKey.startswith("rtest")
        if isTestKey and not IS_TEST or not isTestKey and IS_TEST:
            continue
        if not retryKey.startswith("r"):
            # Bad, archived experiment
            continue

        def deleteExp(reason):
            log("Deleting experiment {}: {}".format(retryKey, reason))
            shutil.rmtree(getPathForResumeKey(retryKey))

        if not os.path.lexists(getPathForResumeKey(retryKey, "settings")):
            # OK, the settings file doesn't even exist.  If it still doesn't in
            # a second (it is created RIGHT after the folder), then remove this
            # experiment.
            time.sleep(1.0)
            if not os.path.lexists(getPathForResumeKey(retryKey, "settings")):
                deleteExp("settings file does not exist")
                continue

        # If we reach here, there are settings to be checked and respected
        try:
            expArgs = pickle.loads(open(getPathForResumeKey(retryKey,
                    "settings"), 'rb').read(), encoding='bytes')
        except pickle.UnpicklingError:
            newDir = getPathForResumeKey("bad_" + retryKey)
            log("Corrupt experiment {}, unpickle of settings failed.  Moving "
                    "to {}".format(
                        retryKey, newDir))
            if os.path.lexists(newDir):
                shutil.rmtree(newDir)
            os.rename(getPathForResumeKey(retryKey), newDir)
            continue
        expMtime = os.path.getmtime(getPathForResumeKey(retryKey, "settings"))
        if os.path.lexists(getPathForResumeKey(retryKey, "heartbeat")):
            expMtime = os.path.getmtime(getPathForResumeKey(retryKey,
                    "heartbeat"))

        # Ensure that the experiment directory still exists
        if not hasattr(expArgs, 'base'):
            deleteExp("Old version, no 'base' folder found")
            continue
        if not os.path.lexists(expArgs.base):
            deleteExp(".git folder containing experiment no longer exists: {}"
                    .format(expArgs.base))
            continue

        isManual = False
        if not hasattr(expArgs, 'setupInfo'):
            log("Experiment did not finish setup, maybe OK?")
        elif not os.path.lexists(expArgs.setupInfo[0]):
            # Might be manual resume, check it
            if os.path.lexists(expArgs.setupInfo[0][:-4] + MANUAL_SUFFIX):
                isManual = True
            else:
                deleteExp("Results directory no longer exists: {}".format(
                        expArgs.setupInfo[0]))
                continue

        if isManual != args.manual:
            continue

        # Extra flags passed to git-results when running this experiment
        forceAbortFlag = [""]
        if isManual:
            oname = expArgs.setupInfo[0]
            nname = expArgs.setupInfo[0][:-4] + MANUAL_SUFFIX
            def checkAbort():
                """Inner function for failing this experiment.

                Returns: True if failed, False if not"""
                if not IS_TEST:
                    r = ""
                    while r not in [ 'y', 'n' ]:
                        r = raw_input("Should git-results abort this "
                                "experiment permanently? (y/n) ")
                        r = r.strip().lower()
                    if r == 'n':
                        return False

                # If we reach this point, should fail the experiment
                # permanently.
                forceAbortFlag[0] = "--internal-retry-abort"
                return True
            if not IS_TEST:
                r = ""
                while r not in [ 'y', 'n' ]:
                    r = raw_input(
                            "git-results supervisor is considering "
                                "restarting: \n"
                            "{}\n\n"
                            "Would you like to retry this experiment? (y/n) "
                            .format(nname))
                    r = r.strip().lower()
                if r != 'y':
                    if not checkAbort():
                        print("SKIPPING {}".format(nname))
                        continue
                else:
                    print("RETRYING {}".format(nname))
            elif IS_TEST_FAIL_MANUAL:
                if not checkAbort():
                    continue
            # Ok, retry it
            os.rename(nname, oname)

        # Note - we allow a 1 second offset since file system times are rounded,
        # which can cause us issues.
        if now - expMtime >= expArgs.retry_delay - 1.0 or forceAbortFlag[0]:
            # Launch it!
            log("{} experiment {}".format(
                    "Resuming" if not forceAbortFlag[0] else "Aborting",
                    retryKey))
            startedCount += 1
            p = subprocess.Popen(shlex.split(
                    "git results {} --internal-retry-continue {}".format(
                        forceAbortFlag[0], retryKey)), stdout = subprocess.PIPE,
                    stderr = subprocess.PIPE)
            if not forceAbortFlag[0]:
                # Note - the ONLY reason that we tee these threads is so that
                # nosetests will capture their output.
                iothreads = [ tee(p.stdout, sys.stdout),
                        tee(p.stderr, sys.stderr) ]
            allStarted.append(p)

    if args.verbose:
        log("Supervisor finished; {} '-r' experiments restarted".format(
                startedCount))
    return allStarted


def run(programArgs = None):
    """Wraps _run, which does the work, so that the cwd is preserved.  Used for
    tests."""
    odir = os.getcwd()
    try:
        return _run(programArgs)
    finally:
        os.chdir(odir)


def _run(programArgs):
    if programArgs is None:
        programArgs = sys.argv[1:]

    if len(programArgs) > 0:
        if programArgs[0] == "move":
            return _runMove(programArgs[1:])
        elif programArgs[0] == "link":
            return _runLink(programArgs[1:])
        elif programArgs[0] == "supervisor":
            return _runSupervisor(programArgs[1:])

    ap = HelpfulParser(description = "A git extension for cataloging "
            "computation results.  Subcommands available: move, link, "
            "supervisor (e.g. git results move -h)")
    ap.add_argument("-i", "--in-place", action = 'store_true',
            help = "Do the build in place.  If you use this, you can't run "
                "several simultaneous git results calls on the same repo.  "
                "Recommended when you're trying to get a build working, or "
                "for testing short experiments, but not otherwise.")
    ap.add_argument("-x", "--extra-file", action = 'append', default = [],
            help = "Of form path1:path2, where path1 is relative or absolute "
                "to the current working directory, and path2 is relative to "
                "the path of the active git-results.cfg.  Copies "
                "file path1 into path2 before running the build and run "
                "commands.  Note that the extra file will be included "
                "as a \"result\".  That is, retained after the experiment "
                "completes.  --extra-file can be specified multiple times.")
    ap.add_argument("-m", "--message", help = "Commit message / "
            "git-results-message content paired with results.  If unspecified, "
            "we'll pop open an editor for you (like git commit)")
    ap.add_argument("--internal-retry-continue", action = 'store_true',
            help = "Used by supervisor, resumes a previously aborted experiment.  "
            "Uses the corresponding values saved in ~/.gitresults/[tagKey]/settings")
    ap.add_argument("--internal-retry-abort", action="store_true",
            help = "Used by supervisor to manually abort experiments.")
    ap.add_argument("tag", help = textwrap.dedent("""
            Path to organize these results under.  For instance, giving a tag of parity/squids
            will create a git tag for the commit as parity/squids, and will create the
            folders "parity" and subfolder "squids" to store the results in.
            "dated" is an invalid option, as that is a unique folder created by
            git-results to help with indexing results by date.
            "latest" is invalid as well, referring to another unique folder."""))
    ap.add_argument("--version", action="version", version="0.2.8")
    args = ap.parse_args(programArgs)
    args.cwd = os.getcwd()

    if args.internal_retry_continue:
        # args.tag is our key for resuming a paused experiment.
        # Preserve internal_retry_abort!
        ofail = args.internal_retry_abort
        args = pickle.loads(open(getPathForResumeKey(args.tag, "settings"),
                'rb').read(), encoding='bytes')
        args.internal_retry_abort = ofail
    else:
        # Starting a new experiment
        _processTagArgs(args, "tag")

        if args.retry_until_stall:
            LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
            header = "r"
            if IS_TEST:
                header = "rtest"
            while True:
                retryKey = header + ''.join([ random.choice(LETTERS)
                        for _ in range(8) ])
                try:
                    os.makedirs(getPathForResumeKey(retryKey))
                    break
                except OSError as e:
                    if e.errno != 17:
                        # Folder already exists
                        raise
            args.retryKey = retryKey
            with open(getPathForResumeKey(retryKey, "settings"), 'wb') as f:
                f.write(pickle.dumps(args))

    # Work in the parent of the results root, which has git-results-* files
    os.chdir(os.path.join(args.base, os.path.dirname(args.tag_root)))
    if not args.run:
        print(textwrap.dedent("""
                To use git-results, you must have a run command in your cfg
                file.  This file must contain a single shell-executable
                line of text that runs your experiment."""))
        sys.exit(1)

    if not hasattr(args, 'setupInfo'):
        # Set up the experiment
        resultsDirRun, datedLinkRun, latestLinkRun, commitTag = setupExperiment(
                args, args.base, args.tag_root, args.tag, args.message)
        if args.retry_until_stall:
            args.setupInfo = [ resultsDirRun, datedLinkRun, latestLinkRun,
                    commitTag ]
            with open(getPathForResumeKey(args.retryKey, "settings.new"), 'wb') as f:
                f.write(pickle.dumps(args))
            os.rename(getPathForResumeKey(args.retryKey, "settings.new"),
                    getPathForResumeKey(args.retryKey, "settings"))
            with open(os.path.join(resultsDirRun, "git-results-retry-key"), 'w') as f:
                f.write(args.retryKey)
    else:
        resultsDirRun, datedLinkRun, latestLinkRun, commitTag = args.setupInfo

    resultsDir = resultsDirRun[:-len(RUN_SUFFIX)]
    datedLink = datedLinkRun[:-len(RUN_SUFFIX)]
    latestLink = latestLinkRun[:-len(RUN_SUFFIX)]

    runFailed = False
    runWillRetry = False
    wasMoveFailure = False
    try:
        # NOTE - For e.g. MPI, or other network tasks, we do not use the
        # standard tempfile methods.  The path has to be accessible on all
        # machines.
        expDir = None
        if args.in_place:
            expDir = None
        else:
            safeMake(os.path.join(args.base, args.tag_root, ".tmp"))
            if args.retry_until_stall:
                # Use the retry key as the temporary folder
                expDir = os.path.join(args.base, args.tag_root, ".tmp",
                        args.retryKey)
                try:
                    # Make the dir, if it already exists that's OK
                    os.mkdir(expDir)
                except OSError as e:
                    if e.errno != 17:
                        raise
            else:
                # We need a temporary folder
                while True:
                    randName = "".join([
                            random.choice("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789")
                            for _ in range(8) ])
                    expDir = os.path.join(args.base, args.tag_root, ".tmp",
                            randName)
                    try:
                        os.mkdir(expDir)
                        break
                    except OSError as e:
                        # Already exists
                        if e.errno != 17:
                            raise

        # Link expDir into git-results-tmp
        tmpDirLink = os.path.join(resultsDirRun, 'git-results-tmp')
        try:
            os.symlink(
                    os.path.join(os.path.abspath(expDir or '.'), os.path.dirname(args.tag_root)),
                    tmpDirLink)
        except OSError as e:
            # Already exists
            if e.errno != 17:
                raise

        cleanupResults = True
        try:
            r, abrt, wasMoveFailure = runExperiment(args, expDir,
                    os.path.dirname(args.tag_root), args.extra_file,
                    resultsDirRun, commitTag, args.trim)
            if r != 0 or abrt:
                runFailed = True
                if abrt:
                    runFailed = abrt

            if args.retry_until_stall:
                # Supervisor will retry, make sure that the experiment's
                # progress is NOT deleted
                runWillRetry = args.isGoingToRetry

            if wasMoveFailure:
                # Prevent deleting of temporary files
                cleanupResults = False
        finally:
            cleanupResults = cleanupResults and not runWillRetry
            if cleanupResults:
                os.unlink(tmpDirLink)
            if expDir is not None and cleanupResults:
                s = time.time()
                failing = True
                while time.time() - s < 10.0:
                    # Wait up to 10 seconds for the program to stop adding files
                    # to the experiment folder (child processes)
                    try:
                        shutil.rmtree(expDir)
                        failing = False
                        break
                    except OSError as e:
                        # Directory not empty
                        if e.errno != 39:
                            raise
                        print("Files still being written...")
                        time.sleep(1.0)
                if failing:
                    print("Could not remove {}".format(expDir))
                safeRollback(os.path.dirname(expDir))
    except KeyboardInterrupt:
        # This can happen.  Should not delete associated tags/links.
        print("*** POTENTIALLY FATAL ERROR: KeyboardInterrupt in outer loop? "
                "{}".format(traceback.format_exc()))
    except:
        # Roll back our tag and results folders...  This catches a build fail
        print("Caught error (presumably in build): {}".format(
                traceback.format_exc()))
        print("Deleting {0} and associated tags / links".format(resultsDirRun))
        checked([ "git", "tag", "-d", commitTag ], nonZeroOk = [ 1 ])
        try:
            os.unlink(datedLinkRun)
        except OSError as e:
            # File does not exist is OK, we were just trying to delete it anyway
            if e.errno != 2:
                raise
        safeRollback(os.path.dirname(datedLinkRun))
        try:
            oldLink = os.readlink(latestLinkRun)
            if os.path.abspath(os.path.join(os.path.dirname(latestLinkRun),
                    oldLink)) == os.path.abspath(resultsDirRun):
                os.unlink(latestLinkRun)
                safeRollback(os.path.dirname(latestLinkRun))
        except OSError as e:
            # Link doesn't exist?
            if e.errno != 2:
                raise
        safeRemoveDir(resultsDirRun)
        indexExpunge(args.base, commitTag)

        if args.retry_until_stall:
            # Get rid of experiment entirely
            shutil.rmtree(getPathForResumeKey(args.retryKey))

        # Build failed for sure!
        sys.exit(1)

    if runWillRetry:
        if runFailed != "manual":
            print("Exiting without cleaning up, supervisor will retry")
        else:
            # Switch to manual mode
            print("Exiting without cleaning up, but supervisor will NOT retry."
                    "\n\nTo retry, run `git results supervisor --manual`.")
            resultsDirNew = resultsDir + MANUAL_SUFFIX
            os.rename(resultsDirRun, resultsDirNew)
        sys.exit(2)

    # Move the RUN_SUFFIX folder to the appropriate suffix
    newSuffix = ""
    indState = IndexStates.OK
    if runFailed:
        # We failed, but not due to a build failure.
        # Move our numbered file to one like "3-fail"
        newSuffix = FAIL_SUFFIX
        indState = IndexStates.FAIL
        if runFailed == "abort":
            newSuffix = ABORT_SUFFIX
            indState = IndexStates.ABORT
        elif runFailed is True:
            pass
        else:
            raise NotImplementedError(runFailed)
    resultsDirNew = resultsDir + newSuffix

    print("Moving {0} to {1}".format(resultsDirRun, resultsDirNew))
    os.rename(resultsDirRun, resultsDirNew)

    indexUpdate(args.base, commitTag, indState)

    oldLink = os.readlink(datedLinkRun)
    os.unlink(datedLinkRun)
    os.symlink(oldLink[:-len(RUN_SUFFIX)] + newSuffix, datedLink + newSuffix)

    try:
        oldLink = os.readlink(latestLinkRun)
    except FileNotFoundError:
        # No latest link; we clearly were not the latest.
        pass
    else:
        if (os.path.abspath(os.path.join(os.path.dirname(latestLinkRun), oldLink))
                == os.path.abspath(resultsDirRun)):
            # We were the latest test, so do the update
            os.unlink(latestLinkRun)
            os.symlink(oldLink[:-len(RUN_SUFFIX)] + newSuffix,
                    latestLink + newSuffix)

    # If we get here, then we're done without hope of retry
    if args.retry_until_stall:
        shutil.rmtree(getPathForResumeKey(args.retryKey))
        os.unlink(os.path.join(resultsDirNew, "git-results-retry-key"))

    if runFailed or wasMoveFailure:
        sys.exit(1)


if __oname__ == '__main__':
    run()