Add error message printed if snakemake#823 is encountered, also exit 1

The main scheduler loop terminates if there are no more runnable
jobs. This tests if there still are jobs that should be run, but that
we cannot seem to get to. If so, print an error message, print the
files affected, and exit 1. The latter is important so Snakemake can,
as hotfix, be rerun in a loop until it's actually completed all
tasks.

snakemake/scheduler.py

@@ -42,6 +42,11 @@ def cumsum(iterable, zero=[0]):
     "Exiting because a job execution failed. " "Look above for error message"
 )
 
+_ERROR_MSG_ISSUE_823 = (
+    "BUG: Out of jobs ready to be started, but not all files built yet."
+    " Please check https://github.com/snakemake/snakemake/issues/823 for more information."
+)
+
 
 class DummyRateLimiter(ContextDecorator):
     def __enter__(self):
@@ -464,11 +469,23 @@ def schedule(self):
                         return False
                     continue
 
-                # normal shutdown because all jobs have been finished
+                # all runnable jobs have finished, normal shutdown
                 if not needrun and (not running or self.workflow.immediate_submit):
                     self._executor.shutdown()
                     if errors:
                         logger.error(_ERROR_MSG_FINAL)
+                    # we still have unfinished jobs. this is not good. direct
+                    # user to github issue
+                    if self.remaining_jobs or self.finished_jobs != len(self.dag):
+                        logger.error(_ERROR_MSG_ISSUE_823)
+                        logger.error(
+                            "Remaining jobs:\n"
+                            + "\n".join(
+                                " - " + str(job) + ": " + ", ".join(job.output)
+                                for job in self.remaining_jobs
+                            )
+                        )
+                        return False
                     return not errors
 
                 # continue if no new job needs to be executed