fix: provide temporary IPYTHONDIR for notebook execution in order to avoid race conditions in https://github.com/ipython/ipython/blob/master/IPython/paths.py#L20 upon execution of multiple notebooks at the same time.

snakemake/notebook.py

@@ -71,37 +71,43 @@ def execute_script(self, fname, edit=None):
             fname_out = os.path.join(os.getcwd(), fname_out)
             output_parameter = "--output {fname_out:q}"
 
-        if edit is not None:
-            logger.info("Opening notebook for editing.")
-            cmd = (
-                "jupyter notebook --browser ':' --no-browser --log-level ERROR --ip {edit.ip} --port {edit.port} "
-                "--NotebookApp.quit_button=True {{fname:q}}".format(edit=edit)
-            )
-        else:
-            cmd = (
-                "jupyter-nbconvert --log-level ERROR --execute {output_parameter} "
-                "--to notebook --ExecutePreprocessor.timeout=-1 {{fname:q}}".format(
-                    output_parameter=output_parameter
+        with tempfile.TemporaryDirectory() as tmp:
+            if edit is not None:
+                logger.info("Opening notebook for editing.")
+                cmd = (
+                    "jupyter notebook --browser ':' --no-browser --log-level ERROR --ip {edit.ip} --port {edit.port} "
+                    "--NotebookApp.quit_button=True {{fname:q}}".format(edit=edit)
+                )
+            else:
+                cmd = (
+                    "jupyter-nbconvert --log-level ERROR --execute {output_parameter} "
+                    "--to notebook --ExecutePreprocessor.timeout=-1 {{fname:q}}".format(
+                        output_parameter=output_parameter,
+                    )
                 )
-            )
 
-        if ON_WINDOWS:
-            fname = fname.replace("\\", "/")
-            fname_out = fname_out.replace("\\", "/") if fname_out else fname_out
+            if ON_WINDOWS:
+                fname = fname.replace("\\", "/")
+                fname_out = fname_out.replace("\\", "/") if fname_out else fname_out
 
-        self._execute_cmd(cmd, fname_out=fname_out, fname=fname)
+            self._execute_cmd(
+                cmd,
+                fname_out=fname_out,
+                fname=fname,
+                additional_envvars={"IPYTHONDIR": tmp},
+            )
 
-        if edit:
-            logger.info("Saving modified notebook.")
-            nb = nbformat.read(fname, as_version=4)
+            if edit:
+                logger.info("Saving modified notebook.")
+                nb = nbformat.read(fname, as_version=4)
 
-            self.remove_preamble_cell(nb)
+                self.remove_preamble_cell(nb)
 
-            # clean up all outputs
-            for cell in nb["cells"]:
-                cell["outputs"] = []
+                # clean up all outputs
+                for cell in nb["cells"]:
+                    cell["outputs"] = []
 
-            nbformat.write(nb, self.local_path)
+                nbformat.write(nb, self.local_path)
 
     def insert_preamble_cell(self, preamble, notebook):
         import nbformat

snakemake/shell.py

@@ -219,6 +219,17 @@ def __new__(
             envvars["TEMPDIR"] = tmpdir_resource
             envvars["TEMP"] = tmpdir_resource
 
+        if "additional_envvars" in kwargs:
+            env = kwargs["additional_envvars"]
+            if not isinstance(env, dict) or not all(
+                isinstance(v, str) for v in env.values()
+            ):
+                raise WorkflowError(
+                    "Given environment variables for shell command have to be a dict of strings, "
+                    "but the following was provided instead:\n{}".format(env)
+                )
+            envvars.update(env)
+
         if conda_env and cls.conda_block_conflicting_envvars:
             # remove envvars that conflict with conda
             for var in ["R_LIBS", "PYTHONPATH", "PERLLIB", "PERL5LIB"]: