fix: cleaned up and rewritten execution backend structure, (fixing #1475

, #860, #1007, #1008) (PR #1491)
snakemake · Mar 21, 2022 · e87cc97 · e87cc97
1 parent 12d6f67
commit e87cc97
Show file tree

Hide file tree

Showing 16 changed files with 486 additions and 503 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -114,18 +114,19 @@ jobs:
           echo CONTAINER_IMAGE=snakemake/snakemake:$GITHUB_SHA >> $GITHUB_ENV
 
       # TODO reactivate in April (we have no free resources left this month)
-      # - name: Test Kubernetes execution
-      #   if: env.GCP_AVAILABLE
-      #   env:
-      #     CI: true
-      #   run: |
-      #     # activate conda env
-      #     export PATH="/usr/share/miniconda/bin:$PATH"
-      #     source activate snakemake
-
-      #     pytest -s -v -x tests/test_kubernetes.py
-
-      - name: Test AWS execution
+      - name: Test Kubernetes execution
+        if: env.GCP_AVAILABLE
+        env:
+          CI: true
+        run: |
+          # activate conda env
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate snakemake
+
+          pytest -s -v -x tests/test_kubernetes.py
+
+      # TODO temporarily deactivate and fix in separate PR.
+      - name: Test Tibanna (AWS) execution
         if: env.AWS_AVAILABLE
         env: 
           CI: true
@@ -134,16 +135,16 @@ jobs:
           export PATH="/usr/share/miniconda/bin:$PATH"
           source activate snakemake
 
-          pytest -v -x tests/test_tibanna.py
+      #     pytest -v -x -s tests/test_tibanna.py
 
       # TODO reactivate in April (we have no free resources left this month)
-      # - name: Test Google Life Sciences Executor
-      #   if: env.GCP_AVAILABLE
-      #   run: |
-      #     # activate conda env
-      #     export PATH="/usr/share/miniconda/bin:$PATH"
-      #     source activate snakemake
-      #     pytest -s -v -x tests/test_google_lifesciences.py
+      - name: Test Google Life Sciences Executor
+        if: env.GCP_AVAILABLE
+        run: |
+          # activate conda env
+          export PATH="/usr/share/miniconda/bin:$PATH"
+          source activate snakemake
+          pytest -s -v -x tests/test_google_lifesciences.py
 
       - name: Test GA4GH TES executor
         run: |

diff --git a/snakemake/__init__.py b/snakemake/__init__.py
@@ -436,6 +436,9 @@ def snakemake(
             # clean up all previously recorded jobids.
             shell.cleanup()
     else:
+        if default_resources is None:
+            # use full default resources if in cluster or cloud mode
+            default_resources = DefaultResources(mode="full")
         if edit_notebook:
             raise WorkflowError(
                 "Notebook edit mode is only allowed with local execution."
@@ -599,6 +602,7 @@ def snakemake(
             check_envvars=not lint,  # for linting, we do not need to check whether requested envvars exist
             all_temp=all_temp,
             local_groupid=local_groupid,
+            keep_metadata=keep_metadata,
             latency_wait=latency_wait,
         )
         success = True
@@ -800,7 +804,6 @@ def snakemake(
                     export_cwl=export_cwl,
                     batch=batch,
                     keepincomplete=keep_incomplete,
-                    keepmetadata=keep_metadata,
                 )
 
     except BrokenPipeError: