NEW: Better package splitting for multi-output recipes with negative glob in outputs/files

conda_build/build.py

@@ -14,6 +14,7 @@
 import subprocess
 import sys
 import time
+import typing
 import warnings
 from collections import OrderedDict, deque
 from os.path import dirname, isdir, isfile, islink, join
@@ -1802,7 +1803,7 @@ def post_process_files(m: MetaData, initial_prefix_files):
     return new_files
 
 
-def bundle_conda(output, metadata: MetaData, env, stats, **kw):
+def bundle_conda(output, metadata: MetaData, env, stats, new_prefix_files: typing.Set[str], **kw):
     log = utils.get_logger(__name__)
     log.info("Packaging %s", metadata.dist())
     get_all_replacements(metadata.config)
@@ -1915,10 +1916,26 @@ def bundle_conda(output, metadata: MetaData, env, stats, **kw):
     if files:
         # Files is specified by the output
         # we exclude the list of files that we want to keep, so post-process picks them up as "new"
-        keep_files = {
-            os.path.normpath(pth)
-            for pth in utils.expand_globs(files, metadata.config.host_prefix)
-        }
+        if isinstance(files, dict):
+            # When file matching with include/exclude lists, only
+            # new_prefix_files are considered. Files in the PREFIX from other
+            # recipes (dependencies) are ignored
+            include = files.get("include", [])
+            exclude = files.get("exclude", [])
+            exclude_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(exclude, metadata.config.host_prefix)
+            }
+            keep_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(include, metadata.config.host_prefix)
+            }
+            keep_files = new_prefix_files.intersection(keep_files) - exclude_files
+        else:
+            keep_files = {
+                os.path.normpath(pth)
+                for pth in utils.expand_globs(files, metadata.config.host_prefix)
+            }
         pfx_files = set(utils.prefix_files(metadata.config.host_prefix))
         initial_files = {
             item
@@ -2093,7 +2110,7 @@ def bundle_conda(output, metadata: MetaData, env, stats, **kw):
     return final_outputs
 
 
-def bundle_wheel(output, metadata: MetaData, env, stats):
+def bundle_wheel(output, metadata: MetaData, env, stats, new_prefix_files: typing.Set[str]):
     ext = ".bat" if utils.on_win else ".sh"
     with TemporaryDirectory() as tmpdir, utils.tmp_chdir(metadata.config.work_dir):
         dest_file = os.path.join(metadata.config.work_dir, "wheel_output" + ext)
@@ -2817,7 +2834,7 @@ def build(
                     # This is wrong, files has not been expanded at this time and could contain
                     # wildcards.  Also well, I just do not understand this, because when this
                     # does contain wildcards, the files in to_remove will slip back in.
-                    if "files" in output_d:
+                    if "files" in output_d and not isinstance(output_d["files"], dict):
                         output_d["files"] = set(output_d["files"]) - to_remove
 
                     # copies the backed-up new prefix files into the newly created host env
@@ -2833,7 +2850,7 @@ def build(
                     with utils.path_prepended(m.config.build_prefix):
                         env = environ.get_dict(m=m)
                     pkg_type = "conda" if not hasattr(m, "type") else m.type
-                    newly_built_packages = bundlers[pkg_type](output_d, m, env, stats)
+                    newly_built_packages = bundlers[pkg_type](output_d, m, env, stats, new_prefix_files)
                     # warn about overlapping files.
                     if "checksums" in output_d:
                         for file, csum in output_d["checksums"].items():

docs/source/resources/define-metadata.rst

@@ -1298,7 +1298,7 @@ build prefix. Explicit file lists support glob expressions.
 Directory names are also supported, and they recursively include
 contents.
 
-.. code-block:: none
+.. code-block:: yaml
 
    outputs:
      - name: subpackage-name
@@ -1308,6 +1308,22 @@ contents.
          - *.some-extension
          - somefolder/*.some-extension
 
+Files can be excluded by specifying `files` as a dictionary separating
+files to `include` from those to `exclude`:
+
+.. code-block:: yaml
+
+   outputs:
+     - name: subpackage-name
+       files:
+         include:
+           - a-file
+           - a-folder
+           - *.some-extension
+           - somefolder/*.some-extension
+         exclude:
+           - *.exclude-extension
+
 Scripts that create or move files into the build prefix can be
 any kind of script. Known script types need only specify the
 script name. Currently the list of recognized extensions is

news/files-exclude.rst

@@ -0,0 +1,25 @@
+Enhancements:
+-------------
+
+* implemented negative matching for `files` for `outputs`
+
+Bug fixes:
+----------
+
+* <news item>
+
+Deprecations:
+-------------
+
+* <news item>
+
+Docs:
+-----
+
+* <news item>
+
+Other:
+------
+
+* <news item>
+

tests/test-recipes/split-packages/copying_files/build.sh

@@ -6,3 +6,4 @@ echo "weee" > $PREFIX/somedir/subpackage_file1
 # test glob patterns
 echo "weee" > $PREFIX/subpackage_file1.ext
 echo "weee" > $PREFIX/subpackage_file2.ext
+echo "weee" > $PREFIX/subpackage_file3.ext

tests/test-recipes/split-packages/copying_files/meta.yaml

@@ -9,9 +9,12 @@ requirements:
 outputs:
   - name: my_script_subpackage
     files:
-      - subpackage_file1
-      - somedir
-      - "*.ext"
+      include:
+        - subpackage_file1
+        - somedir
+        - "*.ext"
+      exclude:
+        - "*3.ext"
     test:
       script: subpackage_test.py
       script_interpreter: python

tests/test-recipes/split-packages/copying_files/subpackage_test.py

@@ -31,3 +31,7 @@
     contents = contents.decode()
 assert "weee" in contents, 'incorrect file contents: %s' % contents
 print("glob OK")
+
+filename = os.path.join(os.environ['PREFIX'], 'subpackage_file3.ext')
+assert not os.path.isfile(filename)
+print("glob exclude OK")