Merge pull request #4100 from easybuilders/4.6.x

release EasyBuild 4.6.2

.github/workflows/bootstrap_script.yml

@@ -54,10 +54,10 @@ jobs:
             lc_all: C
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/container_tests.yml

@@ -10,10 +10,10 @@ jobs:
         python: [2.7, 3.6]
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64

.github/workflows/eb_command.yml

@@ -9,10 +9,10 @@ jobs:
         python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64
@@ -77,6 +77,10 @@ jobs:
               echo "Looking for pattern \"${pattern}\" in eb_version.out..."
               grep "$pattern" eb_version.out
           done
+          if grep -q "Considering ''" eb_version.out; then
+              echo '`eb` did wrongly consider an empty command'
+              false
+          fi
           # also check when specifying Python command via $EB_PYTHON
           for eb_python in "python${pymajver}" "python${pymajminver}"; do
               export EB_PYTHON="${eb_python}"

.github/workflows/linting.yml

@@ -8,10 +8,10 @@ jobs:
         python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/unit_tests.yml

@@ -76,10 +76,10 @@ jobs:
             lc_all: C
       fail-fast: false
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{matrix.python}}
         architecture: x64

RELEASE_NOTES

@@ -4,6 +4,29 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.6.2 (October 21st 2022)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add support for easystack file that contains easyconfig filenames + implement parsing of configuration options (#4021)
+  - skip over unset $EB_PYTHON/$EB_INSTALLPYTHON in eb wrappeer script (#4080)
+  - add GITHUB_RELEASE and GITHUB_LOWER_RELEASE templates (#4084)
+  - add `%(cuda_cc_cmake)s` template (#4087)
+- various bug fixes, including:
+  - make check_sha256_checksums verify all checksums if they're specified as a dict value (#4076)
+  - replace use of symlink with copied files in `alt_location` tests to fix failing EasyBuild installation on BeeGFS (#4083)
+  - fix trying to generate RPATH wrappers for Clang (#4088)
+  - make sure that GitPython version is a proper version before checking minimal required version (#4090, #4091)
+  - first look for patch in alt_location when it is specified (#4093)
+- other changes:
+  - make scripts executable (#4081)
+  - make --inject-checksums inject dictionary value for checksums which maps filename to SHA256 checksum (#4085)
+  - update to v3 of actions/checkout and actions/setup-python in CI workflows (#4089)
+  - use SYSTEM template constant in dependencies instead of True in framework tests (#4094)
+
+
 v4.6.1 (September 12th 2022)
 ----------------------------
 

easybuild/framework/easyblock.py

@@ -753,8 +753,15 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
             foundfile = None
             failedpaths = []
 
-            # always look first in the dir of the current eb file
-            ebpath = [os.path.dirname(self.cfg.path)]
+            # always look first in the dir of the current eb file unless alt_location is set
+            if alt_location is None:
+                ebpath = [os.path.dirname(self.cfg.path)]
+                self.log.info("Considering directory in which easyconfig file is located when searching for %s: %s",
+                              filename, ebpath[0])
+            else:
+                ebpath = []
+                self.log.info("Not considering directory in which easyconfig file is located when searching for %s "
+                              "because alt_location is set to %s", filename, alt_location)
 
             # always consider robot + easyconfigs install paths as a fall back (e.g. for patch files, test cases, ...)
             common_filepaths = []
@@ -2375,33 +2382,34 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
             checksum_issues.append(msg)
 
         for fn, checksum in zip(sources + patches, checksums):
+
+            # a checksum may be specified as a dictionary which maps filename to actual checksum
+            # for example when different source files are used for different CPU architectures
             if isinstance(checksum, dict):
-                # sources entry may be a dictionary rather than just a string value with filename
-                if isinstance(fn, dict):
-                    filename = fn['filename']
-                else:
-                    filename = fn
-                checksum = checksum.get(filename)
-
-            # take into account that we may encounter a tuple of valid SHA256 checksums
-            # (see https://github.com/easybuilders/easybuild-framework/pull/2958)
-            if isinstance(checksum, tuple):
-                # 1st tuple item may indicate checksum type, must be SHA256 or else it's blatently ignored here
-                if len(checksum) == 2 and checksum[0] == CHECKSUM_TYPE_SHA256:
-                    valid_checksums = (checksum[1],)
-                else:
-                    valid_checksums = checksum
+                checksums_to_check = checksum.values()
             else:
-                valid_checksums = (checksum,)
-
-            non_sha256_checksums = [c for c in valid_checksums if not is_sha256_checksum(c)]
-            if non_sha256_checksums:
-                if all(c is None for c in non_sha256_checksums):
-                    print_warning("Found %d None checksum value(s), please make sure this is intended!" %
-                                  len(non_sha256_checksums))
+                checksums_to_check = [checksum]
+
+            for checksum in checksums_to_check:
+                # take into account that we may encounter a tuple of valid SHA256 checksums
+                # (see https://github.com/easybuilders/easybuild-framework/pull/2958)
+                if isinstance(checksum, tuple):
+                    # 1st tuple item may indicate checksum type, must be SHA256 or else it's blatently ignored here
+                    if len(checksum) == 2 and checksum[0] == CHECKSUM_TYPE_SHA256:
+                        valid_checksums = (checksum[1],)
+                    else:
+                        valid_checksums = checksum
                 else:
-                    msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums)
-                    checksum_issues.append(msg)
+                    valid_checksums = (checksum,)
+
+                non_sha256_checksums = [c for c in valid_checksums if not is_sha256_checksum(c)]
+                if non_sha256_checksums:
+                    if all(c is None for c in non_sha256_checksums):
+                        print_warning("Found %d None checksum value(s), please make sure this is intended!" %
+                                      len(non_sha256_checksums))
+                    else:
+                        msg = "Non-SHA256 checksum(s) found for %s: %s" % (fn, valid_checksums)
+                        checksum_issues.append(msg)
 
         return checksum_issues
 
@@ -4415,11 +4423,11 @@ def make_checksum_lines(checksums, indent_level):
         line_indent = INDENT_4SPACES * indent_level
         checksum_lines = []
         for fn, checksum in checksums:
-            checksum_line = "%s'%s',  # %s" % (line_indent, checksum, fn)
+            checksum_line = "%s{'%s': '%s'}," % (line_indent, fn, checksum)
             if len(checksum_line) > MAX_LINE_LENGTH:
                 checksum_lines.extend([
-                    "%s# %s" % (line_indent, fn),
-                    "%s'%s'," % (line_indent, checksum),
+                    "%s{'%s':" % (line_indent, fn),
+                    "%s '%s'}," % (line_indent, checksum),
                 ])
             else:
                 checksum_lines.append(checksum_line)

easybuild/framework/easyconfig/templates.py

@@ -90,6 +90,7 @@
     ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"),
     ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via "
      "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"),
+    ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
     ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
     ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
     ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
@@ -113,6 +114,10 @@
      'GitHub source URL (namelower is used if github_account easyconfig parameter is not specified)'),
     ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive',
      'GitHub source URL (lowercase name, namelower is used if github_account easyconfig parameter is not specified)'),
+    ('GITHUB_RELEASE', 'https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s',
+     'GitHub release URL (namelower is used if github_account easyconfig parameter is not specified)'),
+    ('GITHUB_LOWER_RELEASE', 'https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s',
+     'GitHub release URL (namelower is used if github_account easyconfig parameter is not specified)'),
     ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s',
      'download.savannah.gnu.org source url'),
     ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s',
@@ -353,6 +358,7 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None)
         template_values['cuda_compute_capabilities'] = ','.join(cuda_compute_capabilities)
         template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
         template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
+        template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
         sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
         template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
         template_values['cuda_sm_space_sep'] = ' '.join(sm_values)

easybuild/framework/easystack.py

@@ -41,6 +41,8 @@
     pass
 _log = fancylogger.getLogger('easystack', fname=False)
 
+EASYSTACK_DOC_URL = 'https://docs.easybuild.io/en/latest/Easystack-files.html'
+
 
 def check_value(value, context):
     """
@@ -68,10 +70,16 @@ def __init__(self):
         self.easybuild_version = None
         self.robot = False
         self.software_list = []
+        self.easyconfigs = []  # A list of easyconfig names. May or may not include .eb extension
+        # A dict where keys are easyconfig names, values are dictionary of options that should be
+        # applied for that easyconfig
+        self.ec_opts = {}
 
     def compose_ec_filenames(self):
         """Returns a list of all easyconfig names"""
         ec_filenames = []
+
+        # entries specified via 'software' top-level key
         for sw in self.software_list:
             full_ec_version = det_full_ec_version({
                 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version},
@@ -80,6 +88,10 @@ def compose_ec_filenames(self):
             })
             ec_filename = '%s-%s.eb' % (sw.name, full_ec_version)
             ec_filenames.append(ec_filename)
+
+        # entries specified via 'easyconfigs' top-level key
+        for ec in self.easyconfigs:
+            ec_filenames.append(ec)
         return ec_filenames
 
     # flags applicable to all sw (i.e. robot)
@@ -108,21 +120,89 @@ class EasyStackParser(object):
 
     @staticmethod
     def parse(filepath):
-        """Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
+        """
+        Parses YAML file and assigns obtained values to SW config instances as well as general config instance"""
         yaml_txt = read_file(filepath)
 
         try:
             easystack_raw = yaml.safe_load(yaml_txt)
         except yaml.YAMLError as err:
             raise EasyBuildError("Failed to parse %s: %s" % (filepath, err))
 
+        easystack_data = None
+        top_keys = ('easyconfigs', 'software')
+        keys_found = []
+        for key in top_keys:
+            if key in easystack_raw:
+                keys_found.append(key)
+        # For now, we don't support mixing multiple top_keys, so check that only one was defined
+        if len(keys_found) > 1:
+            keys_string = ', '.join(keys_found)
+            msg = "Specifying multiple top level keys (%s) " % keys_string
+            msg += "in one EasyStack file is currently not supported"
+            msg += ", see %s for documentation." % EASYSTACK_DOC_URL
+            raise EasyBuildError(msg)
+        elif len(keys_found) == 0:
+            msg = "Not a valid EasyStack YAML file: no 'easyconfigs' or 'software' top-level key found"
+            msg += ", see %s for documentation." % EASYSTACK_DOC_URL
+            raise EasyBuildError(msg)
+        else:
+            key = keys_found[0]
+            easystack_data = easystack_raw[key]
+
+        parse_method_name = 'parse_by_' + key
+        parse_method = getattr(EasyStackParser, 'parse_by_%s' % key, None)
+        if parse_method is None:
+            raise EasyBuildError("Easystack parse method '%s' not found!", parse_method_name)
+
+        # assign general easystack attributes
+        easybuild_version = easystack_raw.get('easybuild_version', None)
+        robot = easystack_raw.get('robot', False)
+
+        return parse_method(filepath, easystack_data, easybuild_version=easybuild_version, robot=robot)
+
+    @staticmethod
+    def parse_by_easyconfigs(filepath, easyconfigs, easybuild_version=None, robot=False):
+        """
+        Parse easystack file with 'easyconfigs' as top-level key.
+        """
+
         easystack = EasyStack()
 
-        try:
-            software = easystack_raw["software"]
-        except KeyError:
-            wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found"
-            raise EasyBuildError(wrong_structure_file)
+        for easyconfig in easyconfigs:
+            if isinstance(easyconfig, str):
+                if not easyconfig.endswith('.eb'):
+                    easyconfig = easyconfig + '.eb'
+                easystack.easyconfigs.append(easyconfig)
+            elif isinstance(easyconfig, dict):
+                if len(easyconfig) == 1:
+                    # Get single key from dictionary 'easyconfig'
+                    easyconf_name = list(easyconfig.keys())[0]
+                    # Add easyconfig name to the list
+                    if not easyconf_name.endswith('.eb'):
+                        easyconf_name_with_eb = easyconf_name + '.eb'
+                    else:
+                        easyconf_name_with_eb = easyconf_name
+                    easystack.easyconfigs.append(easyconf_name_with_eb)
+                    # Add options to the ec_opts dict
+                    if 'options' in easyconfig[easyconf_name].keys():
+                        easystack.ec_opts[easyconf_name_with_eb] = easyconfig[easyconf_name]['options']
+                else:
+                    dict_keys = ', '.join(easyconfig.keys())
+                    msg = "Failed to parse easystack file: expected a dictionary with one key (the EasyConfig name), "
+                    msg += "instead found keys: %s" % dict_keys
+                    msg += ", see %s for documentation." % EASYSTACK_DOC_URL
+                    raise EasyBuildError(msg)
+
+        return easystack
+
+    @staticmethod
+    def parse_by_software(filepath, software, easybuild_version=None, robot=False):
+        """
+        Parse easystack file with 'software' as top-level key.
+        """
+
+        easystack = EasyStack()
 
         # assign software-specific easystack attributes
         for name in software:
@@ -224,8 +304,8 @@ def parse(filepath):
                     easystack.software_list.append(sw)
 
             # assign general easystack attributes
-            easystack.easybuild_version = easystack_raw.get('easybuild_version', None)
-            easystack.robot = easystack_raw.get('robot', False)
+            easystack.easybuild_version = easybuild_version
+            easystack.robot = robot
 
         return easystack
 
@@ -243,12 +323,16 @@ def parse_easystack(filepath):
 
     easyconfig_names = easystack.compose_ec_filenames()
 
-    general_options = easystack.get_general_options()
+    # Disabled general options for now. We weren't using them, and first want support for EasyConfig-specific options.
+    # Then, we need a method to resolve conflicts (specific options should win)
+    # general_options = easystack.get_general_options()
 
     _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names)))
-    if len(general_options) != 0:
-        _log.debug("General options for installation are: \n%s" % str(general_options))
-    else:
-        _log.debug("No general options were specified in easystack")
-
-    return easyconfig_names, general_options
+    _log.debug("Using EasyConfig specific options based on the following dict:")
+    _log.debug(easystack.ec_opts)
+    # if len(general_options) != 0:
+    #    _log.debug("General options for installation are: \n%s" % str(general_options))
+    # else:
+    #    _log.debug("No general options were specified in easystack")
+
+    return easyconfig_names, easystack.ec_opts