Merge pull request #4499 from easybuilders/4.9.x

release EasyBuild v4.9.1

.github/workflows/container_tests_apptainer.yml

@@ -29,10 +29,18 @@ jobs:
     - name: install OS & Python packages
       run: |
         # for building CentOS 7 container images
-        sudo apt-get install rpm
-        sudo apt-get install dnf
+        APT_PKGS="rpm dnf"
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        APT_PKGS+=" lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev"
+
+        # Avoid apt-get update, as we don't really need it,
+        # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
+        if ! sudo apt-get install $APT_PKGS; then
+          # Try to update cache, then try again to resolve 404s of old packages
+          sudo apt-get update -yqq || true
+          sudo apt-get install $APT_PKGS
+        fi
+
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then

.github/workflows/unit_tests.yml

@@ -178,7 +178,7 @@ jobs:
               echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
               continue
             fi
-            printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
+            printf "\n\n=====================> Using $module_syntax module syntax <=====================\n\n"
             export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
             export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
 

RELEASE_NOTES

@@ -4,6 +4,41 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.1 (5 April 2024)
+---------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - make `is_rpath_wrapper` faster by only checking file contents if file is not located in subdirectory of RPATH wrapper subdirectory (#4406)
+  - add terse support to `--missing-modules` (#4407)
+  - adapt version pattern for EnvironmentModules to allow using development version (#4416)
+  - use `--all` option with EnvironmentModules v4.6+ to get available hidden modules (#4417)
+  - add support for appending to path environment variables via `modextrapaths_append` + add corresponding `allow_append_abs_path` (#4436)
+  - improve output produced by `--check-github` (#4437)
+  - add script for updating local git repos with `develop` branch (#4438)
+  - show error when multiple PR options are passed (#4440)
+  - improve `findPythonDeps` script to recognize non-canonical package names (#4445)
+  - add support for `--from-commit` and `--include-easyblocks-from-commit` (#4468)
+  - improve logging & handling of (empty) `--optarch` values (#4481)
+  - add `--short` option to `findUpdatedEcs` script (#4488)
+  - add generic GCC and Clang compiler flags for RISC-V (#4489)
+- various bug fixes, including:
+  - clean up log file of `EasyBlock` instance in `check_sha256_checksums` (#4452)
+  - fix description of `backup-modules` configuration option (#4456)
+  - replace `'` with `"` for `printf` in CI workflow for running test suite to have bash replace a variable  (#4461)
+  - use `cp -dR` instead of `cp -a` for shell script "extraction" (#4465)
+  - fix link to documentation in `close_pr` message (#4466)
+  - fix `test_github_merge_pr` by using more recent easyconfigs PR (#4470)
+  - add workaround for 404 error when installing packages in CI workflow for testing Apptainer integration (#4472)
+- other changes:
+  - clean up & speed up environment checks (#4409)
+  - use more performant and concise dict construction by using dict comprehensions (#4410)
+  - remove superflous string formatting (#4411)
+  - clean up uses of `getattr` and `hasattr` (#4412)
+  - update copyright lines to 2024 (#4494)
+
+
 v4.9.0 (30 December 2023)
 -------------------------
 

easybuild/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2015-2023 Ghent University
+# Copyright 2015-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/base/fancylogger.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -147,10 +147,10 @@ def _env_to_boolean(varname, default=False):
       >>> _env_to_boolean('NO_FOOBAR')
       False
     """
-    if varname not in os.environ:
+    try:
+        return os.environ[varname].lower() in ('1', 'yes', 'true', 'y')
+    except KeyError:
         return default
-    else:
-        return os.environ.get(varname).lower() in ('1', 'yes', 'true', 'y')
 
 
 OPTIMIZED_ANSWER = "not available in optimized mode"
@@ -286,7 +286,7 @@ def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=Non
         overwrite make record to use a fancy record (with more options)
         """
         logrecordcls = logging.LogRecord
-        if hasattr(self, 'fancyrecord') and self.fancyrecord:
+        if getattr(self, 'fancyrecord', None):
             logrecordcls = FancyLogRecord
         try:
             new_msg = str(msg)
@@ -911,16 +911,15 @@ def resetroot():
 _default_logTo = None
 if 'FANCYLOG_SERVER' in os.environ:
     server = os.environ['FANCYLOG_SERVER']
-    port = DEFAULT_UDP_PORT
     if ':' in server:
         server, port = server.split(':')
+    else:
+        port = DEFAULT_UDP_PORT
 
     # maybe the port was specified in the FANCYLOG_SERVER_PORT env var. this takes precedence
-    if 'FANCYLOG_SERVER_PORT' in os.environ:
-        port = int(os.environ['FANCYLOG_SERVER_PORT'])
-    port = int(port)
+    port = os.environ.get('FANCYLOG_SERVER_PORT', port)
 
-    logToUDP(server, port)
+    logToUDP(server, int(port))
     _default_logTo = logToUDP
 else:
     # log to screen by default

easybuild/base/generaloption.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2011-2023 Ghent University
+# Copyright 2011-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -90,7 +90,7 @@ def set_columns(cols=None):
                 pass
 
     if cols is not None:
-        os.environ['COLUMNS'] = "%s" % cols
+        os.environ['COLUMNS'] = str(cols)
 
 
 def what_str_list_tuple(name):
@@ -199,7 +199,8 @@ class ExtOption(CompleterOption):
     ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + EXTOPTION_EXTRA_OPTIONS
 
     TYPE_STRLIST = ['%s%s' % (name, klass) for klass in ['list', 'tuple'] for name in ['str', 'path']]
-    TYPE_CHECKER = dict([(x, check_str_list_tuple) for x in TYPE_STRLIST] + list(Option.TYPE_CHECKER.items()))
+    TYPE_CHECKER = {x: check_str_list_tuple for x in TYPE_STRLIST}
+    TYPE_CHECKER.update(Option.TYPE_CHECKER)
     TYPES = tuple(TYPE_STRLIST + list(Option.TYPES))
     BOOLEAN_ACTIONS = ('store_true', 'store_false',) + EXTOPTION_LOG
 
@@ -807,7 +808,7 @@ def get_env_options(self):
         epilogprefixtxt += "eg. --some-opt is same as setting %(prefix)s_SOME_OPT in the environment."
         self.epilog.append(epilogprefixtxt % {'prefix': self.envvar_prefix})
 
-        candidates = dict([(k, v) for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)])
+        candidates = {k: v for k, v in os.environ.items() if k.startswith("%s_" % self.envvar_prefix)}
 
         for opt in self._get_all_options():
             if opt._long_opts is None:
@@ -822,8 +823,8 @@ def get_env_options(self):
                         self.environment_arguments.append("%s=%s" % (lo, val))
                     else:
                         # interpretation of values: 0/no/false means: don't set it
-                        if ("%s" % val).lower() not in ("0", "no", "false",):
-                            self.environment_arguments.append("%s" % lo)
+                        if str(val).lower() not in ("0", "no", "false",):
+                            self.environment_arguments.append(str(lo))
                 else:
                     self.log.debug("Environment variable %s is not set" % env_opt_name)
 
@@ -1031,7 +1032,7 @@ def main_options(self):
         # make_init is deprecated
         if hasattr(self, 'make_init'):
             self.log.debug('main_options: make_init is deprecated. Rename function to main_options.')
-            getattr(self, 'make_init')()
+            self.make_init()
         else:
             # function names which end with _options and do not start with main or _
             reg_main_options = re.compile("^(?!_|main).*_options$")
@@ -1189,7 +1190,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non
                 for extra_detail in details[4:]:
                     if isinstance(extra_detail, (list, tuple,)):
                         # choices
-                        nameds['choices'] = ["%s" % x for x in extra_detail]  # force to strings
+                        nameds['choices'] = [str(x) for x in extra_detail]  # force to strings
                         hlp += ' (choices: %s)' % ', '.join(nameds['choices'])
                     elif isinstance(extra_detail, string_type) and len(extra_detail) == 1:
                         args.insert(0, "-%s" % extra_detail)

easybuild/base/optcomplete.py

@@ -513,14 +513,15 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete
             if option:
                 if option.nargs > 0:
                     optarg = True
-                    if hasattr(option, 'completer'):
+                    try:
                         completer = option.completer
-                    elif option.choices:
-                        completer = ListCompleter(option.choices)
-                    elif option.type in ('string',):
-                        completer = opt_completer
-                    else:
-                        completer = NoneCompleter()
+                    except AttributeError:
+                        if option.choices:
+                            completer = ListCompleter(option.choices)
+                        elif option.type in ('string',):
+                            completer = opt_completer
+                        else:
+                            completer = NoneCompleter()
                 # Warn user at least, it could help him figure out the problem.
                 elif hasattr(option, 'completer'):
                     msg = "Error: optparse option with a completer does not take arguments: %s" % (option)
@@ -616,11 +617,9 @@ class CmdComplete(object):
     def autocomplete(self, completer=None):
         parser = OPTIONPARSER_CLASS(self.__doc__.strip())
         if hasattr(self, 'addopts'):
-            fnc = getattr(self, 'addopts')
-            fnc(parser)
+            self.addopts(parser)
 
-        if hasattr(self, 'completer'):
-            completer = getattr(self, 'completer')
+        completer = getattr(self, 'completer', completer)
 
         return autocomplete(parser, completer)
 

easybuild/base/testing.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2014-2023 Ghent University
+# Copyright 2014-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/__init__.py

@@ -1,5 +1,5 @@
 ##
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyblock.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -1421,6 +1421,14 @@ def make_module_extra(self, altroot=None, altversion=None):
                                      value, type(value))
             lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path']))
 
+        for (key, value) in self.cfg['modextrapaths_append'].items():
+            if isinstance(value, string_type):
+                value = [value]
+            elif not isinstance(value, (tuple, list)):
+                raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple",
+                                     value, type(value))
+            lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path']))
+
         modloadmsg = self.cfg['modloadmsg']
         if modloadmsg:
             # add trailing newline to prevent that shell prompt is 'glued' to module load message

easybuild/framework/easyconfig/__init__.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/constants.py

@@ -1,5 +1,5 @@
 #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),

easybuild/framework/easyconfig/default.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -60,7 +60,7 @@
 # we use a tuple here so we can sort them based on the numbers
 CATEGORY_NAMES = ['BUILD', 'CUSTOM', 'DEPENDENCIES', 'EXTENSIONS', 'FILEMANAGEMENT', 'HIDDEN',
                   'LICENSE', 'MANDATORY', 'MODULES', 'OTHER', 'TOOLCHAIN']
-ALL_CATEGORIES = dict((name, eval(name)) for name in CATEGORY_NAMES)
+ALL_CATEGORIES = {name: eval(name) for name in CATEGORY_NAMES}
 
 # List of tuples. Each tuple has the following format (key, [default, help text, category])
 DEFAULT_CONFIG = {
@@ -189,10 +189,12 @@
     'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS],
 
     # MODULES easyconfig parameters
+    'allow_append_abs_path': [False, "Allow specifying absolute paths to append in modextrapaths_append", MODULES],
     'allow_prepend_abs_path': [False, "Allow specifying absolute paths to prepend in modextrapaths", MODULES],
     'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES],
     'modaliases': [{}, "Aliases to be defined in module file", MODULES],
     'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES],
+    'modextrapaths_append': [{}, "Extra paths to be appended in module file", MODULES],
     'modextravars': [{}, "Extra environment variables to be added to module file", MODULES],
     'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES],
     'modunloadmsg': [{}, "Message that should be printed when generated module is unloaded", MODULES],

easybuild/framework/easyconfig/easyconfig.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2009-2023 Ghent University
+# Copyright 2009-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
@@ -303,7 +303,7 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
     """
     # obtain list of all possible subtoolchains
     _, all_tc_classes = search_toolchain('')
-    subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes)
+    subtoolchains = {tc_class.NAME: getattr(tc_class, 'SUBTOOLCHAIN', None) for tc_class in all_tc_classes}
     optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False))
     composite_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if len(tc_class.__bases__) > 1)
 
@@ -985,7 +985,7 @@ def filter_hidden_deps(self):
         faulty_deps = []
 
         # obtain reference to original lists, so their elements can be changed in place
-        deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']])
+        deps = {key: self.get_ref(key) for key in ('dependencies', 'builddependencies', 'hiddendependencies')}
 
         if 'builddependencies' in self.iterate_options:
             deplists = copy.deepcopy(deps['builddependencies'])
@@ -1229,11 +1229,11 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals
         # templated values should be dumped unresolved
         with self.disable_templating():
             # build dict of default values
-            default_values = dict([(key, DEFAULT_CONFIG[key][0]) for key in DEFAULT_CONFIG])
-            default_values.update(dict([(key, self.extra_options[key][0]) for key in self.extra_options]))
+            default_values = {key: DEFAULT_CONFIG[key][0] for key in DEFAULT_CONFIG}
+            default_values.update({key: self.extra_options[key][0] for key in self.extra_options})
 
             self.generate_template_values()
-            templ_const = dict([(quote_py_str(const[1]), const[0]) for const in TEMPLATE_CONSTANTS])
+            templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS}
 
             # create reverse map of templates, to inject template values where possible
             # longer template values are considered first, shorter template keys get preference over longer ones
@@ -1520,7 +1520,6 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
         # convert tuple to string otherwise python might complain about the formatting
         self.log.debug("Parsing %s as a dependency" % str(dep))
 
-        attr = ['name', 'version', 'versionsuffix', 'toolchain']
         dependency = {
             # full/short module names
             'full_mod_name': None,
@@ -1576,6 +1575,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
                     raise EasyBuildError("Incorrect external dependency specification: %s", dep)
             else:
                 # non-external dependency: tuple (or list) that specifies name/version(/versionsuffix(/toolchain))
+                attr = ('name', 'version', 'versionsuffix', 'toolchain')
                 dependency.update(dict(zip(attr, dep)))
 
         else:
@@ -2049,7 +2049,7 @@ def resolve_template(value, tmpl_dict):
         elif isinstance(value, tuple):
             value = tuple(resolve_template(list(value), tmpl_dict))
         elif isinstance(value, dict):
-            value = dict((resolve_template(k, tmpl_dict), resolve_template(v, tmpl_dict)) for k, v in value.items())
+            value = {resolve_template(k, tmpl_dict): resolve_template(v, tmpl_dict) for k, v in value.items()}
 
     return value
 
@@ -2256,7 +2256,7 @@ def verify_easyconfig_filename(path, specs, parsed_ec=None):
     for ec in ecs:
         found_fullver = det_full_ec_version(ec['ec'])
         if ec['ec']['name'] != specs['name'] or found_fullver != fullver:
-            subspec = dict((key, specs[key]) for key in ['name', 'toolchain', 'version', 'versionsuffix'])
+            subspec = {key: specs[key] for key in ('name', 'toolchain', 'version', 'versionsuffix')}
             error_msg = "Contents of %s does not match with filename" % path
             error_msg += "; expected filename based on contents: %s-%s.eb" % (ec['ec']['name'], found_fullver)
             error_msg += "; expected (relevant) parameters based on filename %s: %s" % (os.path.basename(path), subspec)

easybuild/framework/easyconfig/format/__init__.py

@@ -1,5 +1,5 @@
 # #
-# Copyright 2013-2023 Ghent University
+# Copyright 2013-2024 Ghent University
 #
 # This file is part of EasyBuild,
 # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),