Merge pull request #4077 from easybuilders/4.6.x

release EasyBuild 4.6.1

.github/workflows/bootstrap_script.yml

@@ -3,7 +3,7 @@ name: test EasyBuild bootstrap script
 on: [push, pull_request]
 jobs:
   setup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
         lmod7: Lmod-7.8.22
         lmod8: Lmod-8.7.6
@@ -14,7 +14,7 @@ jobs:
       - run: "true"
   build:
     needs: setup
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         # Don't run for Python 3.8, 3.9 , people should just use `pip install easybuild`
@@ -64,11 +64,11 @@ jobs:
 
     - name: install OS & Python packages
       run: |
-        # disable apt-get update, we don't really need it,
+        # disable apt update, we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
-        # sudo apt-get update
+        # sudo apt update
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then

.github/workflows/container_tests.yml

@@ -3,7 +3,7 @@ name: Tests for container support
 on: [push, pull_request]
 jobs:
   build:
-    # stick to Ubuntu 18.04, where we can still easily install yum via 'apt-get install'
+    # stick to Ubuntu 18.04, where we can still easily install yum via 'apt install'
     runs-on: ubuntu-18.04
     strategy:
       matrix:
@@ -21,12 +21,12 @@ jobs:
     - name: install OS & Python packages
       run: |
         # ensure package list is up to date to avoid 404's for old packages
-        sudo apt-get update -yqq
+        sudo apt update -yqq
         # for building Singularity images
-        sudo apt-get install rpm
-        sudo apt-get install yum
+        sudo apt install rpm
+        sudo apt install yum
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
@@ -49,7 +49,7 @@ jobs:
     - name: install Singularity
       run: |
         # install alien, which can be used to convert RPMs to Debian packages
-        sudo apt-get install alien
+        sudo apt install alien
         alien --version
         # determine latest version of Singularity available in EPEL, and download RPM
         singularity_rpm=$(curl -sL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/s/ | grep singularity | sed 's/.*singularity/singularity/g' | sed 's/rpm.*/rpm/g')

.github/workflows/eb_command.yml

@@ -3,7 +3,7 @@ name: Tests for the 'eb' command
 on: [push, pull_request]
 jobs:
   test-eb:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']
@@ -25,7 +25,7 @@ jobs:
         pip install --upgrade pip
         pip --version
         # install packages required for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then

.github/workflows/linting.yml

@@ -2,7 +2,7 @@ name: Static Analysis
 on: [push, pull_request]
 jobs:
   python-linting:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10']

.github/workflows/unit_tests.yml

@@ -3,7 +3,7 @@ name: EasyBuild framework unit tests
 on: [push, pull_request]
 jobs:
   setup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     outputs:
         lmod7: Lmod-7.8.22
         lmod8: Lmod-8.7.6
@@ -14,7 +14,7 @@ jobs:
       - run: "true"
   build:
     needs: setup
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python: [2.7, 3.6]
@@ -86,20 +86,20 @@ jobs:
 
     - name: install OS & Python packages
       run: |
-        # disable apt-get update, we don't really need it,
+        # disable apt update, we don't really need it,
         # and it does more harm than good (it's fairly expensive, and it results in flaky test runs)
-        # sudo apt-get update
+        # sudo apt update
         # for modules tool
-        sudo apt-get install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
+        sudo apt install lua5.2 liblua5.2-dev lua-filesystem lua-posix tcl tcl-dev
         # fix for lua-posix packaging issue, see https://bugs.launchpad.net/ubuntu/+source/lua-posix/+bug/1752082
         # needed for Ubuntu 18.04, but not for Ubuntu 20.04, so skipping symlinking if posix.so already exists
         if [ ! -e /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so ] ; then
             sudo ln -s /usr/lib/x86_64-linux-gnu/lua/5.2/posix_c.so /usr/lib/x86_64-linux-gnu/lua/5.2/posix.so
         fi
         # for GitPython, python-hglib
-        sudo apt-get install git mercurial
+        sudo apt install git mercurial
         # dep for GC3Pie
-        sudo apt-get install time
+        sudo apt install time
         # Python packages
         pip --version
         pip install --upgrade pip

RELEASE_NOTES

@@ -4,6 +4,29 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.6.1 (September 12th 2022)
+----------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - add script to find dependencies of Python packages (#3839)
+  - add 'ai' default module class (#4053)
+- various bug fixes, including:
+  - fix code style issues reported by recent flake8 linter (#4049)
+  - stick to autopep8 < 1.7.0 with Python 2.7 (#4055)
+  - ensure we call EasyBlock.patch_step for postinstallpatches (#4063)
+  - fix leaked handles in set_columns, complete_cmd, run_cmd_qa, det_terminal_size functions + tests (#4066)
+  - fix `quote_str` when string with both ' and " ends with a double quote (#4068)
+  - fix type-checking of patches to allow dict values + correctly handle patches specified as dict values in --new-pr (#4070)
+  - relax toolchain test by accepting both -march=native (x86_64) and -mcpu=native (aarch64) (#4071)
+- other changes:
+  - run python in the same process as `eb` wrapper script by using `exec` (#4048)
+  - add get_linked_libs_raw function, and use it from both check_linked_shared_libs and sanity_check_rpath (#4051)
+  - update CI workflows (except container tests) to use Ubuntu 20.04, since Ubuntu 18.04 is deprecated (#4064)
+  - use SYSTEM constant for dependency that uses system toolchain in dumped easyconfig (#4069)
+
+
 v4.6.0 (July 8th 2022)
 ----------------------
 

easybuild/base/fancylogger.py

@@ -380,9 +380,9 @@ def streamLog(self, levelno, data):
 
         def write_and_flush_stream(hdlr, data=None):
             """Write to stream and flush the handler"""
-            if (not hasattr(hdlr, 'stream')) or hdlr.stream is None:
+            if getattr(hdlr, 'stream', None) is None:
                 # no stream or not initialised.
-                raise("write_and_flush_stream failed. No active stream attribute.")
+                raise ValueError("write_and_flush_stream failed. No active stream attribute.")
             if data is not None:
                 hdlr.stream.write(data)
                 hdlr.flush()

easybuild/base/generaloption.py

@@ -43,7 +43,7 @@
 
 from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels
 from easybuild.base.optcomplete import autocomplete, CompleterOption
-from easybuild.tools.py2vs3 import StringIO, configparser, string_type
+from easybuild.tools.py2vs3 import StringIO, configparser, string_type, subprocess_popen_text
 from easybuild.tools.utilities import mk_rst_table, nub, shell_quote
 
 try:
@@ -80,7 +80,9 @@ def set_columns(cols=None):
         stty = '/usr/bin/stty'
         if os.path.exists(stty):
             try:
-                cols = int(os.popen('%s size 2>/dev/null' % stty).read().strip().split(' ')[1])
+                with open(os.devnull, 'w') as devnull:
+                    proc = subprocess_popen_text([stty, "size"], stderr=devnull)
+                    cols = int(proc.communicate()[0].strip().split(' ')[1])
             except (AttributeError, IndexError, OSError, ValueError):
                 # do nothing
                 pass

easybuild/framework/easyblock.py

@@ -96,7 +96,8 @@
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
-from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group
+from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw
+from easybuild.tools.systemtools import get_shared_lib_ext, use_group
 from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
@@ -973,7 +974,7 @@ def gen_builddir(self):
         if not self.cfg.get('cleanupoldbuild', False):
             uniq_builddir = builddir
             suff = 0
-            while(os.path.isdir(uniq_builddir)):
+            while os.path.isdir(uniq_builddir):
                 uniq_builddir = "%s.%d" % (builddir, suff)
                 suff += 1
             builddir = uniq_builddir
@@ -2875,7 +2876,9 @@ def apply_post_install_patches(self, patches=None):
 
         self.log.debug("Post-install patches to apply: %s", patches)
         if patches:
-            self.patch_step(beginpath=self.installdir, patches=patches)
+            # self may be inherited from the Bundle easyblock and that patch_step is a no-op
+            # To allow postinstallpatches for Bundle, and derived, easyblocks we directly call EasyBlock.patch_step
+            EasyBlock.patch_step(self, beginpath=self.installdir, patches=patches)
 
     def post_install_step(self):
         """
@@ -2994,24 +2997,15 @@ def sanity_check_rpath(self, rpath_dirs=None):
                 for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]:
                     self.log.debug("Sanity checking RPATH for %s", path)
 
-                    out, ec = run_cmd("file %s" % path, simple=False, trace=False)
-                    if ec:
-                        fail_msg = "Failed to run 'file %s': %s" % (path, out)
-                        self.log.warning(fail_msg)
-                        fails.append(fail_msg)
-
-                    # only run ldd/readelf on dynamically linked executables/libraries
-                    # example output:
-                    # ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), ...
-                    # ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped
-                    if "dynamically linked" in out:
+                    out = get_linked_libs_raw(path)
+
+                    if out is None:
+                        msg = "Failed to determine dynamically linked libraries for %s, "
+                        msg += "so skipping it in RPATH sanity check"
+                        self.log.debug(msg, path)
+                    else:
                         # check whether all required libraries are found via 'ldd'
-                        out, ec = run_cmd("ldd %s" % path, simple=False, trace=False)
-                        if ec:
-                            fail_msg = "Failed to run 'ldd %s': %s" % (path, out)
-                            self.log.warning(fail_msg)
-                            fails.append(fail_msg)
-                        elif not_found_regex.search(out):
+                        if not_found_regex.search(out):
                             fail_msg = "One or more required libraries not found for %s: %s" % (path, out)
                             self.log.warning(fail_msg)
                             fails.append(fail_msg)
@@ -3030,9 +3024,6 @@ def sanity_check_rpath(self, rpath_dirs=None):
                             fails.append(fail_msg)
                         else:
                             self.log.debug("Output of 'readelf -d %s' checked, looks OK", path)
-
-                    else:
-                        self.log.debug("%s is not dynamically linked, so skipping it in RPATH sanity check", path)
             else:
                 self.log.debug("Not sanity checking files in non-existing directory %s", dirpath)
 

easybuild/framework/easyconfig/easyconfig.py

@@ -50,7 +50,7 @@
 import easybuild.tools.filetools as filetools
 from easybuild.base import fancylogger
 from easybuild.framework.easyconfig import MANDATORY
-from easybuild.framework.easyconfig.constants import EXTERNAL_MODULE_MARKER
+from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS, EXTERNAL_MODULE_MARKER
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
 from easybuild.framework.easyconfig.format.convert import Dependency
 from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS
@@ -1589,7 +1589,7 @@ def _parse_dependency(self, dep, hidden=False, build_only=False):
 
         # (true) boolean value simply indicates that a system toolchain is used
         elif isinstance(tc_spec, bool) and tc_spec:
-            tc = {'name': SYSTEM_TOOLCHAIN_NAME, 'version': ''}
+            tc = EASYCONFIG_CONSTANTS['SYSTEM'][0]
 
         # two-element list/tuple value indicates custom toolchain specification
         elif isinstance(tc_spec, (list, tuple,)):

easybuild/framework/easyconfig/format/one.py

@@ -76,16 +76,19 @@ def dump_dependency(dep, toolchain, toolchain_hierarchy=None):
     else:
         # minimal spec: (name, version)
         tup = (dep['name'], dep['version'])
+        res = None
         if all(dep['toolchain'] != subtoolchain for subtoolchain in toolchain_hierarchy):
             if dep[SYSTEM_TOOLCHAIN_NAME]:
-                tup += (dep['versionsuffix'], True)
+                # use SYSTEM constant to indicate that system toolchain should be used for this dependency
+                res = re.sub(r'\)$', ', SYSTEM)', str(tup + (dep['versionsuffix'],)))
             else:
                 tup += (dep['versionsuffix'], (dep['toolchain']['name'], dep['toolchain']['version']))
 
         elif dep['versionsuffix']:
             tup += (dep['versionsuffix'],)
 
-        res = str(tup)
+        if res is None:
+            res = str(tup)
     return res
 
 

easybuild/framework/easyconfig/format/version.py

@@ -117,7 +117,7 @@ def __bool__(self):
 
     def is_valid(self):
         """Check if this is a valid VersionOperator. Suffix can be anything."""
-        return not(self.version is None or self.operator is None)
+        return not (self.version is None or self.operator is None)
 
     def set(self, versop_str):
         """

easybuild/framework/easyconfig/types.py

@@ -349,6 +349,32 @@ def to_list_of_strings_and_tuples(spec):
     return str_tup_list
 
 
+def to_list_of_strings_and_tuples_and_dicts(spec):
+    """
+    Convert a 'list of dicts and tuples/lists and strings' to a 'list of dicts and tuples and strings'
+
+    Example:
+        ['foo', ['bar', 'baz']]
+        to
+        ['foo', ('bar', 'baz')]
+    """
+    str_tup_list = []
+
+    if not isinstance(spec, (list, tuple)):
+        raise EasyBuildError("Expected value to be a list, found %s (%s)", spec, type(spec))
+
+    for elem in spec:
+        if isinstance(elem, (string_type, tuple, dict)):
+            str_tup_list.append(elem)
+        elif isinstance(elem, list):
+            str_tup_list.append(tuple(elem))
+        else:
+            raise EasyBuildError("Expected elements to be of type string, tuple, dict or list, got %s (%s)",
+                                 elem, type(elem))
+
+    return str_tup_list
+
+
 def to_sanity_check_paths_dict(spec):
     """
     Convert a sanity_check_paths dict as received by yaml (a dict with list values that contain either lists or strings)
@@ -526,6 +552,7 @@ def ensure_iterable_license_specs(specs):
         'key_types': [str],
     }
 ))
+STRING_OR_TUPLE_OR_DICT_LIST = (list, as_hashable({'elem_types': [str, TUPLE_OF_STRINGS, STRING_DICT]}))
 SANITY_CHECK_PATHS_DICT = (dict, as_hashable({
     'elem_types': {
         SANITY_CHECK_PATHS_FILES: [STRING_OR_TUPLE_LIST],
@@ -544,7 +571,8 @@ def ensure_iterable_license_specs(specs):
 CHECKSUMS = (list, as_hashable({'elem_types': [str, tuple, STRING_DICT, CHECKSUM_LIST]}))
 
 CHECKABLE_TYPES = [CHECKSUM_LIST, CHECKSUMS, DEPENDENCIES, DEPENDENCY_DICT, LIST_OF_STRINGS,
-                   SANITY_CHECK_PATHS_DICT, STRING_DICT, STRING_OR_TUPLE_LIST, TOOLCHAIN_DICT, TUPLE_OF_STRINGS]
+                   SANITY_CHECK_PATHS_DICT, STRING_DICT, STRING_OR_TUPLE_LIST, STRING_OR_TUPLE_OR_DICT_LIST,
+                   TOOLCHAIN_DICT, TUPLE_OF_STRINGS]
 
 # easy types, that can be verified with isinstance
 EASY_TYPES = [string_type, bool, dict, int, list, str, tuple]
@@ -555,7 +583,7 @@ def ensure_iterable_license_specs(specs):
     'docurls': LIST_OF_STRINGS,
     'name': string_type,
     'osdependencies': STRING_OR_TUPLE_LIST,
-    'patches': STRING_OR_TUPLE_LIST,
+    'patches': STRING_OR_TUPLE_OR_DICT_LIST,
     'sanity_check_paths': SANITY_CHECK_PATHS_DICT,
     'toolchain': TOOLCHAIN_DICT,
     'version': string_type,
@@ -575,4 +603,5 @@ def ensure_iterable_license_specs(specs):
     TOOLCHAIN_DICT: to_toolchain_dict,
     SANITY_CHECK_PATHS_DICT: to_sanity_check_paths_dict,
     STRING_OR_TUPLE_LIST: to_list_of_strings_and_tuples,
+    STRING_OR_TUPLE_OR_DICT_LIST: to_list_of_strings_and_tuples_and_dicts,
 }