Skip to content

Commit

Permalink
Merge pull request #4294 from easybuilders/4.8.x
Browse files Browse the repository at this point in the history
release EasyBuild v4.8.0
  • Loading branch information
boegel committed Jul 7, 2023
2 parents 21d8ebc + 6e245e5 commit e082752
Show file tree
Hide file tree
Showing 19 changed files with 308 additions and 65 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/container_tests.yml
Expand Up @@ -14,13 +14,13 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
python: [2.7, 3.7]
python: [3.7]
fail-fast: false
steps:
- uses: actions/checkout@v3

- name: set up Python
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: ${{matrix.python}}
architecture: x64
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/container_tests_apptainer.yml
Expand Up @@ -14,14 +14,14 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
python: [2.7, 3.7]
python: [3.7]
apptainer: [1.0.0, 1.1.7]
fail-fast: false
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3

- name: set up Python
uses: actions/setup-python@v2
uses: actions/setup-python@v3
with:
python-version: ${{matrix.python}}
architecture: x64
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/eb_command.yml
Expand Up @@ -14,13 +14,13 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
fail-fast: false
steps:
- uses: actions/checkout@v3

- name: set up Python
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: ${{matrix.python}}
architecture: x64
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/linting.yml
Expand Up @@ -13,13 +13,13 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']

steps:
- uses: actions/checkout@v3

- name: set up Python
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}

Expand Down
8 changes: 2 additions & 6 deletions .github/workflows/unit_tests.yml
Expand Up @@ -25,7 +25,7 @@ jobs:
runs-on: ubuntu-20.04
strategy:
matrix:
python: [2.7, 3.6]
python: [3.6]
modules_tool:
# use variables defined by 'setup' job above, see also
# https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
Expand Down Expand Up @@ -81,7 +81,7 @@ jobs:
- uses: actions/checkout@v3

- name: set up Python
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: ${{matrix.python}}
architecture: x64
Expand Down Expand Up @@ -207,10 +207,6 @@ jobs:
# create file owned by root but writable by anyone (used by test_copy_file)
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
# silence deprecation warning when using Python 2, since it breaks a bunch of tests
if [ "${{matrix.python}}" == '2.7' ]; then
export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2
fi
# run test suite
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
# try and make sure output of running tests is clean (no printed messages/warnings)
Expand Down
16 changes: 16 additions & 0 deletions RELEASE_NOTES
Expand Up @@ -4,6 +4,22 @@ For more detailed information, please see the git log.
These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.


v4.8.0 (7 July 2023)
--------------------

feature release

- various enhancements, including:
- use version suffixes in PR title instead of Python versions when using --new-pr (#4253 + #4286)
- add script to find updated ECs for installed module (#4271)
- various bug fixes, including:
- fix regex for extracting glibc version from output of '`ldd --version`' in Gentoo Linux (#4290)
- allow versionsuffix to be set to None in `det_full_ec_version` (#4292)
- other changes:
- stop running tests with Python 2.7 since it is no longer supported in GitHub Action (#4267)
- replace imp.load_source by leveraging importlib.util.module_from_spec when using Python 3 (#4280)


v4.7.2 (27 May 2023)
--------------------

Expand Down
130 changes: 130 additions & 0 deletions easybuild/scripts/findUpdatedEcs.sh
@@ -0,0 +1,130 @@
#!/usr/bin/env bash

set -euo pipefail

GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[0;33m'
NC='\033[0m'

function printError {
echo -e "${RED}$@${NC}"
}

verbose=0
function printVerbose {
(( verbose == 0 )) || echo "$@"
}

function checkModule {
moduleFolder="$1"
moduleName="$(basename "$(dirname "$moduleFolder")")"
moduleVersion="$(basename "$moduleFolder")"
moduleStr="$moduleName/$moduleVersion"
printVerbose "Processing $moduleStr"
ec_glob=( "$moduleFolder/easybuild/"*.eb )
if [[ ! -e "${ec_glob[@]}" ]]; then
printError "=== Did not find installed EC for $moduleStr"
return
fi
ec_installed="$ec_glob"
ec_filename=$(basename "$ec_installed")
# Try with most likely location first for speed
first_letter=${ec_filename:0:1}
letterPath=$easyconfigFolder/${first_letter,,}
if [[ -d "$letterPath" ]]; then
ec_new="$(find "$letterPath" -type f -name "$ec_filename")"
else
ec_new=
fi
# Fallback if not found
[[ -n "$ec_new" ]] || ec_new="$(find "$easyconfigFolder" -type f -name "$ec_filename")"
if [[ -z "$ec_new" ]]; then
printError "=== Did not find new EC $ec_filename"
elif [[ ! -e "$ec_new" ]]; then
printError "=== Found multiple new ECs: $ec_new"
elif ! out=$(diff "$ec_installed" "$ec_new"); then
echo -e "${YELLOW}=== Needs updating: ${GREEN}${ec_installed}${YELLOW} vs ${GREEN}${ec_new}${NC}"
if ((showDiff == 1)); then
echo "$out"
fi
fi
}

ecDefaultFolder=
if path=$(which eb 2>/dev/null); then
path=$(dirname "$path")
for p in "$path" "$(dirname "$path")"; do
if [ -d "$p/easybuild/easyconfigs" ]; then
ecDefaultFolder=$p
break
fi
done
fi

function usage {
echo "Usage: $(basename "$0") [--verbose] [--diff] --loaded|--modules INSTALLPATH --easyconfigs EC-FOLDER"
echo
echo "Check installed modules against the source EasyConfig (EC) files to determine which have changed."
echo "Can either check the currently loaded modules or all modules installed in a specific location"
echo
echo "--verbose Verbose status output while checking"
echo "--loaded Check only currently loaded modules"
echo "--diff Show diff of changed module files"
echo "--modules INSTALLPATH Check all modules in the specified (software) installpath, i.e. the root of module-binaries"
echo "--easyconfigs EC-FOLDER Path to the folder containg the current/updated EasyConfigs. ${ecDefaultFolder:+Defaults to $ecDefaultFolder}"
exit 0
}

checkLoadedModules=0
showDiff=0
modulesFolder=""
easyconfigFolder=$ecDefaultFolder

while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help)
usage;;
-v|--verbose)
verbose=1;;
-d|--diff)
showDiff=1;;
-l|--loaded)
checkLoadedModules=1;;
-m|--modules)
modulesFolder="$2"
shift;;
-e|--easyconfigs)
easyconfigFolder="$2"
shift;;
*)
printError "Unknown argument: $1"
exit 1;;
esac
shift
done

if [ -z "$easyconfigFolder" ]; then
printError "Folder to easyconfigs not given!" && exit 1
fi
if [ -z "$modulesFolder" ]; then
if (( checkLoadedModules == 0 )); then
printError "Need either --modules or --loaded to specify what to check!" && exit 1
fi
elif (( checkLoadedModules == 1 )); then
printError "Cannot specify --modules and --loaded!" && exit 1
fi

if [ -d "$easyconfigFolder/easybuild/easyconfigs" ]; then
easyconfigFolder="$easyconfigFolder/easybuild/easyconfigs"
fi

if (( checkLoadedModules == 1 )); then
for varname in $(compgen -A variable | grep '^EBROOT'); do
checkModule "${!varname}"
done
else
for module in "$modulesFolder"/*/*/easybuild; do
checkModule "$(dirname "$module")"
done
fi
5 changes: 2 additions & 3 deletions easybuild/tools/filetools.py
Expand Up @@ -44,7 +44,6 @@
import difflib
import glob
import hashlib
import imp
import inspect
import itertools
import os
Expand All @@ -66,7 +65,7 @@
from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
from easybuild.tools.config import build_option, install_path
from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar
from easybuild.tools.py2vs3 import HTMLParser, std_urllib, string_type
from easybuild.tools.py2vs3 import HTMLParser, load_source, std_urllib, string_type
from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg

try:
Expand Down Expand Up @@ -2801,7 +2800,7 @@ def install_fake_vsc():
def get_easyblock_class_name(path):
"""Make sure file is an easyblock and get easyblock class name"""
fn = os.path.basename(path).split('.')[0]
mod = imp.load_source(fn, path)
mod = load_source(fn, path)
clsmembers = inspect.getmembers(mod, inspect.isclass)
for cn, co in clsmembers:
if co.__module__ == mod.__name__:
Expand Down
73 changes: 40 additions & 33 deletions easybuild/tools/github.py
Expand Up @@ -1581,6 +1581,45 @@ def new_branch_github(paths, ecs, commit_msg=None):
return res


def det_pr_title(ecs):
"""
Create title for PR based on first easyconfigs
:param ecs: list of parsed easyconfigs
"""

# only use most common toolchain(s) in toolchain label of PR title
toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in ecs]
toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)])
toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]])

# only use most common module class(es) in moduleclass label of PR title
classes = [ec['moduleclass'] for ec in ecs]
classes_counted = sorted([(classes.count(c), c) for c in nub(classes)])
class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]])

names_and_versions = nub(["%s v%s" % (ec.name, ec.version) for ec in ecs])
if len(names_and_versions) <= 3:
main_title = ', '.join(names_and_versions)
else:
main_title = ', '.join(names_and_versions[:3] + ['...'])

title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title)

# Find all suffixes
suffixes = []
for ec in ecs:
if 'versionsuffix' in ec and ec['versionsuffix']:
suffixes.append(ec['versionsuffix'].strip('-').replace('-', ' '))
if suffixes:
suffixes = sorted(nub(suffixes))
if len(suffixes) <= 2:
title += ' w/ ' + ', '.join(suffixes)
else:
title += ' w/ ' + ', '.join(suffixes[:2] + ['...'])

return title


@only_if_module_is_available('git', pkgname='GitPython')
def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, pr_metadata=None, commit_msg=None):
"""
Expand Down Expand Up @@ -1691,42 +1730,10 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None,

labels = det_pr_labels(file_info, pr_target_repo)

if pr_target_repo == GITHUB_EASYCONFIGS_REPO:
# only use most common toolchain(s) in toolchain label of PR title
toolchains = ['%(name)s/%(version)s' % ec['toolchain'] for ec in file_info['ecs']]
toolchains_counted = sorted([(toolchains.count(tc), tc) for tc in nub(toolchains)])
toolchain_label = ','.join([tc for (cnt, tc) in toolchains_counted if cnt == toolchains_counted[-1][0]])

# only use most common module class(es) in moduleclass label of PR title
classes = [ec['moduleclass'] for ec in file_info['ecs']]
classes_counted = sorted([(classes.count(c), c) for c in nub(classes)])
class_label = ','.join([tc for (cnt, tc) in classes_counted if cnt == classes_counted[-1][0]])

if title is None:
if pr_target_repo == GITHUB_EASYCONFIGS_REPO:
if file_info['ecs'] and all(file_info['new']) and not deleted_paths:
# mention software name/version in PR title (only first 3)
names_and_versions = nub(["%s v%s" % (ec.name, ec.version) for ec in file_info['ecs']])
if len(names_and_versions) <= 3:
main_title = ', '.join(names_and_versions)
else:
main_title = ', '.join(names_and_versions[:3] + ['...'])

title = "{%s}[%s] %s" % (class_label, toolchain_label, main_title)

# if Python is listed as a dependency, then mention Python version(s) in PR title
pyver = []
for ec in file_info['ecs']:
# iterate over all dependencies (incl. build dependencies & multi-deps)
for dep in ec.dependencies():
if dep['name'] == 'Python':
# check whether Python is listed as a multi-dep if it's marked as a build dependency
if dep['build_only'] and 'Python' not in ec['multi_deps']:
continue
else:
pyver.append(dep['version'])
if pyver:
title += " w/ Python %s" % ' + '.join(sorted(nub(pyver)))
title = det_pr_title(file_info['ecs'])
elif pr_target_repo == GITHUB_EASYBLOCKS_REPO:
if file_info['eb_names'] and all(file_info['new']) and not deleted_paths:
plural = 's' if len(file_info['eb_names']) > 1 else ''
Expand Down
4 changes: 2 additions & 2 deletions easybuild/tools/hooks.py
Expand Up @@ -30,10 +30,10 @@
* Kenneth Hoste (Ghent University)
"""
import difflib
import imp
import os

from easybuild.base import fancylogger
from easybuild.tools.py2vs3 import load_source
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option

Expand Down Expand Up @@ -123,7 +123,7 @@ def load_hooks(hooks_path):
_log.info("Importing hooks implementation from %s...", hooks_path)
try:
# import module that defines hooks, and collect all functions of which name ends with '_hook'
imported_hooks = imp.load_source(hooks_filename, hooks_path)
imported_hooks = load_source(hooks_filename, hooks_path)
for attr in dir(imported_hooks):
if attr.endswith(HOOK_SUFF):
hook = getattr(imported_hooks, attr)
Expand Down

0 comments on commit e082752

Please sign in to comment.