armbian-desktop/aggregation: introduce DESKTOP_COMMON aggregati…

…on variant, where arch/config/appgroups are ignored; see armbian#5439 (re-submitted)

- this has been tested with complete/default targets and works

lib/functions/artifacts/artifact-armbian-desktop.sh

@@ -8,20 +8,15 @@
 # https://github.com/armbian/build/
 
 function artifact_armbian-desktop_config_dump() {
+	# Those are what keys the package name.
 	artifact_input_variables[RELEASE]="${RELEASE}"
 	artifact_input_variables[DESKTOP_ENVIRONMENT]="${DESKTOP_ENVIRONMENT}"
-
-	# Include a hash of the results of aggregation.
-	declare aggregation_hash="undetermined"
-	aggregation_hash="$(echo "${AGGREGATED_DESKTOP_POSTINST} ${AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE} ${AGGREGATED_PACKAGES_DESKTOP_COMMA}" | sha256sum | cut -d' ' -f1)"
-	artifact_input_variables[DESKTOP_AGGREGATION_RESULTS]="${aggregation_hash}"
 }
 
 function artifact_armbian-desktop_prepare_version() {
 	: "${RELEASE:?RELEASE is not set}"
 
 	: "${DESKTOP_ENVIRONMENT:?DESKTOP_ENVIRONMENT is not set}"
-	: "${DESKTOP_ENVIRONMENT_CONFIG_NAME:?DESKTOP_ENVIRONMENT_CONFIG_NAME is not set}" # Not keyed, but required.
 
 	artifact_version="undetermined"        # outer scope
 	artifact_version_reason="undetermined" # outer scope
@@ -32,9 +27,9 @@ function artifact_armbian-desktop_prepare_version() {
 
 	# Hash variables that affect the contents of desktop package
 	declare -a vars_to_hash=(
-		"${AGGREGATED_DESKTOP_POSTINST}"
-		"${AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE}"
-		"${AGGREGATED_PACKAGES_DESKTOP_COMMA}"
+		"${AGGREGATED_DESKTOP_COMMON_POSTINST}"
+		"${AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE}"
+		"${AGGREGATED_PACKAGES_DESKTOP_COMMON_COMMA}"
 	)
 	declare hash_variables="undetermined"                                        # will be set by calculate_hash_for_variables()...
 	do_normalize_src_path="no" calculate_hash_for_variables "${vars_to_hash[@]}" # ... where do_normalize_src_path="yes" is the default
@@ -51,7 +46,7 @@ function artifact_armbian-desktop_prepare_version() {
 
 	declare -a reasons=(
 		"Armbian armbian-desktop"
-		"vars hash \"${vars_config_hash}\""
+		"vars hash \"${hash_variables}\""
 		"framework bash hash \"${bash_hash}\""
 	)
 
@@ -81,7 +76,6 @@ function artifact_armbian-desktop_cli_adapter_pre_run() {
 function artifact_armbian-desktop_cli_adapter_config_prep() {
 	: "${RELEASE:?RELEASE is not set}"
 	: "${DESKTOP_ENVIRONMENT:?DESKTOP_ENVIRONMENT is not set}"
-	: "${DESKTOP_ENVIRONMENT_CONFIG_NAME:?DESKTOP_ENVIRONMENT_CONFIG_NAME is not set}"
 
 	# this requires aggregation, and thus RELEASE, but also everything else.
 	declare -g artifact_version_requires_aggregation="yes"

lib/functions/compilation/packages/armbian-desktop-deb.sh

@@ -17,16 +17,15 @@ function compile_armbian-desktop() {
 	: "${RELEASE:?RELEASE is not set}"
 	: "${DISTRIBUTION:?DISTRIBUTION is not set}"
 	: "${DESKTOP_ENVIRONMENT:?DESKTOP_ENVIRONMENT is not set}"
-	: "${DESKTOP_ENVIRONMENT_CONFIG_NAME:?DESKTOP_ENVIRONMENT_CONFIG_NAME is not set}"
 
 	assert_requires_aggregation # this requires aggregation to have been run
-	: "${AGGREGATED_DESKTOP_POSTINST:?AGGREGATED_DESKTOP_POSTINST is not set}"
-	: "${AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE:?AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE is not set}"
-	: "${AGGREGATED_PACKAGES_DESKTOP_COMMA:?AGGREGATED_PACKAGES_DESKTOP_COMMA is not set}"
+	# We use the "DESKTOP_COMMON" aggregation results: it does not vary per-arch, nor per-appgroups, and config is always config_base.
+	: "${AGGREGATED_DESKTOP_COMMON_POSTINST:?AGGREGATED_DESKTOP_COMMON_POSTINST is not set}"
+	: "${AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE:?AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE is not set}"
+	: "${AGGREGATED_PACKAGES_DESKTOP_COMMON_COMMA:?AGGREGATED_PACKAGES_DESKTOP_COMMON_COMMA is not set}"
 
 	# produced by aggregation.py
-	display_alert "bsp-desktop: AGGREGATED_PACKAGES_DESKTOP_COMMA" "'${AGGREGATED_PACKAGES_DESKTOP_COMMA}'" "debug"
-	# @TODO: AGGREGATED_PACKAGES_DESKTOP_COMMA includes appgroups, which can vary.
+	display_alert "bsp-desktop: AGGREGATED_PACKAGES_DESKTOP_COMMON_COMMA" "'${AGGREGATED_PACKAGES_DESKTOP_COMMON_COMMA}'" "debug"
 
 	display_alert "Creating common package for '${DESKTOP_ENVIRONMENT}' desktops" "${artifact_name} :: ${artifact_version}" "info"
 
@@ -44,19 +43,19 @@ function compile_armbian-desktop() {
 		Maintainer: $MAINTAINER <$MAINTAINERMAIL>
 		Section: xorg
 		Priority: optional
-		Recommends: ${AGGREGATED_PACKAGES_DESKTOP_COMMA}, armbian-bsp-desktop
+		Recommends: ${AGGREGATED_PACKAGES_DESKTOP_COMMON_COMMA}, armbian-bsp-desktop
 		Provides: armbian-${RELEASE}-desktop
 		Conflicts: gdm3
 		Description: Armbian desktop for ${DISTRIBUTION} ${RELEASE} ${DESKTOP_ENVIRONMENT}
 	EOF
 
 	# postinst. generated script, gathered from scripts in files in configuration. # @TODO: extensions could do this much better
-	generic_artifact_package_hook_helper "postinst" "${AGGREGATED_DESKTOP_POSTINST}"
+	generic_artifact_package_hook_helper "postinst" "${AGGREGATED_DESKTOP_COMMON_POSTINST}"
 
 	# @TODO: error information? This is very likely to explode, and a bad implementation of extensibility.
-	display_alert "Running desktop-specific aggregated prepare script" "AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE" "debug"
-	eval "${AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE}"
-	display_alert "Running desktop-specific aggregated prepare script" "AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE" "debug"
+	display_alert "Running desktop-specific aggregated prepare script" "AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE" "debug"
+	eval "${AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE}"
+	display_alert "Running desktop-specific aggregated prepare script" "AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE" "debug"
 
 	fakeroot_dpkg_deb_build "${destination}" "armbian-desktop"
 

lib/tools/aggregation.py

@@ -48,20 +48,31 @@
 # Show the environment
 armbian_utils.show_incoming_environment()
 
+COMMON_FIXED_DESKTOP_ENVIRONMENT_CONFIG_NAME = "config_base"
+
 util.SELECTED_CONFIGURATION = armbian_utils.get_from_env_or_bomb("SELECTED_CONFIGURATION")  # "cli_standard"
 util.DESKTOP_APPGROUPS_SELECTED = armbian_utils.parse_env_for_tokens("DESKTOP_APPGROUPS_SELECTED")  # ["browsers", "chat"]
 util.SRC = armbian_build_directory
 
+# All the search roots, including ARCH-varying ones.
 util.AGGREGATION_SEARCH_ROOT_ABSOLUTE_DIRS = [
 	f"{armbian_build_directory}/config",
 	f"{armbian_build_directory}/config/optional/_any_board/_config",
 	f"{armbian_build_directory}/config/optional/architectures/{ARCH}/_config",
 	f"{USERPATCHES_PATH}"
 ]
 
+# Only the common, non-arch-varying search roots.
+util.AGGREGATION_SEARCH_ROOT_ABSOLUTE_DIRS_COMMON = [
+	f"{armbian_build_directory}/config",
+	f"{armbian_build_directory}/config/optional/_any_board/_config",
+	f"{USERPATCHES_PATH}"
+]
+
 util.DEBOOTSTRAP_SEARCH_RELATIVE_DIRS = ["cli/_all_distributions/debootstrap", f"cli/{RELEASE}/debootstrap"]
 util.CLI_SEARCH_RELATIVE_DIRS = ["cli/_all_distributions/main", f"cli/{RELEASE}/main"]
 
+# The complete, arch/config_name varying desktop relative directories.
 util.DESKTOP_ENVIRONMENTS_SEARCH_RELATIVE_DIRS = [
 	f"desktop/_all_distributions/environments/_all_environments",
 	f"desktop/_all_distributions/environments/{DESKTOP_ENVIRONMENT}",
@@ -70,6 +81,18 @@
 	f"desktop/{RELEASE}/environments/{DESKTOP_ENVIRONMENT}",
 	f"desktop/{RELEASE}/environments/{DESKTOP_ENVIRONMENT}/{DESKTOP_ENVIRONMENT_CONFIG_NAME}"]
 
+# Only the common, non-arch, fixed-config-name desktop relative directories.
+util.DESKTOP_ENVIRONMENTS_SEARCH_RELATIVE_DIRS_COMMON = [
+	f"desktop/_all_distributions/environments/_all_environments",
+	f"desktop/_all_distributions/environments/{DESKTOP_ENVIRONMENT}",
+	f"desktop/_all_distributions/environments/{DESKTOP_ENVIRONMENT}/{DESKTOP_ENVIRONMENT_CONFIG_NAME}",
+	f"desktop/{RELEASE}/environments/_all_environments",
+	f"desktop/{RELEASE}/environments/{DESKTOP_ENVIRONMENT}",
+	f"desktop/{RELEASE}/environments/{DESKTOP_ENVIRONMENT}/{COMMON_FIXED_DESKTOP_ENVIRONMENT_CONFIG_NAME}"
+	# Attention: fixed "config_base" for common version
+]
+
+# This will _not_ be included in the common version.
 util.DESKTOP_APPGROUPS_SEARCH_RELATIVE_DIRS = [
 	f"desktop/_all_distributions/appgroups",
 	f"desktop/_all_distributions/environments/{DESKTOP_ENVIRONMENT}/appgroups",
@@ -90,14 +113,22 @@
 	rootfs_packages_additional = util.aggregate_all_cli("packages.additional")
 	rootfs_packages_all = util.merge_lists(rootfs_packages_all, rootfs_packages_additional, "add")
 
-# Desktop environment packages; packages + packages.external
+# Desktop environment packages; packages + packages.external (varying per arch, config_name, and appgroups)
 desktop_packages_main = util.aggregate_all_desktop("packages")
 desktop_packages_external = util.aggregate_all_desktop("packages.external")
 desktop_packages_additional = util.aggregate_all_desktop("packages.additional")
 desktop_packages_all = util.merge_lists(desktop_packages_main, desktop_packages_external, "add")
 desktop_packages_all = util.merge_lists(desktop_packages_all, desktop_packages_additional, "add")
 desktop_packages_remove = util.aggregate_all_desktop("packages.remove")
 
+# Common desktop environment packages; packages + packages.external (non-arch, fixed config_name, and non-appgroups)
+desktop_common_packages_main = util.aggregate_all_desktop_common("packages")
+desktop_common_packages_external = util.aggregate_all_desktop_common("packages.external")
+desktop_common_packages_additional = util.aggregate_all_desktop_common("packages.additional")
+desktop_common_packages_all = util.merge_lists(desktop_common_packages_main, desktop_common_packages_external, "add")
+desktop_common_packages_all = util.merge_lists(desktop_common_packages_all, desktop_common_packages_additional, "add")
+desktop_common_packages_remove = util.aggregate_all_desktop_common("packages.remove")
+
 env_list_remove = util.parse_env_for_list("REMOVE_PACKAGES")
 env_list_extra_rootfs = util.parse_env_for_list("EXTRA_PACKAGES_ROOTFS")
 env_list_extra_image = util.parse_env_for_list("EXTRA_PACKAGES_IMAGE")
@@ -133,6 +164,9 @@
 AGGREGATED_PACKAGES_DESKTOP = util.merge_lists(AGGREGATED_PACKAGES_DESKTOP, env_package_list_family_remove, "remove")
 AGGREGATED_PACKAGES_DESKTOP = util.merge_lists(AGGREGATED_PACKAGES_DESKTOP, env_list_remove, "remove")
 
+# The desktop (common: non-arch, non-appgroups, fixed config_name) list. This is NOT affected by env's.
+AGGREGATED_PACKAGES_DESKTOP_COMMON = util.merge_lists(desktop_common_packages_all, desktop_common_packages_remove, "remove")
+
 # the image list; this comes from env only; apply the removals.
 AGGREGATED_PACKAGES_IMAGE = util.merge_lists(env_list_extra_image, env_package_list_board, "add")
 AGGREGATED_PACKAGES_IMAGE = util.merge_lists(AGGREGATED_PACKAGES_IMAGE, env_package_list_family, "add")
@@ -157,14 +191,22 @@
 
 # We need to aggregate some desktop stuff, which are not package lists, postinst contents and such.
 # For this case just find the potentials, and for each found, take the whole contents and join via newlines.
-AGGREGATED_DESKTOP_POSTINST = util.aggregate_all_desktop(
-	"debian/postinst", util.aggregate_simple_contents_potential)
-AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE = util.aggregate_all_desktop(
-	"armbian/create_desktop_package.sh", util.aggregate_simple_contents_potential)
-AGGREGATED_DESKTOP_BSP_POSTINST = util.aggregate_all_desktop(
-	"debian/armbian-bsp-desktop/postinst", util.aggregate_simple_contents_potential)
-AGGREGATED_DESKTOP_BSP_PREPARE = util.aggregate_all_desktop(
-	"debian/armbian-bsp-desktop/prepare.sh", util.aggregate_simple_contents_potential)
+AGGREGATED_DESKTOP_POSTINST = util.aggregate_all_desktop("debian/postinst", util.aggregate_simple_contents_potential)
+AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE = util.aggregate_all_desktop("armbian/create_desktop_package.sh", util.aggregate_simple_contents_potential)
+AGGREGATED_DESKTOP_BSP_POSTINST = util.aggregate_all_desktop("debian/armbian-bsp-desktop/postinst", util.aggregate_simple_contents_potential)
+AGGREGATED_DESKTOP_BSP_PREPARE = util.aggregate_all_desktop("debian/armbian-bsp-desktop/prepare.sh", util.aggregate_simple_contents_potential)
+
+# Common (non-arch, non-appgroups, fixed config_name) version of the above.
+AGGREGATED_DESKTOP_COMMON_POSTINST = util.aggregate_all_desktop_common("debian/postinst", util.aggregate_simple_contents_potential)
+AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE = util.aggregate_all_desktop_common(
+	"armbian/create_desktop_package.sh",
+	util.aggregate_simple_contents_potential)
+AGGREGATED_DESKTOP_COMMON_BSP_POSTINST = util.aggregate_all_desktop_common(
+	"debian/armbian-bsp-desktop/postinst",
+	util.aggregate_simple_contents_potential)
+AGGREGATED_DESKTOP_COMMON_BSP_PREPARE = util.aggregate_all_desktop_common(
+	"debian/armbian-bsp-desktop/prepare.sh",
+	util.aggregate_simple_contents_potential)
 
 # Aggregate the apt-sources; only done if BUILD_DESKTOP is True, otherwise empty.
 AGGREGATED_APT_SOURCES = {}
@@ -183,6 +225,7 @@
 	("rootfs", "AGGREGATED_PACKAGES_ROOTFS", AGGREGATED_PACKAGES_ROOTFS, None),
 	("image", "AGGREGATED_PACKAGES_IMAGE", AGGREGATED_PACKAGES_IMAGE, None),
 	("desktop", "AGGREGATED_PACKAGES_DESKTOP", AGGREGATED_PACKAGES_DESKTOP, None),
+	("desktop-common", "AGGREGATED_PACKAGES_DESKTOP_COMMON", AGGREGATED_PACKAGES_DESKTOP_COMMON, None),
 	("apt-sources", "AGGREGATED_APT_SOURCES", AGGREGATED_APT_SOURCES, util.encode_source_base_path_extra)
 ]
 
@@ -209,15 +252,21 @@
 		f"declare -g -r AGGREGATED_DEBOOTSTRAP_COMPONENTS_COMMA='{AGGREGATED_DEBOOTSTRAP_COMPONENTS_COMMA}'\n")
 
 	# Single string stuff for desktop packages postinst's and preparation. @TODO use functions instead of eval.
-	bash.write(util.prepare_bash_output_single_string(
-		"AGGREGATED_DESKTOP_POSTINST", AGGREGATED_DESKTOP_POSTINST))
-	bash.write(util.prepare_bash_output_single_string(
-		"AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE", AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE))
-	bash.write(util.prepare_bash_output_single_string(
-		"AGGREGATED_DESKTOP_BSP_POSTINST", AGGREGATED_DESKTOP_BSP_POSTINST))
-	bash.write(util.prepare_bash_output_single_string(
-		"AGGREGATED_DESKTOP_BSP_PREPARE", AGGREGATED_DESKTOP_BSP_PREPARE))
-	bash.write("\n## End of aggregation output\n");
+	bash.write(util.prepare_bash_output_single_string("AGGREGATED_DESKTOP_POSTINST", AGGREGATED_DESKTOP_POSTINST))
+	bash.write(util.prepare_bash_output_single_string("AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE", AGGREGATED_DESKTOP_CREATE_DESKTOP_PACKAGE))
+	bash.write(util.prepare_bash_output_single_string("AGGREGATED_DESKTOP_BSP_POSTINST", AGGREGATED_DESKTOP_BSP_POSTINST))
+	bash.write(util.prepare_bash_output_single_string("AGGREGATED_DESKTOP_BSP_PREPARE", AGGREGATED_DESKTOP_BSP_PREPARE))
+
+	# Common version (non-arch, non-appgroups, fixed config_name) of the above.
+	bash.write(util.prepare_bash_output_single_string("AGGREGATED_DESKTOP_COMMON_POSTINST", AGGREGATED_DESKTOP_COMMON_POSTINST))
+	bash.write(
+		util.prepare_bash_output_single_string(
+			"AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE",
+			AGGREGATED_DESKTOP_COMMON_CREATE_DESKTOP_PACKAGE))
+	bash.write(util.prepare_bash_output_single_string("AGGREGATED_DESKTOP_COMMON_BSP_POSTINST", AGGREGATED_DESKTOP_COMMON_BSP_POSTINST))
+	bash.write(util.prepare_bash_output_single_string("AGGREGATED_DESKTOP_COMMON_BSP_PREPARE", AGGREGATED_DESKTOP_COMMON_BSP_PREPARE))
+
+	bash.write("\n## End of aggregation output\n")
 
 	# 2) @TODO: Some removals... uninstall-inside-cache and such. (debsums case? also some gnome stuff)
 

lib/tools/common/aggregation_utils.py

@@ -16,9 +16,11 @@
 log: logging.Logger = logging.getLogger("aggregation_utils")
 
 AGGREGATION_SEARCH_ROOT_ABSOLUTE_DIRS = []
+AGGREGATION_SEARCH_ROOT_ABSOLUTE_DIRS_COMMON = []
 DEBOOTSTRAP_SEARCH_RELATIVE_DIRS = []
 CLI_SEARCH_RELATIVE_DIRS = []
 DESKTOP_ENVIRONMENTS_SEARCH_RELATIVE_DIRS = []
+DESKTOP_ENVIRONMENTS_SEARCH_RELATIVE_DIRS_COMMON = []
 DESKTOP_APPGROUPS_SEARCH_RELATIVE_DIRS = []
 SELECTED_CONFIGURATION = None
 DESKTOP_APPGROUPS_SELECTED = []
@@ -228,6 +230,13 @@ def aggregate_all_desktop(artifact, aggregation_function=aggregate_packages_from
 	return aggregation_function(process_common_path_for_potentials(potential_paths))
 
 
+# 'common' version for desktops: does not include any arch or appgroups; config_name is always 'config_base'
+def aggregate_all_desktop_common(artifact, aggregation_function=aggregate_packages_from_potential):
+	potential_paths = calculate_potential_paths(
+		AGGREGATION_SEARCH_ROOT_ABSOLUTE_DIRS_COMMON, DESKTOP_ENVIRONMENTS_SEARCH_RELATIVE_DIRS_COMMON, ["."], artifact)
+	return aggregation_function(process_common_path_for_potentials(potential_paths))
+
+
 def join_refs_for_bash_single_string(refs):
 	single_line_refs = []
 	for ref in refs: