From df59ff880d2f7656c8eba5657da97436f7330673 Mon Sep 17 00:00:00 2001 From: Roman Anasal Date: Sun, 15 Nov 2020 16:05:25 +0100 Subject: [PATCH] Implement service profiles Implement profiles as introduced in compose-spec/compose-spec#110 fixes #7919 closes #1896 closes #6742 closes #7539 --- compose/cli/command.py | 18 ++++- compose/cli/main.py | 3 +- compose/config/compose_spec.json | 1 + compose/config/config.py | 3 +- compose/project.py | 63 ++++++++++++--- tests/acceptance/cli_test.py | 92 ++++++++++++++++++++++ tests/fixtures/profiles/docker-compose.yml | 20 +++++ tests/fixtures/profiles/merge-profiles.yml | 5 ++ 8 files changed, 192 insertions(+), 13 deletions(-) create mode 100644 tests/fixtures/profiles/docker-compose.yml create mode 100644 tests/fixtures/profiles/merge-profiles.yml diff --git a/compose/cli/command.py b/compose/cli/command.py index d471e78df1a..94fb6d2d8a2 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -66,7 +66,8 @@ def project_from_options(project_dir, options, additional_options=None): environment=environment, override_dir=override_dir, interpolate=(not additional_options.get('--no-interpolate')), - environment_file=environment_file + environment_file=environment_file, + enabled_profiles=get_profiles_from_options(options, environment) ) @@ -115,9 +116,21 @@ def unicode_paths(paths): return None +def get_profiles_from_options(options, environment): + profile_option = options.get('--profile') + if profile_option: + return profile_option + + profiles = environment.get('COMPOSE_PROFILE') + if profiles: + return profiles.split(':') + + return [] + + def get_project(project_dir, config_path=None, project_name=None, verbose=False, context=None, environment=None, override_dir=None, - interpolate=True, environment_file=None): + interpolate=True, environment_file=None, enabled_profiles=[]): if not environment: environment = Environment.from_env_file(project_dir) config_details = config.find(project_dir, config_path, environment, override_dir) @@ -139,6 +152,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, client, environment.get('DOCKER_DEFAULT_PLATFORM'), execution_context_labels(config_details, environment_file), + enabled_profiles, ) diff --git a/compose/cli/main.py b/compose/cli/main.py index 6776dae9e5f..21399a17bb6 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -182,7 +182,7 @@ class TopLevelCommand: """Define and run multi-container applications with Docker. Usage: - docker-compose [-f ...] [options] [--] [COMMAND] [ARGS...] + docker-compose [-f ...] [--profile ...] [options] [--] [COMMAND] [ARGS...] docker-compose -h|--help Options: @@ -190,6 +190,7 @@ class TopLevelCommand: (default: docker-compose.yml) -p, --project-name NAME Specify an alternate project name (default: directory name) + --profile NAME Specify a profile to enable -c, --context NAME Specify a context name --verbose Show more output --log-level LEVEL Set log level (DEBUG, INFO, WARNING, ERROR, CRITICAL) diff --git a/compose/config/compose_spec.json b/compose/config/compose_spec.json index 26825674427..fba9d211d5e 100644 --- a/compose/config/compose_spec.json +++ b/compose/config/compose_spec.json @@ -328,6 +328,7 @@ "uniqueItems": true }, "privileged": {"type": "boolean"}, + "profiles": {"$ref": "#/definitions/list_of_strings"}, "pull_policy": {"type": "string", "enum": [ "always", "never", "if_not_present" ]}, diff --git a/compose/config/config.py b/compose/config/config.py index 1b067e78877..eee6b12dd7b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -133,6 +133,7 @@ 'logging', 'network_mode', 'platform', + 'profiles', 'scale', 'stop_grace_period', ] @@ -1047,7 +1048,7 @@ def merge_service_dicts(base, override, version): for field in [ 'cap_add', 'cap_drop', 'expose', 'external_links', - 'volumes_from', 'device_cgroup_rules', + 'volumes_from', 'device_cgroup_rules', 'profiles', ]: md.merge_field(field, merge_unique_items_lists, default=[]) diff --git a/compose/project.py b/compose/project.py index 420cb6548bc..6f8575c57a7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -68,13 +68,14 @@ class Project: """ A collection of services. """ - def __init__(self, name, services, client, networks=None, volumes=None, config_version=None): + def __init__(self, name, services, client, networks=None, volumes=None, config_version=None, enabled_profiles=[]): self.name = name self.services = services self.client = client self.volumes = volumes or ProjectVolumes({}) self.networks = networks or ProjectNetworks({}, False) self.config_version = config_version + self.enabled_profiles = enabled_profiles def labels(self, one_off=OneOffFilter.exclude, legacy=False): name = self.name @@ -86,7 +87,7 @@ def labels(self, one_off=OneOffFilter.exclude, legacy=False): return labels @classmethod - def from_config(cls, name, config_data, client, default_platform=None, extra_labels=None): + def from_config(cls, name, config_data, client, default_platform=None, extra_labels=None, enabled_profiles=[]): """ Construct a Project from a config.Config object. """ @@ -98,7 +99,7 @@ def from_config(cls, name, config_data, client, default_platform=None, extra_lab networks, use_networking) volumes = ProjectVolumes.from_config(name, config_data, client) - project = cls(name, [], client, project_networks, volumes, config_data.version) + project = cls(name, [], client, project_networks, volumes, config_data.version, enabled_profiles) for service_dict in config_data.services: service_dict = dict(service_dict) @@ -185,7 +186,7 @@ def validate_service_names(self, service_names): if name not in valid_names: raise NoSuchService(name) - def get_services(self, service_names=None, include_deps=False): + def get_services(self, service_names=None, include_deps=False, auto_enable_profiles=True): """ Returns a list of this project's services filtered by the provided list of names, or all services if service_names is None @@ -198,15 +199,42 @@ def get_services(self, service_names=None, include_deps=False): reordering as needed to resolve dependencies. Raises NoSuchService if any of the named services do not exist. + + Raises ConfigurationError if any service depended on is not enabled by active profiles """ + enabled_profiles = self.enabled_profiles.copy() + if service_names is None or len(service_names) == 0: - service_names = self.service_names + auto_enable_profiles = False + service_names = [ + service.name + for service in self.services + # only include services without profiles or in enabled profiles + if 'profiles' not in service.options or [ + e for e in service.options.get('profiles') + if e in enabled_profiles + ] + ] unsorted = [self.get_service(name) for name in service_names] services = [s for s in self.services if s in unsorted] + if auto_enable_profiles: + # enable profiles of explicitly targeted services + for service in services: + if 'profiles' in service.options: + [ + enabled_profiles.append(p) + for p in service.options.get('profiles') + if p not in enabled_profiles + ] + if include_deps: - services = reduce(self._inject_deps, services, []) + services = reduce( + lambda acc, s: self._inject_deps(acc, s, enabled_profiles), + services, + [] + ) uniques = [] [uniques.append(s) for s in services if s not in uniques] @@ -412,10 +440,12 @@ def down( self.remove_images(remove_image_type) def remove_images(self, remove_image_type): - for service in self.get_services(): + for service in self.services: service.remove_image(remove_image_type) def restart(self, service_names=None, **options): + # filter service_names by enabled profiles + service_names = [s.name for s in self.get_services(service_names)] containers = self.containers(service_names, stopped=True) parallel.parallel_execute( @@ -830,14 +860,29 @@ def _find(): ) ) - def _inject_deps(self, acc, service): + def _inject_deps(self, acc, service, enabled_profiles): dep_names = service.get_dependency_names() if len(dep_names) > 0: dep_services = self.get_services( service_names=list(set(dep_names)), - include_deps=True + include_deps=True, + auto_enable_profiles=False ) + + for dep in dep_services: + if 'profiles' in dep.options and not [ + e for e in dep.options.get('profiles') + if e in enabled_profiles + ]: + raise ConfigurationError( + 'Service "{dep_name}" was pulled in as a dependency of ' + 'service "{service_name}" but is not enabled by the ' + 'active profiles. ' + 'You may fix this by adding a common profile to ' + '"{dep_name}" and "{service_name}".' + .format(dep_name=dep.name, service_name=service.name) + ) else: dep_services = [] diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a7ccd148940..2ff51573982 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1719,6 +1719,98 @@ def test_up_with_ipc_mode(self): shareable_mode_container = self.project.get_service('shareable').containers()[0] assert shareable_mode_container.get('HostConfig.IpcMode') == 'shareable' + def test_profiles_up_with_no_profile(self): + self.base_dir = 'tests/fixtures/profiles' + self.dispatch(['up']) + + containers = self.project.containers(stopped=True) + service_names = [c.service for c in containers] + + assert 'foo' in service_names + assert len(containers) == 1 + + def test_profiles_up_with_profile(self): + self.base_dir = 'tests/fixtures/profiles' + self.dispatch(['--profile', 'test', 'up']) + + containers = self.project.containers(stopped=True) + service_names = [c.service for c in containers] + + assert 'foo' in service_names + assert 'bar' in service_names + assert 'baz' in service_names + assert len(containers) == 3 + + def test_profiles_up_invalid_dependency(self): + self.base_dir = 'tests/fixtures/profiles' + result = self.dispatch(['--profile', 'debug', 'up'], returncode=1) + + assert ('Service "bar" was pulled in as a dependency of service "zot" ' + 'but is not enabled by the active profiles.') in result.stderr + + def test_profiles_up_with_multiple_profiles(self): + self.base_dir = 'tests/fixtures/profiles' + self.dispatch(['--profile', 'debug', '--profile', 'test', 'up']) + + containers = self.project.containers(stopped=True) + service_names = [c.service for c in containers] + + assert 'foo' in service_names + assert 'bar' in service_names + assert 'baz' in service_names + assert 'zot' in service_names + assert len(containers) == 4 + + def test_profiles_up_with_profile_enabled_by_service(self): + self.base_dir = 'tests/fixtures/profiles' + self.dispatch(['up', 'bar']) + + containers = self.project.containers(stopped=True) + service_names = [c.service for c in containers] + + assert 'bar' in service_names + assert len(containers) == 1 + + def test_profiles_up_with_dependency_and_profile_enabled_by_service(self): + self.base_dir = 'tests/fixtures/profiles' + self.dispatch(['up', 'baz']) + + containers = self.project.containers(stopped=True) + service_names = [c.service for c in containers] + + assert 'bar' in service_names + assert 'baz' in service_names + assert len(containers) == 2 + + def test_profiles_up_with_invalid_dependency_for_target_service(self): + self.base_dir = 'tests/fixtures/profiles' + result = self.dispatch(['up', 'zot'], returncode=1) + + assert ('Service "bar" was pulled in as a dependency of service "zot" ' + 'but is not enabled by the active profiles.') in result.stderr + + def test_profiles_up_with_profile_for_dependency(self): + self.base_dir = 'tests/fixtures/profiles' + self.dispatch(['--profile', 'test', 'up', 'zot']) + + containers = self.project.containers(stopped=True) + service_names = [c.service for c in containers] + + assert 'bar' in service_names + assert 'zot' in service_names + assert len(containers) == 2 + + def test_profiles_up_with_merged_profiles(self): + self.base_dir = 'tests/fixtures/profiles' + self.dispatch(['-f', 'docker-compose.yml', '-f', 'merge-profiles.yml', 'up', 'zot']) + + containers = self.project.containers(stopped=True) + service_names = [c.service for c in containers] + + assert 'bar' in service_names + assert 'zot' in service_names + assert len(containers) == 2 + def test_exec_without_tty(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'console']) diff --git a/tests/fixtures/profiles/docker-compose.yml b/tests/fixtures/profiles/docker-compose.yml new file mode 100644 index 00000000000..ba77f03b446 --- /dev/null +++ b/tests/fixtures/profiles/docker-compose.yml @@ -0,0 +1,20 @@ +version: "3" +services: + foo: + image: busybox:1.31.0-uclibc + bar: + image: busybox:1.31.0-uclibc + profiles: + - test + baz: + image: busybox:1.31.0-uclibc + depends_on: + - bar + profiles: + - test + zot: + image: busybox:1.31.0-uclibc + depends_on: + - bar + profiles: + - debug diff --git a/tests/fixtures/profiles/merge-profiles.yml b/tests/fixtures/profiles/merge-profiles.yml new file mode 100644 index 00000000000..42b0cfa4308 --- /dev/null +++ b/tests/fixtures/profiles/merge-profiles.yml @@ -0,0 +1,5 @@ +version: "3" +services: + bar: + profiles: + - debug