Implement service profiles

Implement profiles as introduced in compose-spec/compose-spec#110
fixes docker#7919
closes docker#1896
closes docker#6742
closes docker#7539

Signed-off-by: Roman Anasal <roman.anasal@bdsu.de>

compose/cli/command.py

@@ -66,7 +66,8 @@ def project_from_options(project_dir, options, additional_options=None):
         environment=environment,
         override_dir=override_dir,
         interpolate=(not additional_options.get('--no-interpolate')),
-        environment_file=environment_file
+        environment_file=environment_file,
+        enabled_profiles=get_profiles_from_options(options, environment)
     )
 
 
@@ -115,9 +116,21 @@ def unicode_paths(paths):
     return None
 
 
+def get_profiles_from_options(options, environment):
+    profile_option = options.get('--profile')
+    if profile_option:
+        return profile_option
+
+    profiles = environment.get('COMPOSE_PROFILE')
+    if profiles:
+        return profiles.split(',')
+
+    return []
+
+
 def get_project(project_dir, config_path=None, project_name=None, verbose=False,
                 context=None, environment=None, override_dir=None,
-                interpolate=True, environment_file=None):
+                interpolate=True, environment_file=None, enabled_profiles=[]):
     if not environment:
         environment = Environment.from_env_file(project_dir)
     config_details = config.find(project_dir, config_path, environment, override_dir)
@@ -139,6 +152,7 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False,
             client,
             environment.get('DOCKER_DEFAULT_PLATFORM'),
             execution_context_labels(config_details, environment_file),
+            enabled_profiles,
         )
 
 

compose/cli/main.py

@@ -182,14 +182,15 @@ class TopLevelCommand:
     """Define and run multi-container applications with Docker.
 
     Usage:
-      docker-compose [-f <arg>...] [options] [--] [COMMAND] [ARGS...]
+      docker-compose [-f <arg>...] [--profile <name>...] [options] [--] [COMMAND] [ARGS...]
       docker-compose -h|--help
 
     Options:
       -f, --file FILE             Specify an alternate compose file
                                   (default: docker-compose.yml)
       -p, --project-name NAME     Specify an alternate project name
                                   (default: directory name)
+      --profile NAME              Specify a profile to enable
       -c, --context NAME          Specify a context name
       --verbose                   Show more output
       --log-level LEVEL           Set log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)

compose/config/compose_spec.json

@@ -328,6 +328,7 @@
           "uniqueItems": true
         },
         "privileged": {"type": "boolean"},
+        "profiles": {"$ref": "#/definitions/list_of_strings"},
         "pull_policy": {"type": "string", "enum": [
           "always", "never", "if_not_present"
         ]},

compose/config/config.py

@@ -133,6 +133,7 @@
     'logging',
     'network_mode',
     'platform',
+    'profiles',
     'scale',
     'stop_grace_period',
 ]
@@ -1047,7 +1048,7 @@ def merge_service_dicts(base, override, version):
 
     for field in [
         'cap_add', 'cap_drop', 'expose', 'external_links',
-        'volumes_from', 'device_cgroup_rules',
+        'volumes_from', 'device_cgroup_rules', 'profiles',
     ]:
         md.merge_field(field, merge_unique_items_lists, default=[])
 

compose/project.py

@@ -68,13 +68,15 @@ class Project:
     """
     A collection of services.
     """
-    def __init__(self, name, services, client, networks=None, volumes=None, config_version=None):
+    def __init__(self, name, services, client, networks=None, volumes=None, config_version=None,
+                 enabled_profiles=[]):
         self.name = name
         self.services = services
         self.client = client
         self.volumes = volumes or ProjectVolumes({})
         self.networks = networks or ProjectNetworks({}, False)
         self.config_version = config_version
+        self.enabled_profiles = enabled_profiles
 
     def labels(self, one_off=OneOffFilter.exclude, legacy=False):
         name = self.name
@@ -86,7 +88,8 @@ def labels(self, one_off=OneOffFilter.exclude, legacy=False):
         return labels
 
     @classmethod
-    def from_config(cls, name, config_data, client, default_platform=None, extra_labels=None):
+    def from_config(cls, name, config_data, client, default_platform=None, extra_labels=None,
+                    enabled_profiles=[]):
         """
         Construct a Project from a config.Config object.
         """
@@ -98,7 +101,7 @@ def from_config(cls, name, config_data, client, default_platform=None, extra_lab
             networks,
             use_networking)
         volumes = ProjectVolumes.from_config(name, config_data, client)
-        project = cls(name, [], client, project_networks, volumes, config_data.version)
+        project = cls(name, [], client, project_networks, volumes, config_data.version, enabled_profiles)
 
         for service_dict in config_data.services:
             service_dict = dict(service_dict)
@@ -186,7 +189,7 @@ def validate_service_names(self, service_names):
             if name not in valid_names:
                 raise NoSuchService(name)
 
-    def get_services(self, service_names=None, include_deps=False):
+    def get_services(self, service_names=None, include_deps=False, auto_enable_profiles=True):
         """
         Returns a list of this project's services filtered
         by the provided list of names, or all services if service_names is None
@@ -199,15 +202,42 @@ def get_services(self, service_names=None, include_deps=False):
         reordering as needed to resolve dependencies.
 
         Raises NoSuchService if any of the named services do not exist.
+
+        Raises ConfigurationError if any service depended on is not enabled by active profiles
         """
+        enabled_profiles = self.enabled_profiles.copy()
+
         if service_names is None or len(service_names) == 0:
-            service_names = self.service_names
+            auto_enable_profiles = False
+            service_names = [
+                service.name
+                for service in self.services
+                # only include services without profiles or in enabled profiles
+                if 'profiles' not in service.options or [
+                    e for e in service.options.get('profiles')
+                    if e in enabled_profiles
+                ]
+            ]
 
         unsorted = [self.get_service(name) for name in service_names]
         services = [s for s in self.services if s in unsorted]
 
+        if auto_enable_profiles:
+            # enable profiles of explicitly targeted services
+            for service in services:
+                if 'profiles' in service.options:
+                    [
+                        enabled_profiles.append(p)
+                        for p in service.options.get('profiles')
+                        if p not in enabled_profiles
+                    ]
+
         if include_deps:
-            services = reduce(self._inject_deps, services, [])
+            services = reduce(
+                lambda acc, s: self._inject_deps(acc, s, enabled_profiles),
+                services,
+                []
+            )
 
         uniques = []
         [uniques.append(s) for s in services if s not in uniques]
@@ -438,10 +468,12 @@ def down(
         self.remove_images(remove_image_type)
 
     def remove_images(self, remove_image_type):
-        for service in self.get_services():
+        for service in self.services:
             service.remove_image(remove_image_type)
 
     def restart(self, service_names=None, **options):
+        # filter service_names by enabled profiles
+        service_names = [s.name for s in self.get_services(service_names)]
         containers = self.containers(service_names, stopped=True)
 
         parallel.parallel_execute(
@@ -856,14 +888,29 @@ def _find():
                 )
             )
 
-    def _inject_deps(self, acc, service):
+    def _inject_deps(self, acc, service, enabled_profiles):
         dep_names = service.get_dependency_names()
 
         if len(dep_names) > 0:
             dep_services = self.get_services(
                 service_names=list(set(dep_names)),
-                include_deps=True
+                include_deps=True,
+                auto_enable_profiles=False
             )
+
+            for dep in dep_services:
+                if 'profiles' in dep.options and not [
+                    e for e in dep.options.get('profiles')
+                    if e in enabled_profiles
+                ]:
+                    raise ConfigurationError(
+                        'Service "{dep_name}" was pulled in as a dependency of '
+                        'service "{service_name}" but is not enabled by the '
+                        'active profiles. '
+                        'You may fix this by adding a common profile to '
+                        '"{dep_name}" and "{service_name}".'
+                        .format(dep_name=dep.name, service_name=service.name)
+                    )
         else:
             dep_services = []
 

tests/acceptance/cli_test.py

@@ -1719,6 +1719,98 @@ def test_up_with_ipc_mode(self):
         shareable_mode_container = self.project.get_service('shareable').containers()[0]
         assert shareable_mode_container.get('HostConfig.IpcMode') == 'shareable'
 
+    def test_profiles_up_with_no_profile(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        self.dispatch(['up'])
+
+        containers = self.project.containers(stopped=True)
+        service_names = [c.service for c in containers]
+
+        assert 'foo' in service_names
+        assert len(containers) == 1
+
+    def test_profiles_up_with_profile(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        self.dispatch(['--profile', 'test', 'up'])
+
+        containers = self.project.containers(stopped=True)
+        service_names = [c.service for c in containers]
+
+        assert 'foo' in service_names
+        assert 'bar' in service_names
+        assert 'baz' in service_names
+        assert len(containers) == 3
+
+    def test_profiles_up_invalid_dependency(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        result = self.dispatch(['--profile', 'debug', 'up'], returncode=1)
+
+        assert ('Service "bar" was pulled in as a dependency of service "zot" '
+                'but is not enabled by the active profiles.') in result.stderr
+
+    def test_profiles_up_with_multiple_profiles(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        self.dispatch(['--profile', 'debug', '--profile', 'test', 'up'])
+
+        containers = self.project.containers(stopped=True)
+        service_names = [c.service for c in containers]
+
+        assert 'foo' in service_names
+        assert 'bar' in service_names
+        assert 'baz' in service_names
+        assert 'zot' in service_names
+        assert len(containers) == 4
+
+    def test_profiles_up_with_profile_enabled_by_service(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        self.dispatch(['up', 'bar'])
+
+        containers = self.project.containers(stopped=True)
+        service_names = [c.service for c in containers]
+
+        assert 'bar' in service_names
+        assert len(containers) == 1
+
+    def test_profiles_up_with_dependency_and_profile_enabled_by_service(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        self.dispatch(['up', 'baz'])
+
+        containers = self.project.containers(stopped=True)
+        service_names = [c.service for c in containers]
+
+        assert 'bar' in service_names
+        assert 'baz' in service_names
+        assert len(containers) == 2
+
+    def test_profiles_up_with_invalid_dependency_for_target_service(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        result = self.dispatch(['up', 'zot'], returncode=1)
+
+        assert ('Service "bar" was pulled in as a dependency of service "zot" '
+                'but is not enabled by the active profiles.') in result.stderr
+
+    def test_profiles_up_with_profile_for_dependency(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        self.dispatch(['--profile', 'test', 'up', 'zot'])
+
+        containers = self.project.containers(stopped=True)
+        service_names = [c.service for c in containers]
+
+        assert 'bar' in service_names
+        assert 'zot' in service_names
+        assert len(containers) == 2
+
+    def test_profiles_up_with_merged_profiles(self):
+        self.base_dir = 'tests/fixtures/profiles'
+        self.dispatch(['-f', 'docker-compose.yml', '-f', 'merge-profiles.yml', 'up', 'zot'])
+
+        containers = self.project.containers(stopped=True)
+        service_names = [c.service for c in containers]
+
+        assert 'bar' in service_names
+        assert 'zot' in service_names
+        assert len(containers) == 2
+
     def test_exec_without_tty(self):
         self.base_dir = 'tests/fixtures/links-composefile'
         self.dispatch(['up', '-d', 'console'])

tests/fixtures/profiles/docker-compose.yml

@@ -0,0 +1,20 @@
+version: "3"
+services:
+  foo:
+    image: busybox:1.31.0-uclibc
+  bar:
+    image: busybox:1.31.0-uclibc
+    profiles:
+      - test
+  baz:
+    image: busybox:1.31.0-uclibc
+    depends_on:
+      - bar
+    profiles:
+      - test
+  zot:
+    image: busybox:1.31.0-uclibc
+    depends_on:
+      - bar
+    profiles:
+      - debug

tests/fixtures/profiles/merge-profiles.yml

@@ -0,0 +1,5 @@
+version: "3"
+services:
+  bar:
+    profiles:
+      - debug