Feature/SK-702 | Base-services config by Envs

Dockerfile

@@ -9,9 +9,6 @@ ARG REQUIREMENTS=""
 
 # Add FEDn and default configs
 COPY fedn /app/fedn
-COPY config/settings-client.yaml.template /app/config/settings-client.yaml
-COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml
-COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml
 COPY $REQUIREMENTS /app/config/requirements.txt
 
 # Install developer tools (needed for psutil)

config/reducer-settings.override.yaml

@@ -6,4 +6,4 @@ services:
   reducer:
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
-      - ${HOST_REPO_DIR:-.}/config/settings-reducer.yaml:/app/config/settings-reducer.yaml
+      - ${HOST_REPO_DIR:-.}/config/settings-controller.yaml:/app/config/settings-controller.yaml

docker-compose.yaml

@@ -64,8 +64,28 @@ services:
       - USER=test
       - PROJECT=project
       - FLASK_DEBUG=1
-      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml
-      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml
+      # Controller
+      - FEDN_CONTROLLER_HOST=api-server
+      - FEDN_CONTROLLER_PORT=8092
+      - FEDN_CONTROLLER_DEBUG=True
+      - FEDN_CONTROLLER_CONFIG=/app/config/settings-controller.yaml
+      # Statestore
+      - FEDN_STATESTORE_HOST=mongo
+      - FEDN_STATESTORE_PORT=6534
+      - FEDN_STATESTORE_USER=fedn_admin
+      - FEDN_STATESTORE_PASSWORD=password
+      # will be overriden by env vars above
+      - FEDN_STATESTORE_CONFIG=/app/config/settings-controller.yaml
+      # Modelstorage
+      - FEDN_MODELSTORAGE_HOST=minio
+      - FEDN_MODELSTORAGE_PORT=9000
+      - FEDN_MODELSTORAGE_ACCESS_KEY=fedn_admin
+      - FEDN_MODELSTORAGE_SECRET_KEY=password
+      - FEDN_MODELSTORAGE_BUCKET=fedn-models
+      - FEDN_MODELSTORAGE_CONTEXT_BUCKET=fedn-context
+      - FEDN_MODELSTORAGE_SECURE_MODE=False
+      # will be overriden by env vars above
+      - FEDN_MODELSTORAGE_CONFIG=/app/config/settings-controller.yaml
     build:
       context: .
       args:
@@ -95,9 +115,10 @@ services:
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}/config:/app/config
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run combiner --init config/settings-combiner.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run combiner --init config/settings-combiner.yaml.template"
     ports:
       - 12080:12080
     healthcheck:
@@ -124,9 +145,10 @@ services:
     working_dir: /app
     volumes:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+      - ${HOST_REPO_DIR:-.}/config:/app/config
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run client --init config/settings-client.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e /app/fedn && /venv/bin/fedn run client --init config/settings-client.yaml.template"
     deploy:
       replicas: 0
     depends_on:

fedn/cli/run_cmd.py

@@ -170,6 +170,7 @@ def combiner_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn,
     :param max_clients:
     :param init:
     """
+    # TODO: Should use fedn.common.config.get_combiner_config() instead of this
     config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host,
               'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients,
               'init': init}

fedn/fedn/common/config.py

@@ -6,53 +6,127 @@
 global MODELSTORAGE_CONFIG
 
 
+def get_env(key, default=None):
+    """ Get environment variable.
+
+    :param key: The environment variable key.
+    :type key: str
+    :param default: The default value if the environment variable is not set (optional).
+    :type default: str
+    :return: The environment variable value.
+    :rtype: str
+    """
+    return os.environ.get(key, default)
+
+
+def get_boolean_env(key, default=False):
+    """ Get environment variable as boolean.
+
+    :param key: The environment variable key.
+    :type key: str
+    :param default: The default value if the environment variable is not set (optional).
+    :type default: bool
+    :return: The environment variable value.
+    :rtype: bool
+    """
+    value = os.environ.get(key, default)
+    if isinstance(value, str):
+        return value.lower() in ['true', '1']
+    return value
+
+
 def get_environment_config():
     """ Get the configuration from environment variables.
     """
     global STATESTORE_CONFIG
     global MODELSTORAGE_CONFIG
+    global COMBINER_CONFIG
+    global CONTROLLER_CONFIG
 
-    STATESTORE_CONFIG = os.environ.get('STATESTORE_CONFIG',
-                                       '/workspaces/fedn/config/settings-reducer.yaml.template')
-    MODELSTORAGE_CONFIG = os.environ.get('MODELSTORAGE_CONFIG',
-                                         '/workspaces/fedn/config/settings-reducer.yaml.template')
+    STATESTORE_CONFIG = os.environ.get('FEDN_STATESTORE_CONFIG',
+                                       '/workspaces/fedn/config/settings-controller.yaml.template')
+    MODELSTORAGE_CONFIG = os.environ.get('FEDN_MODELSTORAGE_CONFIG',
+                                         '/workspaces/fedn/config/settings-controller.yaml.template')
+    COMBINER_CONFIG = os.environ.get('FEDN_COMBINER_CONFIG',
+                                     '/workspaces/fedn/config/settings-combiner.yaml.template')
+    CONTROLLER_CONFIG = os.environ.get('FEDN_CONTROLLER_CONFIG',
+                                       '/workspaces/fedn/config/settings-controller.yaml.template')
 
 
-def get_statestore_config(file=None):
-    """ Get the statestore configuration from file.
+def read_config_file(file):
+    """ Read a yaml configuration file.
 
-    :param file: The statestore configuration file (yaml) path (optional).
+    :param file: The configuration file path.
     :type file: str
-    :return: The statestore configuration as a dict.
+    :return: The configuration as a dict.
     :rtype: dict
     """
-    if file is None:
-        get_environment_config()
-        file = STATESTORE_CONFIG
     with open(file, 'r') as config_file:
         try:
-            settings = dict(yaml.safe_load(config_file))
+            config = dict(yaml.safe_load(config_file))
         except yaml.YAMLError as e:
             raise (e)
+    return config
+
+
+def get_statestore_config(file=None):
+    """ Get the statestore configuration.
+
+    :param file: The statestore configuration file (yaml) path (optional).
+    :type file: str
+    :return: The statestore configuration as a dict.
+    :rtype: dict
+    """
+    if file is None:
+        # check if environment variables are set
+        host = get_env("FEDN_STATESTORE_HOST")
+        if host is not None:
+            return {
+                "type": "MongoDB",
+                "mongo_config": {
+                    "host": host,
+                    "port": int(get_env("FEDN_STATESTORE_PORT", 6534)),
+                    "username": get_env("FEDN_STATESTORE_USER", "fedn_admin"),
+                    "password": get_env("FEDN_STATESTORE_PASSWORD"),
+                }
+            }
+        else:
+            # else use the default file
+            get_environment_config()
+            file = STATESTORE_CONFIG
+    settings = read_config_file(file)
     return settings["statestore"]
 
 
 def get_modelstorage_config(file=None):
-    """ Get the model storage configuration from file.
+    """ Get the model storage configuration.
 
     :param file: The model storage configuration file (yaml) path (optional).
     :type file: str
     :return: The model storage configuration as a dict.
     :rtype: dict
     """
     if file is None:
-        get_environment_config()
-        file = MODELSTORAGE_CONFIG
-    with open(file, 'r') as config_file:
-        try:
-            settings = dict(yaml.safe_load(config_file))
-        except yaml.YAMLError as e:
-            raise (e)
+        # check if environment variables are set
+        host = get_env("FEDN_MODELSTORAGE_HOST")
+        if host is not None:
+            return {
+                "storage_type": "S3",
+                "storage_config": {
+                    "storage_hostname": host,
+                    "storage_port": int(get_env("FEDN_MODELSTORAGE_PORT", 9000)),
+                    "storage_access_key": get_env("FEDN_MODELSTORAGE_ACCESS_KEY"),
+                    "storage_secret_key": get_env("FEDN_MODELSTORAGE_SECRET_KEY"),
+                    "storage_bucket": get_env("FEDN_MODELSTORAGE_BUCKET", "fedn-models"),
+                    "context_bucket": get_env("FEDN_MODELSTORAGE_CONTEXT_BUCKET", "fedn-context"),
+                    "storage_secure_mode": get_boolean_env("FEDN_MODELSTORAGE_SECURE_MODE", False),
+                }
+            }
+        else:
+            # else use the config file
+            get_environment_config()
+            file = MODELSTORAGE_CONFIG
+    settings = read_config_file(file)
     return settings["storage"]
 
 
@@ -65,30 +139,71 @@ def get_network_config(file=None):
     :rtype: str
     """
     if file is None:
-        get_environment_config()
-        file = STATESTORE_CONFIG
-    with open(file, 'r') as config_file:
-        try:
-            settings = dict(yaml.safe_load(config_file))
-        except yaml.YAMLError as e:
-            raise (e)
+        # check if environment variables are set
+        network_id = get_env("NETWORK_ID", "fedn-network")
+        if network_id is not None:
+            return network_id
+        else:
+            # else use the default file
+            get_environment_config()
+            # TODO: This is a temporary fix, network_id is the database name in the statestore
+            file = STATESTORE_CONFIG
+    settings = read_config_file(file)
     return settings["network_id"]
 
 
 def get_controller_config(file=None):
-    """ Get the controller configuration from file.
+    """ Get the controller configuration.
 
     :param file: The controller configuration file (yaml) path (optional).
     :type file: str
     :return: The controller configuration as a dict.
     :rtype: dict
     """
     if file is None:
-        get_environment_config()
-        file = STATESTORE_CONFIG
-    with open(file, 'r') as config_file:
-        try:
-            settings = dict(yaml.safe_load(config_file))
-        except yaml.YAMLError as e:
-            raise (e)
+        # check if environment variables are set
+        host = get_env("FEDN_CONTROLLER_HOST")
+        if host is not None:
+            return {
+                "host": host,
+                "port": int(get_env("FEDN_CONTROLLER_PORT", 8092)),
+                "debug": get_boolean_env("FEDN_CONTROLLER_DEBUG", False),
+            }
+        else:
+            # else use the default file
+            get_environment_config()
+            file = CONTROLLER_CONFIG
+    settings = read_config_file(file)
     return settings["controller"]
+
+
+def get_combiner_config(file=None):
+    """ Get the combiner configuration.
+
+    :param file: The combiner configuration file (yaml) path (optional).
+    :type file: str
+    :return: The combiner configuration as a dict.
+    :rtype: dict
+    """
+    if file is None:
+        # check if environment variables are set
+        host = get_env("FEDN_COMBINER_HOST")
+        if host is not None:
+            return {
+                "host": host,
+                "name": get_env("FEDN_COMBINER_NAME"),
+                "port": int(get_env("FEDN_COMBINER_PORT", 12080)),
+                "max_clients": int(get_env("FEDN_COMBINER_MAX_CLIENTS", 30)),
+                "network_id": get_network_config(),
+                "discovery_host": get_env("FEDN_CONTROLLER_HOST"),
+                "discovery_port": int(get_env("FEDN_CONTROLLER_PORT", 8092)),
+                "fqdn": get_env("FEDN_COMBINER_FQDN"),
+                "secure": get_boolean_env("FEDN_GRPC_SECURE", False),
+                "verify": get_boolean_env("FEDN_VERIFY_TLS", False),
+            }
+        else:
+            # else use the default file
+            get_environment_config()
+            file = COMBINER_CONFIG
+    settings = read_config_file(file)
+    return settings["combiner"]