Feature/SK-805 | FEDn cli - new order

docker-compose.yaml

@@ -97,7 +97,7 @@ services:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn run combiner --init config/settings-combiner.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn combiner start --init config/settings-combiner.yaml"
     ports:
       - 12080:12080
     healthcheck:
@@ -127,7 +127,7 @@ services:
       - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
     entrypoint: [ "sh", "-c" ]
     command:
-      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn run client --init config/settings-client.yaml"
+      - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn client start --init config/settings-client.yaml"
     deploy:
       replicas: 0
     depends_on:

docs/faq.rst

@@ -19,6 +19,17 @@ However, during development of a new model it will be necessary to reinitialize.
 
    2. Restart the clients. 
 
+Q: Can I skip fetching the remote package and instead use a local folder when developing the compute package
+------------------------------------------------------------------------------------------------------------
+
+Yes, to facilitate interactive development of the compute package you can start a client that uses a local folder 'client' in your current working directory by: 
+
+.. code-block:: bash
+
+    fedn client start --remote=False -in client.yaml 
+
+
+Note that in production federations this options should in most cases be disallowed. 
 
 Q: How can other aggregation algorithms can be defined?
 -------------------------------------------------------
@@ -45,7 +56,7 @@ Yes! You can toggle which message streams a client subscibes to when starting th
 
 .. code-block:: bash
 
-    fedn run client --trainer=False -in client.yaml 
+    fedn client start --trainer=False -in client.yaml 
 
 
 Q: How do you approach the question of output privacy? 

docs/quickstart.rst

@@ -115,7 +115,7 @@ For example, to split the data in 10 parts and start a client using the 8th part
          export FEDN_PACKAGE_EXTRACT_DIR=package
          export FEDN_NUM_DATA_SPLITS=10
          export FEDN_DATA_PATH=./data/clients/8/mnist.pt
-         fedn run client -in client.yaml --secure=True --force-ssl
+         fedn client start -in client.yaml --secure=True --force-ssl
 
     .. code-tab:: bash
          :caption: Windows (Powershell)

examples/flower-client/README.rst

@@ -62,7 +62,7 @@ On your local machine / client, start the FEDn client:
 
 .. code-block::
 
-   fedn run client -in client.yaml --secure=True --force-ssl
+   fedn client start -in client.yaml --secure=True --force-ssl
 
 
 Or, if you prefer to use Docker (this might take a long time):

examples/mnist-pytorch/README.rst

@@ -73,7 +73,7 @@ For example, to split the data in 10 parts and start a client using the 8th part
    export FEDN_PACKAGE_EXTRACT_DIR=package
    export FEDN_NUM_DATA_SPLITS=10
    export FEDN_DATA_PATH=./data/clients/8/mnist.pt
-   fedn run client -in client.yaml --secure=True --force-ssl
+   fedn client start -in client.yaml --secure=True --force-ssl
 
 The default is to split the data into 2 partitions and use the first partition. 
 

fedn/cli/__init__.py

@@ -1,2 +1,11 @@
+from .client_cmd import client_cmd  # noqa: F401
+from .combiner_cmd import combiner_cmd  # noqa: F401
+from .config_cmd import config_cmd  # noqa: F401
 from .main import main  # noqa: F401
+from .model_cmd import model_cmd  # noqa: F401
+from .package_cmd import package_cmd  # noqa: F401
+from .round_cmd import round_cmd  # noqa: F401
 from .run_cmd import run_cmd  # noqa: F401
+from .session_cmd import session_cmd  # noqa: F401
+from .status_cmd import status_cmd  # noqa: F401
+from .validation_cmd import validation_cmd  # noqa: F401

fedn/cli/client_cmd.py

@@ -0,0 +1,142 @@
+import uuid
+
+import click
+import requests
+
+from fedn.common.exceptions import InvalidClientConfig
+from fedn.network.clients.client import Client
+
+from .main import main
+from .shared import (CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token,
+                     print_response)
+
+
+def validate_client_config(config):
+    """Validate client configuration.
+
+    :param config: Client config (dict).
+    """
+
+    try:
+        if config['discover_host'] is None or \
+                config['discover_host'] == '':
+            raise InvalidClientConfig("Missing required configuration: discover_host")
+        if 'discover_port' not in config.keys():
+            config['discover_port'] = None
+    except Exception:
+        raise InvalidClientConfig("Could not load config from file. Check config")
+
+
+@main.group('client')
+@click.pass_context
+def client_cmd(ctx):
+    """
+
+    :param ctx:
+    """
+    pass
+
+
+@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)')
+@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)')
+@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)')
+@click.option('-t', '--token', required=False, help='Authentication token')
+@click.option('--n_max', required=False, help='Number of items to list')
+@client_cmd.command('list')
+@click.pass_context
+def list_clients(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None):
+    """
+    Return:
+    ------
+    - count: number of clients
+    - result: list of clients
+
+    """
+    url = get_api_url(protocol=protocol, host=host, port=port, endpoint='clients')
+    headers = {}
+
+    if n_max:
+        headers['X-Limit'] = n_max
+
+    _token = get_token(token)
+
+    if _token:
+        headers['Authorization'] = _token
+
+    click.echo(f'\nListing clients: {url}\n')
+    click.echo(f'Headers: {headers}')
+
+    try:
+        response = requests.get(url, headers=headers)
+        print_response(response, 'clients')
+    except requests.exceptions.ConnectionError:
+        click.echo(f'Error: Could not connect to {url}')
+
+
+@client_cmd.command('start')
+@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services(reducer).')
+@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).')
+@click.option('--token', required=False, help='Set token provided by reducer if enabled')
+@click.option('-n', '--name', required=False, default="client" + str(uuid.uuid4())[:8])
+@click.option('-i', '--client_id', required=False)
+@click.option('--local-package', is_flag=True, help='Enable local compute package')
+@click.option('--force-ssl', is_flag=True, help='Force SSL/TLS for REST service')
+@click.option('-u', '--dry-run', required=False, default=False)
+@click.option('-s', '--secure', required=False, default=False)
+@click.option('-pc', '--preshared-cert', required=False, default=False)
+@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST service')
+@click.option('-c', '--preferred-combiner', required=False, default=False)
+@click.option('-va', '--validator', required=False, default=True)
+@click.option('-tr', '--trainer', required=False, default=True)
+@click.option('-in', '--init', required=False, default=None,
+              help='Set to a filename to (re)init client from file state.')
+@click.option('-l', '--logfile', required=False, default=None,
+              help='Set logfile for client log to file.')
+@click.option('--heartbeat-interval', required=False, default=2)
+@click.option('--reconnect-after-missed-heartbeat', required=False, default=30)
+@click.option('--verbosity', required=False, default='INFO', type=click.Choice(['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'], case_sensitive=False))
+@click.pass_context
+def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_package, force_ssl, dry_run, secure, preshared_cert,
+               verify, preferred_combiner, validator, trainer, init, logfile, heartbeat_interval, reconnect_after_missed_heartbeat,
+               verbosity):
+    """
+
+    :param ctx:
+    :param discoverhost:
+    :param discoverport:
+    :param token:
+    :param name:
+    :param client_id:
+    :param remote:
+    :param dry_run:
+    :param secure:
+    :param preshared_cert:
+    :param verify_cert:
+    :param preferred_combiner:
+    :param init:
+    :param logfile:
+    :param hearbeat_interval
+    :param reconnect_after_missed_heartbeat
+    :param verbosity
+    :return:
+    """
+    remote = False if local_package else True
+    config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'name': name,
+              'client_id': client_id, 'remote_compute_context': remote, 'force_ssl': force_ssl, 'dry_run': dry_run, 'secure': secure,
+              'preshared_cert': preshared_cert, 'verify': verify, 'preferred_combiner': preferred_combiner,
+              'validator': validator, 'trainer': trainer, 'logfile': logfile, 'heartbeat_interval': heartbeat_interval,
+              'reconnect_after_missed_heartbeat': reconnect_after_missed_heartbeat, 'verbosity': verbosity}
+
+    if init:
+        apply_config(init, config)
+        click.echo(f'\nClient configuration loaded from file: {init}')
+        click.echo('Values set in file override defaults and command line arguments...\n')
+
+    try:
+        validate_client_config(config)
+    except InvalidClientConfig as e:
+        click.echo(f'Error: {e}')
+        return
+
+    client = Client(config)
+    client.run()

fedn/cli/combiner_cmd.py

@@ -0,0 +1,96 @@
+import uuid
+
+import click
+import requests
+
+from fedn.network.combiner.combiner import Combiner
+
+from .main import main
+from .shared import (CONTROLLER_DEFAULTS, apply_config, get_api_url, get_token,
+                     print_response)
+
+
+@main.group('combiner')
+@click.pass_context
+def combiner_cmd(ctx):
+    """
+
+    :param ctx:
+    """
+    pass
+
+
+@combiner_cmd.command('start')
+@click.option('-d', '--discoverhost', required=False, help='Hostname for discovery services (reducer).')
+@click.option('-p', '--discoverport', required=False, help='Port for discovery services (reducer).')
+@click.option('-t', '--token', required=False, help='Set token provided by reducer if enabled')
+@click.option('-n', '--name', required=False, default="combiner" + str(uuid.uuid4())[:8], help='Set name for combiner.')
+@click.option('-h', '--host', required=False, default="combiner", help='Set hostname.')
+@click.option('-i', '--port', required=False, default=12080, help='Set port.')
+@click.option('-f', '--fqdn', required=False, default=None, help='Set fully qualified domain name')
+@click.option('-s', '--secure', is_flag=True, help='Enable SSL/TLS encrypted gRPC channels.')
+@click.option('-v', '--verify', is_flag=True, help='Verify SSL/TLS for REST discovery service (reducer)')
+@click.option('-c', '--max_clients', required=False, default=30, help='The maximal number of client connections allowed.')
+@click.option('-in', '--init', required=False, default=None,
+              help='Path to configuration file to (re)init combiner.')
+@click.pass_context
+def start_cmd(ctx, discoverhost, discoverport, token, name, host, port, fqdn, secure, verify, max_clients, init):
+    """
+
+    :param ctx:
+    :param discoverhost:
+    :param discoverport:
+    :param token:
+    :param name:
+    :param hostname:
+    :param port:
+    :param secure:
+    :param max_clients:
+    :param init:
+    """
+    config = {'discover_host': discoverhost, 'discover_port': discoverport, 'token': token, 'host': host,
+              'port': port, 'fqdn': fqdn, 'name': name, 'secure': secure, 'verify': verify, 'max_clients': max_clients}
+
+    if init:
+        apply_config(init, config)
+        click.echo(f'\nCombiner configuration loaded from file: {init}')
+        click.echo('Values set in file override defaults and command line arguments...\n')
+
+    combiner = Combiner(config)
+    combiner.run()
+
+
+@click.option('-p', '--protocol', required=False, default=CONTROLLER_DEFAULTS['protocol'], help='Communication protocol of controller (api)')
+@click.option('-H', '--host', required=False, default=CONTROLLER_DEFAULTS['host'], help='Hostname of controller (api)')
+@click.option('-P', '--port', required=False, default=CONTROLLER_DEFAULTS['port'], help='Port of controller (api)')
+@click.option('-t', '--token', required=False, help='Authentication token')
+@click.option('--n_max', required=False, help='Number of items to list')
+@combiner_cmd.command('list')
+@click.pass_context
+def list_combiners(ctx, protocol: str, host: str, port: str, token: str = None, n_max: int = None):
+    """
+    Return:
+    ------
+    - count: number of combiners
+    - result: list of combiners
+
+    """
+    url = get_api_url(protocol=protocol, host=host, port=port, endpoint='combiners')
+    headers = {}
+
+    if n_max:
+        headers['X-Limit'] = n_max
+
+    _token = get_token(token)
+
+    if _token:
+        headers['Authorization'] = _token
+
+    click.echo(f'\nListing combiners: {url}\n')
+    click.echo(f'Headers: {headers}')
+
+    try:
+        response = requests.get(url, headers=headers)
+        print_response(response, 'combiners')
+    except requests.exceptions.ConnectionError:
+        click.echo(f'Error: Could not connect to {url}')

fedn/cli/config_cmd.py

@@ -0,0 +1,54 @@
+import os
+
+import click
+
+from .main import main
+
+envs = [
+    {
+        "name": "FEDN_CONTROLLER_PROTOCOL",
+        "description": "The protocol to use for communication with the controller."
+    },
+    {
+        "name": "FEDN_CONTROLLER_HOST",
+        "description": "The host to use for communication with the controller."
+    },
+    {
+        "name": "FEDN_CONTROLLER_PORT",
+        "description": "The port to use for communication with the controller."
+    },
+    {
+        "name": "FEDN_AUTH_TOKEN",
+        "description": "The authentication token to use for communication with the controller and combiner."
+    },
+    {
+        "name": "FEDN_AUTH_SCHEME",
+        "description": "The authentication scheme to use for communication with the controller and combiner."
+    },
+    {
+        "name": "FEDN_CONTROLLER_URL",
+        "description": "The URL of the controller. Overrides FEDN_CONTROLLER_PROTOCOL, FEDN_CONTROLLER_HOST and FEDN_CONTROLLER_PORT."
+    },
+    {
+        "name": "FEDN_PACKAGE_EXTRACT_DIR",
+        "description": "The directory to extract packages to."
+    }
+]
+
+
+@main.group('config', invoke_without_command=True)
+@click.pass_context
+def config_cmd(ctx):
+    """
+    - Configuration commands for the FEDn CLI.
+    """
+    if ctx.invoked_subcommand is None:
+        click.echo('\n--- FEDn Cli Configuration ---\n')
+        click.echo('Current configuration:\n')
+
+        for env in envs:
+            name = env['name']
+            value = os.environ.get(name)
+            click.echo(f'{name}: {value or "Not set"}')
+            click.echo(f'{env["description"]}\n')
+        click.echo('\n')