From b2c61e80ff6fca1bb6d8faaa6cca929aa06d0d1a Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Thu, 22 Feb 2024 09:42:16 +0100 Subject: [PATCH 1/3] Temporarily disabled exposing the Dask diagnostic dashboard port to the docker setup. Executing docker containers after each other using docker compose, leads sometimes to the case that the port is already in use. --- src/fondant/pipeline/compiler.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/fondant/pipeline/compiler.py b/src/fondant/pipeline/compiler.py index be1004af..cd15420a 100644 --- a/src/fondant/pipeline/compiler.py +++ b/src/fondant/pipeline/compiler.py @@ -27,6 +27,12 @@ logger = logging.getLogger(__name__) DASK_DIAGNOSTIC_DASHBOARD_PORT = 8787 + +# Temporarily disabled exposing the Dask diagnostic dashboard port to the docker setup. +# Executing docker containers after each other using docker compose, +# leads sometimes to the case that the port is already in use. +EXPOSING_DASK_DIAGNOSTIC_DASHBOARD = False + KubeflowCommandArguments = t.List[t.Union[str, t.Dict[str, str]]] @@ -259,16 +265,16 @@ def _generate_spec( volumes.extend(extra_volumes) ports: t.List[t.Union[str, dict]] = [] - ports.append( - f"{DASK_DIAGNOSTIC_DASHBOARD_PORT}:{DASK_DIAGNOSTIC_DASHBOARD_PORT}", - ) + if EXPOSING_DASK_DIAGNOSTIC_DASHBOARD: + ports.append( + f"{DASK_DIAGNOSTIC_DASHBOARD_PORT}:{DASK_DIAGNOSTIC_DASHBOARD_PORT}", + ) services[component_id] = { "entrypoint": entrypoint, "command": command, "depends_on": depends_on, "volumes": volumes, - "ports": ports, "labels": { "pipeline_description": pipeline.description, }, From 77a5b2d3d734b83dfdf600d8156dce5f5f772042 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Mon, 26 Feb 2024 13:31:15 +0100 Subject: [PATCH 2/3] Addressing comments --- src/fondant/pipeline/compiler.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/fondant/pipeline/compiler.py b/src/fondant/pipeline/compiler.py index cd15420a..51796332 100644 --- a/src/fondant/pipeline/compiler.py +++ b/src/fondant/pipeline/compiler.py @@ -28,11 +28,6 @@ DASK_DIAGNOSTIC_DASHBOARD_PORT = 8787 -# Temporarily disabled exposing the Dask diagnostic dashboard port to the docker setup. -# Executing docker containers after each other using docker compose, -# leads sometimes to the case that the port is already in use. -EXPOSING_DASK_DIAGNOSTIC_DASHBOARD = False - KubeflowCommandArguments = t.List[t.Union[str, t.Dict[str, str]]] @@ -264,17 +259,14 @@ def _generate_spec( if extra_volumes: volumes.extend(extra_volumes) - ports: t.List[t.Union[str, dict]] = [] - if EXPOSING_DASK_DIAGNOSTIC_DASHBOARD: - ports.append( - f"{DASK_DIAGNOSTIC_DASHBOARD_PORT}:{DASK_DIAGNOSTIC_DASHBOARD_PORT}", - ) + ports: t.List[int] = [DASK_DIAGNOSTIC_DASHBOARD_PORT] services[component_id] = { "entrypoint": entrypoint, "command": command, "depends_on": depends_on, "volumes": volumes, + "ports": ports, "labels": { "pipeline_description": pipeline.description, }, From 15569bdc00d4248a7e91a30996212b54eef845ef Mon Sep 17 00:00:00 2001 From: Robbe Sneyders Date: Wed, 6 Mar 2024 21:04:52 +0100 Subject: [PATCH 3/3] Make dask diagnostics port configurable --- src/fondant/pipeline/compiler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/fondant/pipeline/compiler.py b/src/fondant/pipeline/compiler.py index 51796332..8d2eecd9 100644 --- a/src/fondant/pipeline/compiler.py +++ b/src/fondant/pipeline/compiler.py @@ -26,7 +26,8 @@ logger = logging.getLogger(__name__) -DASK_DIAGNOSTIC_DASHBOARD_PORT = 8787 +# export DASK_DIAGNOSTICS_PORT="" to get a dynamic port assigned +DASK_DIAGNOSTICS_PORT = os.environ.get("DASK_DIAGNOSTICS_PORT", ":8787") KubeflowCommandArguments = t.List[t.Union[str, t.Dict[str, str]]] @@ -259,7 +260,7 @@ def _generate_spec( if extra_volumes: volumes.extend(extra_volumes) - ports: t.List[int] = [DASK_DIAGNOSTIC_DASHBOARD_PORT] + ports = [f"8787{DASK_DIAGNOSTICS_PORT}"] services[component_id] = { "entrypoint": entrypoint,