Merge pull request #8610 from OpenMined/rasswanth/fix_devspace_ports

Fix Multi Cluster Deployment in Devspace.
OpenMined · May 8, 2024 · 2f0cce7 · 2f0cce7
2 parents 9d45306 + e193aaf
commit 2f0cce7
Show file tree

Hide file tree

Showing 7 changed files with 116 additions and 39 deletions.
diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh
@@ -23,7 +23,6 @@ if [[ ${DEV_MODE} == "True" ]];
 then
     echo "DEV_MODE Enabled"
     RELOAD="--reload"
-    pip install --user -e "$APPDIR/syft[telemetry,data_science]"
 fi
 
 # only set by kubernetes to avoid conflict with docker tests

diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml
@@ -59,14 +59,6 @@ images:
     context: ./seaweedfs
     tags:
       - dev-${DEVSPACE_TIMESTAMP}
-  attestation:
-    image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_ENCLAVE_ATTESTATION}"
-    buildKit:
-      args: ["--platform", "linux/amd64"]
-    dockerfile: ./enclave/attestation/attestation.dockerfile
-    context: ./enclave/attestation
-    tags:
-      - dev-${DEVSPACE_TIMESTAMP}
 
 # This is a list of `deployments` that DevSpace can create for this project
 deployments:
@@ -119,10 +111,8 @@ dev:
         sync:
           - path: ./backend/grid:/root/app/grid
           - path: ../syft:/root/app/syft
-      enclave-attestation:
-        sync:
-          - path: ./enclave/attestation/server:/app/server
-      ssh: {}
+        ssh:
+          localPort: 3480
 
 profiles:
   - name: gateway
@@ -135,6 +125,22 @@ profiles:
       - op: remove
         path: dev.seaweedfs
 
+      # Port Re-Mapping
+      # Mongo
+      - op: replace
+        path: dev.mongo.ports[0].port
+        value: 27018:27017
+
+      # Backend
+      - op: replace
+        path: dev.backend.ports[0].port
+        value: 5679:5678
+
+      # Backend Container SSH
+      - op: replace
+        path: dev.backend.containers.backend-container.ssh.localPort
+        value: 3481
+
   - name: gcp
     patches:
       - op: replace
@@ -155,3 +161,57 @@ profiles:
         path: deployments.syft.helm.valuesFiles
         value:
           - ./helm/examples/azure/azure.high.yaml
+
+  - name: enclave
+    patches:
+      - op: replace
+        path: deployments.syft.helm.values.node.type
+        value: "enclave"
+
+      - op: add
+        path: images
+        value:
+          enclave-attestation:
+            image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_ENCLAVE_ATTESTATION}"
+            buildKit:
+              args: ["--platform", "linux/amd64"]
+            dockerfile: ./enclave/attestation/attestation.dockerfile
+            context: ./enclave/attestation
+            tags:
+              - dev-${DEVSPACE_TIMESTAMP}
+      - op: add
+        path: dev.backend.containers
+        value:
+          enclave-attestation:
+            sync:
+              - path: ./enclave/attestation/server:/app/server
+
+      - op: add
+        path: deployments.syft.helm.values
+        value:
+          attestation:
+            enabled: true
+
+      # Port Re-Mapping
+      # Mongo
+      - op: replace
+        path: dev.mongo.ports[0].port
+        value: 27019:27017
+
+      # Backend
+      - op: replace
+        path: dev.backend.ports[0].port
+        value: 5680:5678
+
+      # Backend Container SSH
+      - op: replace
+        path: dev.backend.containers.backend-container.ssh.localPort
+        value: 3482
+
+      - op: replace
+        path: dev.seaweedfs.ports
+        value:
+          - port: "9334:9333" # admin
+          - port: "8889:8888" # filer
+          - port: "8334:8333" # S3
+          - port: "4002:4001" # mount api
diff --git a/packages/grid/enclave/attestation/enclave-development.md b/packages/grid/enclave/attestation/enclave-development.md
@@ -103,12 +103,13 @@ client.add_verifier(attestation.Devices.GPU, attestation.Environment.REMOTE, NRA
 client.attest()
 ```
 
-### Instructions for using helm charts
+### Instructions for Development (Devspace)
 
-- The attestation container runs inside the backend pod (so backend pod has two containers now). However, in order to run the attestation container, you need to uncomment the attestation flags in `packages/grid/helm/values.dev.yaml`
-- Next, we run the deployment. Since k3d creates an intermediate layer of nesting, we need to mount some volumes from host to k3d registry. Thus, when launching, use the following tox command `tox -e dev.k8s.start -- --volume /sys/kernel/security:/sys/kernel/security --volume /dev/tmprm0:/dev/tmprm0`
-- Finally, note that the GPU privileges/drivers etc. have not been completed so while the GPU attestation endpoints should work, they will not produce the expected tokens. To test the GPU code, follow the steps provided in [For GPU Attestation
-  ](#for-gpu-attestation) to look at the tokens.
+We could launch an enclave stack by the command.
+
+```sh
+tox -e dev.k8s.launch.enclave
+```
 
 ### Local Client-side Verification
 

diff --git a/scripts/k8s/delete_stack.sh b/scripts/k8s/delete_stack.sh
diff --git a/scripts/k8s/launch_domain.sh b/scripts/k8s/launch_domain.sh
diff --git a/scripts/k8s/launch_gateway.sh b/scripts/k8s/launch_gateway.sh
diff --git a/tox.ini b/tox.ini
@@ -988,6 +988,44 @@ commands =
     bash -c 'devspace cleanup images --kube-context k3d-${CLUSTER_NAME} --no-warn --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 || true'
     bash -c 'kubectl --context k3d-${CLUSTER_NAME} delete namespace syft --now=true || true'
 
+[testenv:dev.k8s.launch.gateway]
+description = Launch a single gateway on K8s
+passenv = HOME, USER
+setenv=
+    CLUSTER_NAME = {env:CLUSTER_NAME:test-gateway-1}
+    CLUSTER_HTTP_PORT={env:CLUSTER_HTTP_PORT:9081}
+    DEVSPACE_PROFILE=gateway
+allowlist_externals =
+    tox
+commands =
+    tox -e dev.k8s.start
+    tox -e dev.k8s.{posargs:deploy}
+
+[testenv:dev.k8s.launch.domain]
+description = Launch a single domain  on K8s
+passenv = HOME, USER
+setenv=
+    CLUSTER_NAME = {env:CLUSTER_NAME:test-domain-1}
+    CLUSTER_HTTP_PORT={env:CLUSTER_HTTP_PORT:9082}
+allowlist_externals =
+    tox
+commands =
+    tox -e dev.k8s.start
+    tox -e dev.k8s.{posargs:deploy}
+
+[testenv:dev.k8s.launch.enclave]
+description = Launch a single Enclave  on K8s
+passenv = HOME, USER
+setenv=
+    CLUSTER_NAME = {env:CLUSTER_NAME:test-enclave-1}
+    CLUSTER_HTTP_PORT={env:CLUSTER_HTTP_PORT:9083}
+    DEVSPACE_PROFILE=enclave
+allowlist_externals =
+    tox
+commands =
+    tox -e dev.k8s.start -- --volume /sys/kernel/security:/sys/kernel/security --volume /dev/tmprm0:/dev/tmprm0
+    tox -e dev.k8s.{posargs:deploy}
+
 [testenv:dev.k8s.destroy]
 description = Destroy local Kubernetes cluster
 changedir = {toxinidir}/packages/grid