feat(chart): expose all services through ALB (#2713)

* feat(chart): expose all services through ALB * feat(chart): expose all services through ALB * feat(chart): add staging env * feat(chart): use common template for ingress TLS * feat(chart): add openapi redirect on github * feat(chart): feedback fix * feat(chart): block /metrics on alb * feat(chart): fix lint
huggingface · May 15, 2024 · 4a9d682 · 4a9d682
1 parent 58c3ce9
commit 4a9d682
Show file tree

Hide file tree

Showing 10 changed files with 187 additions and 24 deletions.
diff --git a/chart/env/prod.yaml b/chart/env/prod.yaml
@@ -302,6 +302,11 @@ admin:
   replicas: 2
   service:
     type: NodePort
+  ingress:
+    enabled: true
+    annotations:
+      alb.ingress.kubernetes.io/group.order: "1"
+      alb.ingress.kubernetes.io/target-node-labels: role-datasets-server=true
   resources:
     requests:
       cpu: 1
@@ -323,6 +328,12 @@ api:
   replicas: 12
   service:
     type: NodePort
+  ingress:
+    enabled: true
+    annotations:
+      alb.ingress.kubernetes.io/target-node-labels: role-datasets-server-api=true
+      alb.ingress.kubernetes.io/actions.openapi-redirect: '{"Type":"redirect","RedirectConfig":{"Host":"raw.githubusercontent.com","Path":"/huggingface/dataset-viewer/main/docs/source/openapi.json","Port":"443","Protocol":"HTTPS","Query":"#{query}","StatusCode":"HTTP_307"}}'
+      alb.ingress.kubernetes.io/actions.metrics-unauthorized: '{"type":"fixed-response","fixedResponseConfig":{"contentType":"text/plain","statusCode":"401","messageBody":"401 Unauthorized"}}'
   resources:
     requests:
       cpu: 4
@@ -342,6 +353,11 @@ rows:
   replicas: 12
   service:
     type: NodePort
+  ingress:
+    enabled: true
+    annotations:
+      alb.ingress.kubernetes.io/group.order: "2"
+      alb.ingress.kubernetes.io/target-node-labels: role-datasets-server-rows=true
   resources:
     requests:
       cpu: 1
@@ -364,7 +380,7 @@ search:
   ingress:
     enabled: true
     annotations:
-      alb.ingress.kubernetes.io/group.order: "1"
+      alb.ingress.kubernetes.io/group.order: "3"
       alb.ingress.kubernetes.io/target-node-labels: role-datasets-server-search=true
       alb.ingress.kubernetes.io/target-group-attributes: stickiness.enabled=true,stickiness.lb_cookie.duration_seconds=300
       alb.ingress.kubernetes.io/target-type: ip
@@ -386,6 +402,11 @@ sseApi:
   replicas: 2
   service:
     type: NodePort
+  ingress:
+    enabled: true
+    annotations:
+      alb.ingress.kubernetes.io/group.order: "4"
+      alb.ingress.kubernetes.io/target-node-labels: role-datasets-server=true
   resources:
     requests:
       cpu: 1

diff --git a/chart/env/staging.yaml b/chart/env/staging.yaml
@@ -206,6 +206,7 @@ ingress:
     alb.ingress.kubernetes.io/scheme: "internet-facing"
     alb.ingress.kubernetes.io/group.name: "datasets-server"
     kubernetes.io/ingress.class: "alb"
+    alb.ingress.kubernetes.io/group.order: "100"
 
 # --- services ---
 
@@ -214,6 +215,10 @@ admin:
   replicas: 1
   service:
     type: NodePort
+  ingress:
+    enabled: true
+    annotations:
+      alb.ingress.kubernetes.io/group.order: "1"
   resources:
     requests:
       cpu: 100m
@@ -227,6 +232,8 @@ api:
   replicas: 1
   service:
     type: NodePort
+  ingress:
+    enabled: true
   resources:
     requests:
       cpu: 100m
@@ -240,6 +247,10 @@ rows:
   replicas: 1
   service:
     type: NodePort
+  ingress:
+    enabled: true
+    annotations:
+      alb.ingress.kubernetes.io/group.order: "2"
   resources:
     requests:
       cpu: 100m
@@ -253,6 +264,10 @@ search:
   replicas: 1
   service:
     type: NodePort
+  ingress:
+    enabled: true
+    annotations:
+      alb.ingress.kubernetes.io/group.order: "3"
   resources:
     requests:
       cpu: 100m

diff --git a/chart/templates/_common/_ingressTls.tpl b/chart/templates/_common/_ingressTls.tpl
@@ -0,0 +1,16 @@
+{{/*
+TLS part of ingress template
+*/}}
+{{- define "ingress.tls" -}}
+{{- if include "hf.common.ingress.certManagerRequest" ( dict "annotations" .annotations ) }}
+tls:
+  - hosts:
+      - {{ include "datasetsServer.ingress.hostname" . }}
+    secretName: {{ printf "%s-tls" (include "datasetsServer.ingress.hostname" $) }}
+{{- else if .Values.ingress.tls -}}
+{{- with .Values.ingress.tls }}
+tls:
+  {{- tpl (toYaml .) $ | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/chart/templates/ingress.yaml b/chart/templates/ingress.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.global.huggingface.ingress.enabled .Values.ingress.enabled -}}
+{{- if and .Values.global.huggingface.ingress.enabled .Values.ingress.enabled .Values.reverseProxy.ingress.enabled -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -19,15 +19,5 @@ spec:
                   name: http
             path: /
             pathType: Prefix
-  {{- if include "hf.common.ingress.certManagerRequest" ( dict "annotations" $annotations ) }}
-  tls:
-    - hosts:
-        - {{ include "datasetsServer.ingress.hostname" . }}
-      secretName: {{ printf "%s-tls" (include "datasetsServer.ingress.hostname" .) }}
-  {{- else if .Values.ingress.tls -}}
-  {{- with .Values.ingress.tls }}
-  tls:
-    {{- tpl (toYaml .) $ | nindent 4 }}
-  {{- end }}
-  {{- end }}
+{{- include "ingress.tls" (merge (dict "annotations" $annotations) $ ) | indent 2}}
 {{- end }}
diff --git a/chart/templates/services/admin/ingress.yaml b/chart/templates/services/admin/ingress.yaml
@@ -0,0 +1,24 @@
+{{- if and .Values.global.huggingface.ingress.enabled .Values.ingress.enabled .Values.admin.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  {{- $annotations := fromYaml (include "datasetsServer.instance.ingress.annotations" (dict "instance" .Values.admin "context" $ )) }}
+  annotations: {{ toYaml $annotations | nindent 4 }}
+  labels: {{ include "labels.admin" . | nindent 4 }}
+  name: "{{ include "name" . }}-admin"
+  namespace: {{ .Release.Namespace }}
+spec:
+  rules:
+    - host: {{ include "datasetsServer.ingress.hostname" . }}
+      http:
+        paths:
+          - path: /admin
+            backend:
+              service:
+                name: "{{ include "name" . }}-admin"
+                port:
+                  name: http
+
+            pathType: Prefix
+{{- include "ingress.tls" (merge (dict "annotations" $annotations) $ ) | indent 2}}
+{{- end }}
diff --git a/chart/templates/services/api/ingress.yaml b/chart/templates/services/api/ingress.yaml
@@ -0,0 +1,37 @@
+{{- if and .Values.global.huggingface.ingress.enabled .Values.ingress.enabled .Values.api.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  {{- $annotations := fromYaml (include "datasetsServer.instance.ingress.annotations" (dict "instance" .Values.api "context" $ )) }}
+  annotations: {{ toYaml $annotations | nindent 4}}
+  labels: {{ include "labels.api" . | nindent 4 }}
+  name: "{{ include "name" . }}-api"
+  namespace: {{ .Release.Namespace }}
+spec:
+  rules:
+    - host: {{ include "datasetsServer.ingress.hostname" . }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: "{{ include "name" . }}-api"
+                port:
+                  name: http
+            path: /
+            pathType: Prefix
+          {{ if hasKey $annotations "alb.ingress.kubernetes.io/actions.openapi-redirect" -}}
+          - path: /openapi.json
+            pathType: Exact
+            backend:
+              serviceName: openapi-redirect
+              servicePort: use-annotation
+          {{- end }}
+          {{ if hasKey $annotations "alb.ingress.kubernetes.io/actions.metrics-unauthorized" -}}
+          - path: /metrics
+            pathType: Exact
+            backend:
+              serviceName: metrics-unauthorized
+              servicePort: use-annotation
+          {{- end -}}
+{{- include "ingress.tls" (merge (dict "annotations" $annotations) $ ) | indent 2}}
+{{- end }}
diff --git a/chart/templates/services/rows/ingress.yaml b/chart/templates/services/rows/ingress.yaml
@@ -0,0 +1,23 @@
+{{- if and .Values.global.huggingface.ingress.enabled .Values.ingress.enabled .Values.rows.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  {{- $annotations := fromYaml (include "datasetsServer.instance.ingress.annotations" (dict "instance" .Values.rows "context" $ )) }}
+  annotations: {{ toYaml $annotations | nindent 4}}
+  labels: {{ include "labels.rows" . | nindent 4 }}
+  name: "{{ include "name" . }}-rows"
+  namespace: {{ .Release.Namespace }}
+spec:
+  rules:
+    - host: {{ include "datasetsServer.ingress.hostname" . }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: "{{ include "name" . }}-rows"
+                port:
+                  name: http
+            path: /rows
+            pathType: Prefix
+{{- include "ingress.tls" (merge (dict "annotations" $annotations) $ ) | indent 2}}
+{{- end }}
diff --git a/chart/templates/services/search/ingress.yaml b/chart/templates/services/search/ingress.yaml
@@ -19,15 +19,15 @@ spec:
                   name: http
             path: /search
             pathType: Prefix
-  {{- if include "hf.common.ingress.certManagerRequest" ( dict "annotations" $annotations ) }}
-  tls:
-    - hosts:
-        - {{ include "datasetsServer.ingress.hostname" . }}
-      secretName: {{ printf "%s-tls" (include "datasetsServer.ingress.hostname" .) }}
-  {{- else if .Values.ingress.tls -}}
-  {{- with .Values.ingress.tls }}
-  tls:
-    {{- tpl (toYaml .) $ | nindent 4 }}
-  {{- end }}
-  {{- end }}
+    - host: {{ include "datasetsServer.ingress.hostname" . }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: "{{ include "name" . }}-search"
+                port:
+                  name: http
+            path: /filter
+            pathType: Prefix
+{{- include "ingress.tls" (merge (dict "annotations" $annotations) $ ) | indent 2}}
 {{- end }}
diff --git a/chart/templates/services/sse-api/ingress.yaml b/chart/templates/services/sse-api/ingress.yaml
@@ -0,0 +1,23 @@
+{{- if and .Values.global.huggingface.ingress.enabled .Values.ingress.enabled .Values.sseApi.ingress.enabled -}}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  {{- $annotations := fromYaml (include "datasetsServer.instance.ingress.annotations" (dict "instance" .Values.sseApi "context" $ )) }}
+  annotations: {{ toYaml $annotations | nindent 4}}
+  labels: {{ include "labels.sseApi" . | nindent 4 }}
+  name: "{{ include "name" . }}-sse-api"
+  namespace: {{ .Release.Namespace }}
+spec:
+  rules:
+    - host: {{ include "datasetsServer.ingress.hostname" . }}
+      http:
+        paths:
+          - backend:
+              service:
+                name: "{{ include "name" . }}-sse-api"
+                port:
+                  name: http
+            path: /sse
+            pathType: Prefix
+{{- include "ingress.tls" (merge (dict "annotations" $annotations) $ ) | indent 2}}
+{{- end }}
diff --git a/chart/values.yaml b/chart/values.yaml
@@ -390,6 +390,8 @@ reverseProxy:
     type: ""
     annotations: {}
   tolerations: []
+  ingress:
+    enabled: false
 
 ingress:
   enabled: true
@@ -431,6 +433,9 @@ admin:
   service:
     type: ""
     annotations: {}
+  ingress:
+    enabled: true
+    annotations: {}
   tolerations: []
 
 hf:
@@ -471,6 +476,9 @@ api:
   service:
     type: ""
     annotations: {}
+  ingress:
+    enabled: true
+    annotations: {}
   tolerations: []
 
 rows:
@@ -498,6 +506,9 @@ rows:
   service:
     type: ""
     annotations: {}
+  ingress:
+    enabled: true
+    annotations: {}
   tolerations: []
 
 search:
@@ -555,6 +566,9 @@ sseApi:
   service:
     type: ""
     annotations: {}
+  ingress:
+    enabled: true
+    annotations: {}
   tolerations: []
 
 workers: