From c1817cbe043f81aff1b43a1869f9d545685a034a Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Thu, 28 Mar 2024 16:39:27 -0700 Subject: [PATCH] add horizontal pod autoscaler for backend and frontend via helm charts (#1633) Supports horizontal pod autoscaling (hpa) for backend and frontend pods: - use cpu and memory averages - adjust base memory + cpu for backend - threshold set to 80% cpu and 95% memory utilization by default (configurable in values.yaml) - instead of backend and frontend replicas, set max replicas in values.yaml - only enable hpa if backend_max_replicas or frontend_max_replicas is >1, default to 1 for now --- chart/templates/backend.yaml | 32 +++++++++++++++++++++++++++++++- chart/templates/frontend.yaml | 32 ++++++++++++++++++++++++++++++-- chart/values.yaml | 28 ++++++++++++++++++++++++---- docs/deploy/customization.md | 30 ++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 7 deletions(-) diff --git a/chart/templates/backend.yaml b/chart/templates/backend.yaml index dd148d6352..ea9c744183 100644 --- a/chart/templates/backend.yaml +++ b/chart/templates/backend.yaml @@ -10,7 +10,9 @@ spec: matchLabels: app: {{ .Values.name }} role: backend - replicas: {{ .Values.backend_num_replicas }} +{{- if eq (int .Values.backend_max_replicas) 1 }} + replicas: 1 +{{- end }} template: metadata: labels: @@ -232,3 +234,31 @@ spec: - protocol: TCP port: {{ .Values.opPort }} name: operator + +{{- if gt (int .Values.backend_max_replicas) 1 }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: backend-autoscaler +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.name }}-backend + minReplicas: 1 + maxReplicas: {{ .Values.backend_max_replicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.backend_avg_cpu_threshold }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.backend_avg_memory_threshold }} +{{- end }} diff --git a/chart/templates/frontend.yaml b/chart/templates/frontend.yaml index f3a3d39a23..4527858630 100644 --- a/chart/templates/frontend.yaml +++ b/chart/templates/frontend.yaml @@ -10,7 +10,9 @@ spec: matchLabels: app: {{ .Values.name }} role: frontend - replicas: {{ .Values.frontend_num_replicas | default 1 }} + {{- if eq (int .Values.frontend_max_replicas) 1 }} + replicas: 1 + {{- end }} template: metadata: labels: @@ -110,4 +112,30 @@ spec: {{- end }} - +{{- if gt (int .Values.frontend_max_replicas) 1 }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: frontend-autoscaler +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Values.name }}-frontend + minReplicas: 1 + maxReplicas: {{ .Values.frontend_max_replicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.frontend_avg_cpu_threshold }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.frontend_avg_memory_threshold }} +{{ end }} diff --git a/chart/values.yaml b/chart/values.yaml index cbae7fb46d..f2cea90de6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -91,13 +91,10 @@ backend_pull_policy: "Always" backend_password_secret: "PASSWORD!" -# number of backend pods -backend_num_replicas: 1 - # number of workers per pod backend_workers: 1 -backend_cpu: "25m" +backend_cpu: "100m" backend_memory: "350Mi" @@ -114,6 +111,17 @@ profile_browser_idle_seconds: 60 # mostly intended for debugging / testing # log_failed_crawl_lines: 200 +# Autoscale +# --------- +# max number of backend pods to scale to +# if > 1, will enable HPA for backend +backend_max_replicas: 1 + +# scale up if avg cpu utilization exceeds +backend_avg_cpu_threshold: 80 + +# scale up if avg memory utilization exceeds +backend_avg_memory_threshold: 95 # Nginx Image # ========================================= @@ -132,6 +140,18 @@ local_service_port: 30870 frontend_alias: "http://browsertrix-cloud-frontend" +# Autoscaling +# ----------- +# max number of backend pods to scale to +# if > 1, will enable HPA for frontend +frontend_max_replicas: 1 + +# scale up if avg cpu utilization exceeds +frontend_avg_cpu_threshold: 80 + +# scale up if avg memory utilization exceeds +frontend_avg_memory_threshold: 95 + # MongoDB Image # ========================================= diff --git a/docs/deploy/customization.md b/docs/deploy/customization.md index f145ff7fd3..e50349e97a 100644 --- a/docs/deploy/customization.md +++ b/docs/deploy/customization.md @@ -78,6 +78,36 @@ storages: endpoint_url: "http://s3provider.example.com" ``` +## Horizontal Autoscaling + +Browsertrix also includes support for horizontal auto-scaling for both the backend and frontend pods. +The auto-scaling will start a new pod when memory/cpu utilization reaches the thresholds. + +To use auto-scaling, the [metrics-server](https://github.com/kubernetes-sigs/metrics-server) cluster add-on is required. +Many k8s provides include metrics server by default, others, like MicroK8S, make it available as an add-on. + +To enable auto-scaling, set `backend_max_replicas` and/or `frontend_max_replicas` to a value >1. + +```yaml +backend_max_replicas: 2 + +frontend_max_replicas: 2 +``` + +By default, the auto-scaling uses the following thresholds for deciding when to start a new pod can also +be modified. The default values are: + +```yaml +backend_avg_cpu_threshold: 80 + +backend_avg_memory_threshold: 95 + +frontend_avg_cpu_threshold: 80 + +frontend_avg_memory_threshold: 95 +``` + + ## Email / SMTP Server Browsertrix sends user invitations, password resets, background job failure notifications, and other important messages via email. The `email` setting can be used to configure the SMTP server used to send emails. To avoid email messages from Browsertrix being flagged as spam, be sure to use the same domain for `sender_email` and `reply_to_email`.