example_airflow_worker_config.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: "4"
  creationTimestamp: "2020-10-05T15:56:07Z"
  generation: 4
  labels:
    run: airflow-worker
  name: airflow-worker
  namespace: composer-1-12-1-airflow-1-10-10-4d4b34a3
  resourceVersion: "101165"
  selfLink: /apis/apps/v1/namespaces/composer-1-12-1-airflow-1-10-10-4d4b34a3/deployments/airflow-worker
  uid: 343a5694-7b9e-490e-9f20-c58776c0aeb4
spec:
  progressDeadlineSeconds: 600
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      run: airflow-worker
  strategy:
    type: Recreate
  template:
    metadata:
      creationTimestamp: null
      labels:
        run: airflow-worker
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: run
                      operator: In
                      values:
                        - airflow-worker
                namespaces:
                  - composer-1-12-1-airflow-1-10-10-4d4b34a3
                topologyKey: kubernetes.io/hostname
              weight: 100
      containers:
        - args:
            - worker
          env:
            - name: CLOUDSDK_METRICS_ENVIRONMENT
              value: 1.10.10+composer
            - name: GCS_BUCKET
              value: us-east1-testing-etl-4d4b34a3-bucket
            - name: GCP_PROJECT
              value: test-project-291320
            - name: COMPOSER_LOCATION
              value: us-east1
            - name: COMPOSER_GKE_ZONE
              value: us-east1-b
            - name: COMPOSER_GKE_NAME
              value: us-east1-testing-etl-4d4b34a3-gke
            - name: COMPOSER_PYTHON_VERSION
              value: "3"
            - name: COMPOSER_ENVIRONMENT
              value: testing-etl
            - name: AIRFLOW_HOME
              value: /etc/airflow
            - name: DAGS_FOLDER
              value: /home/airflow/gcs/dags
            - name: SQL_DATABASE
              value: composer-1-12-1-airflow-1-10-10-4d4b34a3
            - name: SQL_USER
              value: root
            - name: SQL_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: sql_password
                  name: airflow-secrets
            - name: AIRFLOW__WEBSERVER__BASE_URL
              value: https://fd8f0e16236bbdd0ep-tp.appspot.com
            - name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
              value: mysql+mysqldb://$(SQL_USER):$(SQL_PASSWORD)@airflow-sqlproxy-service.default.svc.cluster.local/$(SQL_DATABASE)?charset=utf8
            - name: AIRFLOW__CORE__FERNET_KEY
              valueFrom:
                secretKeyRef:
                  key: fernet_key
                  name: airflow-secrets
            - name: C_FORCE_ROOT
              value: "TRUE"
          image: us.gcr.io/test-project-291320/ceb7f107-71af-4213-a7b1-3f385d02ccf3
          imagePullPolicy: IfNotPresent
          lifecycle:
            postStart:
              exec:
                command:
                  - /bin/bash
                  - /etc/scripts/poststart.sh
            preStop:
              exec:
                command:
                  - bash
                  - -c
                  - pkill -f "MainProcess"
          livenessProbe:
            exec:
              command:
                - /var/local/worker_checker.py
            failureThreshold: 3
            initialDelaySeconds: 120
            periodSeconds: 60
            successThreshold: 1
            timeoutSeconds: 1
          name: airflow-worker
          ports:
            - containerPort: 8793
              protocol: TCP
          resources: {}
          securityContext:
            privileged: true
            runAsUser: 0
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /etc/airflow/airflow_cfg
              name: airflow-config
            - mountPath: /home/airflow/gcs
              name: gcsdir
            - mountPath: /var/run/docker.sock
              name: docker-host
            - mountPath: /bin/docker
              name: docker-app
            - mountPath: /home/airflow/etlData
              name: etl-data
            - mountPath: /etc/scripts
              name: config-volume
        - args:
            - /home/airflow/gcs
          env:
            - name: GCS_BUCKET
              value: us-east1-testing-etl-4d4b34a3-bucket
            - name: SQL_DATABASE
              value: composer-1-12-1-airflow-1-10-10-4d4b34a3
            - name: SQL_USER
              value: root
            - name: SQL_PASSWORD
              valueFrom:
                secretKeyRef:
                  key: sql_password
                  name: airflow-secrets
            - name: COMPOSER_GKE_ZONE
              value: us-east1-b
            - name: COMPOSER_GKE_NAME
              value: us-east1-testing-etl-4d4b34a3-gke
          image: us.gcr.io/cloud-airflow-releaser/gcs-syncd:cloud_composer_service_2020-09-22-RC0
          imagePullPolicy: IfNotPresent
          name: gcs-syncd
          resources:
            limits:
              memory: 600Mi
            requests:
              cpu: 10m
              memory: 50Mi
          terminationMessagePath: /dev/termination-log
          terminationMessagePolicy: File
          volumeMounts:
            - mountPath: /home/airflow/gcs
              name: gcsdir
      dnsPolicy: ClusterFirst
      nodeSelector:
        cloud.google.com/gke-nodepool: default-pool
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
        - configMap:
            defaultMode: 420
            name: airflow-configmap
          name: airflow-config
        - emptyDir: {}
          name: gcsdir
        - hostPath:
            path: /var/run/docker.sock
            type: ""
          name: docker-host
        - hostPath:
            path: /usr/bin/docker
            type: ""
          name: docker-app
        - hostPath:
            path: /home/airflow/etlData
            type: DirectoryOrCreate
          name: etl-data
        - configMap:
            defaultMode: 511
            name: start-config
          name: config-volume
status:
  availableReplicas: 3
  conditions:
    - lastTransitionTime: "2020-10-05T19:28:33Z"
      lastUpdateTime: "2020-10-05T19:28:41Z"
      message: ReplicaSet "airflow-worker-745846fc95" has successfully progressed.
      reason: NewReplicaSetAvailable
      status: "True"
      type: Progressing
    - lastTransitionTime: "2020-10-05T22:55:11Z"
      lastUpdateTime: "2020-10-05T22:55:11Z"
      message: Deployment has minimum availability.
      reason: MinimumReplicasAvailable
      status: "True"
      type: Available
  observedGeneration: 4
  readyReplicas: 3
  replicas: 3
  updatedReplicas: 3