add service pre condition

jasondavindev · Nov 25, 2021 · d3e8051 · d3e8051
1 parent 9d26597
commit d3e8051
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 0 deletions.
diff --git a/airflow/entrypoint.sh b/airflow/entrypoint.sh
@@ -6,6 +6,41 @@ function bootstrap {
   done
 }
 
+function wait_for_it()
+{
+  local serviceport=$1
+  local service=${serviceport%%:*}
+  local port=${serviceport#*:}
+  local retry_seconds=5
+  local max_try=100
+  let i=1
+
+  nc -z $service $port
+  result=$?
+
+  until [ $result -eq 0 ]; do
+    echo "[$i/$max_try] check for ${service}:${port}..."
+    echo "[$i/$max_try] ${service}:${port} is not available yet"
+    if (( $i == $max_try )); then
+      echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/"
+      exit 1
+    fi
+
+    echo "[$i/$max_try] try in ${retry_seconds}s once again ..."
+    let "i++"
+    sleep $retry_seconds
+
+    nc -z $service $port
+    result=$?
+  done
+  echo "[$i/$max_try] $service:${port} is available."
+}
+
+for i in ${SERVICE_PRECONDITION[@]}
+do
+  wait_for_it ${i}
+done
+
 case "$1" in
   webserver)
     bootstrap

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -16,6 +16,7 @@ x-airflow-common: &airflow-common
     HDFS_PORT: "8020"
     INGESTION_API_URL: "http://api:3000"
     SCRIPTS_PATH_PREFIX: /scripts/spark/
+    SERVICE_PRECONDITION: airflow-db:5432
   depends_on:
     - airflow-db
 
@@ -77,6 +78,7 @@ services:
     command: "hive --service metastore"
     environment:
       NODE_TYPE: metastore
+      SERVICE_PRECONDITION: hive-postgres:5432 namenode:50070
     ports:
       - 9083:9083
 

diff --git a/hadoop/docker/hive/entrypoint.sh b/hadoop/docker/hive/entrypoint.sh
@@ -1,4 +1,39 @@
 #!/bin/bash
+function wait_for_it()
+{
+  local serviceport=$1
+  local service=${serviceport%%:*}
+  local port=${serviceport#*:}
+  local retry_seconds=5
+  local max_try=100
+  let i=1
+
+  nc -z $service $port
+  result=$?
+
+  until [ $result -eq 0 ]; do
+    echo "[$i/$max_try] check for ${service}:${port}..."
+    echo "[$i/$max_try] ${service}:${port} is not available yet"
+    if (( $i == $max_try )); then
+      echo "[$i/$max_try] ${service}:${port} is still not available; giving up after ${max_try} tries. :/"
+      exit 1
+    fi
+
+    echo "[$i/$max_try] try in ${retry_seconds}s once again ..."
+    let "i++"
+    sleep $retry_seconds
+
+    nc -z $service $port
+    result=$?
+  done
+  echo "[$i/$max_try] $service:${port} is available."
+}
+
+for i in ${SERVICE_PRECONDITION[@]}
+do
+  wait_for_it ${i}
+done
+
 if [ ! -f populated -a "$NODE_TYPE" == "metastore" ]; then
     schematool -dbType postgres -initSchema
     touch populated

diff --git a/kubernetes/airflow/scheduler.yml b/kubernetes/airflow/scheduler.yml
@@ -24,6 +24,8 @@ spec:
               value: airflow-postgres-svc.airflow.svc.cluster.local
             - name: POSTGRES_PORT
               value: "5432"
+            - name: SERVICE_PRECONDITION
+              value: airflow-postgres-svc.airflow.svc.cluster.local:5432
             - name: EXECUTOR
               value: KubernetesExecutor
             - name: HDFS_HOST

diff --git a/kubernetes/airflow/webserver.yml b/kubernetes/airflow/webserver.yml
@@ -24,6 +24,8 @@ spec:
               value: airflow-postgres-svc.airflow.svc.cluster.local
             - name: POSTGRES_PORT
               value: "5432"
+            - name: SERVICE_PRECONDITION
+              value: airflow-postgres-svc.airflow.svc.cluster.local:5432
             - name: EXECUTOR
               value: KubernetesExecutor
             - name: HDFS_HOST

diff --git a/kubernetes/hive/metastore.yml b/kubernetes/hive/metastore.yml
@@ -24,6 +24,8 @@ spec:
           env:
             - name: NODE_TYPE
               value: metastore
+            - name: SERVICE_PRECONDITION
+              value: hive-postgres-svc.dataplatform.svc.cluster.local:5432 namenode-svc.dataplatform.svc.cluster.local:50070
           args:
             - "bash"
             - "-c"