chore: refactor to call secrets manager

JIRA:CLOUDSEC-12
edx · Oct 12, 2023 · a581dc3 · a581dc3
1 parent b1ddbbc
commit a581dc3
Show file tree

Hide file tree

Showing 8 changed files with 63 additions and 0 deletions.
diff --git a/dataeng/resources/secrets-manager.sh b/dataeng/resources/secrets-manager.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+secret_to_call="$1"
+secret_name="$2"
+set +x
+
+SECRET_JSON=$(aws secretsmanager get-secret-value --secret-id $secret_to_call --region "us-east-1" --output json)
+# Check the exit status of the AWS CLI command
+
+echo "$SECRET_JSON"
+extract_and_store_secret_value() {
+
+    value=$(echo "$SECRET_JSON" | jq -r ".SecretString | fromjson.$secret_name" 2>/dev/null)
+    eval "$secret_name"='$value'
+}
+
+if [ $? -eq 0 ]; then
+    # Use jq to extract the values from the JSON response
+    extract_and_store_secret_value $SECRET_JSON $secret_name
+else
+    echo "AWS CLI command failed"
+fi
diff --git a/dataeng/resources/snowflake-collect-metrics.sh b/dataeng/resources/snowflake-collect-metrics.sh
@@ -10,6 +10,12 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS KEY_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS PASSPHRASE_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS USER
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS ACCOUNT
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS METRIC_NAME
+
 python collect-metrics.py \
     --metric_name $METRIC_NAME \
     --key_path $WORKSPACE/analytics-secure/snowflake/rsa_key_snowflake_task_automation_user.p8 \

diff --git a/dataeng/resources/snowflake-demographics-cleanup.sh b/dataeng/resources/snowflake-demographics-cleanup.sh
@@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS KEY_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS PASSPHRASE_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS USER
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS ACCOUNT
+
 python demographics_cleanup.py \
     --key_path $WORKSPACE/analytics-secure/$KEY_PATH \
     --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \

diff --git a/dataeng/resources/snowflake-public-grants-cleaner.sh b/dataeng/resources/snowflake-public-grants-cleaner.sh
@@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS KEY_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS PASSPHRASE_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS USER
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS ACCOUNT
+
 python snowflake_public_grants_cleaner.py \
     --key_path $WORKSPACE/analytics-secure/$KEY_PATH \
     --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \

diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh
@@ -10,6 +10,17 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS KEY_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PASSPHRASE_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS USER
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS ACCOUNT
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS SCHEMA
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PIPE_NAME
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS TABLE_NAME
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS DELAY
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS LIMIT
+
+
 python refresh_snowpipe.py \
     --key_path $WORKSPACE/analytics-secure/$KEY_PATH \
     --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \

diff --git a/dataeng/resources/snowflake-user-retirement-status-cleanup.sh b/dataeng/resources/snowflake-user-retirement-status-cleanup.sh
@@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS KEY_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS PASSPHRASE_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS USER
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS ACCOUNT
+
 python retirement_cleanup.py \
     --key_path $WORKSPACE/analytics-secure/$KEY_PATH \
     --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \

diff --git a/dataeng/resources/snowflake-validate-stitch.sh b/dataeng/resources/snowflake-validate-stitch.sh
@@ -14,6 +14,11 @@ COMPARISON_START_TIME=$(date --utc --iso=minutes -d "${COMPARISON_END_TIME} - 15
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS KEY_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS PASSPHRASE_PATH
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS USER
+source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS ACCOUNT
+
 python stitch_vs_sqoop_validation.py \
     --key_path $WORKSPACE/analytics-secure/${SNOWFLAKE_KEY_PATH} \
     --passphrase_path $WORKSPACE/analytics-secure/${SNOWFLAKE_PASSPHRASE_PATH} \

diff --git a/dataeng/resources/stitch-snowflake-lag-monitor.sh b/dataeng/resources/stitch-snowflake-lag-monitor.sh
@@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate"
 cd $WORKSPACE/analytics-tools/snowflake
 make requirements
 
+source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS KEY_PATH
+source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS PASSPHRASE_PATH
+source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS USER
+source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS ACCOUNT
+
 python stitch-snowflake-monitoring.py \
     --key_path $WORKSPACE/analytics-secure/$KEY_PATH \
     --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \