Skip to content

Commit

Permalink
chore: fix refresh job
Browse files Browse the repository at this point in the history
  • Loading branch information
katebygrace committed Apr 17, 2024
1 parent 1af59b6 commit a6df4ff
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 26 deletions.
2 changes: 1 addition & 1 deletion dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
Expand Up @@ -82,7 +82,7 @@ class SnowflakeRefreshSnowpipe {
}
publishers common_publishers(allVars)
steps {
shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh'))
shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager.sh'))
shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-refresh-snowpipe.sh'))
}
}
Expand Down
71 changes: 49 additions & 22 deletions dataeng/resources/secrets-manager.sh
@@ -1,38 +1,65 @@
#!/bin/bash

# Define the location of the script in the Jenkins workspace
SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"

echo "running setup"

# Write the script content to the specified location
cat <<EOF > "$SCRIPT_PATH"
#!/usr/bin/env bash
extract_value_from_json() {
local json="$1"
local key="$2"
local value=$(echo "$json" | jq -r ".$key")
local secret_json="\$1"
local secret_key="\$2"
local secret_value=$(echo "$secret_json" | jq -r ".$secret_key")
}
fetch_whole_secret() {
local secret_name="$1"
local variable_name="$2"
local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
local secret_name="\$1"
local variable_name="\$2"
SECRET_JSON=\$(aws secretsmanager get-secret-value --secret-id "\$secret_name" --region "us-east-1" --output json)
echo "\$SECRET_JSON"
value=\$(echo "\$SECRET_JSON" | jq -r ".SecretString | fromjson.\"\$variable_name\"")
echo $value
#set whole file as env var
declare "${secret_name%=*}=${secret_value}"
echo "$secret_value" > "$WORKSPACE/$variable_name"
# Output the contents of the file to verify
cat "$WORKSPACE/variable_name"
#declare "$variable_name=$secret_value"
#declare "$variable_name=\"$secret_value\""
#what brian said to do
#declare "${secret_name%=*}=${secret_value}"
}
fetch_specific_key() {
local secret_name="$1"
local key="$2"
local secret_name="\$1"
local key="\$2"
local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
local extracted_value=$(extract_value_from_json "$secret_value" "$key")
declare "${key%=*}=${extracted_value}"
}
# Main script
if [[ "$1" == "-w" ]]; then
if [ $# -ne 3 ]; then
echo "Usage: $0 -w <name_of_file> <name_of_variable>"
exit 1
secret_script() {
echo "\$1"
echo "\$2"
echo "\$3"
if [[ "\$1" == "-w" ]]; then
if [ \$# -ne 3 ]; then
echo "Usage: $0 -w <name_of_file> <name_of_variable>"
exit 1
fi
fetch_whole_secret "\$2" "\$3"
else
if [ $# -ne 2 ]; then
echo "Usage: $0 <name_of_file> <name_of_key>"
exit 1
fi
fetch_specific_key "\$1" "\$2"
fi
fetch_whole_secret "$2" "$3"
else
if [ $# -ne 2 ]; then
echo "Usage: $0 <name_of_file> <name_of_key>"
exit 1
fi
fetch_specific_key "$1" "$2"
fi
}
EOF
13 changes: 10 additions & 3 deletions dataeng/resources/snowflake-refresh-snowpipe.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
set -ex


# Creating Python virtual env
PYTHON_VENV="python_venv"
virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}"
Expand All @@ -10,14 +11,18 @@ source "${PYTHON_VENV}/bin/activate"
cd $WORKSPACE/analytics-tools/snowflake
make requirements

# Define the location of the script in the Jenkins workspace
SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"


# Source the secrets-manager.sh script to make the function available
source $WORKSPACE/secrets-manager.sh
# Fetch the secrets from AWS
set +x


secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_snowpipe_user.p8
secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_passphrase_snowpipe_user
echo "Running secrets manager script"
secret_script -w analytics-secure/snowflake/rsa_key_snowpipe_user.p8 rsa_key_snowpipe_user
secret_script -w analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user rsa_key_passphrase_snowpipe_user

set -x

Expand All @@ -31,3 +36,5 @@ python refresh_snowpipe.py \
--limit $LIMIT
--key_file $KEY_PATH \
--passphrase_file $PASSPHRASE_PATH


0 comments on commit a6df4ff

Please sign in to comment.