Skip to content

Commit

Permalink
chore: fix refresh job
Browse files Browse the repository at this point in the history
  • Loading branch information
katebygrace committed May 6, 2024
1 parent 1af59b6 commit 6f59269
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 34 deletions.
2 changes: 1 addition & 1 deletion dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
Expand Up @@ -82,7 +82,7 @@ class SnowflakeRefreshSnowpipe {
}
publishers common_publishers(allVars)
steps {
shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager-setup.sh'))
shell(dslFactory.readFileFromWorkspace('dataeng/resources/secrets-manager.sh'))
shell(dslFactory.readFileFromWorkspace('dataeng/resources/snowflake-refresh-snowpipe.sh'))
}
}
Expand Down
74 changes: 51 additions & 23 deletions dataeng/resources/secrets-manager.sh
@@ -1,38 +1,66 @@
#!/bin/bash

# Define the location of the script in the Jenkins workspace
SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"

echo "running setup"

# Write the script content to the specified location
cat <<EOF > "$SCRIPT_PATH"
#!/usr/bin/env bash
extract_value_from_json() {
local json="$1"
local key="$2"
local value=$(echo "$json" | jq -r ".$key")
local secret_json="\$1"
local secret_key="\$2"
local secret_value=\$(echo "$secret_json" | jq -r ".$secret_key")
}
fetch_whole_secret() {
local secret_name="$1"
local variable_name="$2"
local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
#set whole file as env var
declare "${secret_name%=*}=${secret_value}"
local secret_name="\$1"
local variable_name="\$2"
echo "\$secret_name"
echo "\$variable_name"
SECRET_JSON=\$(aws secretsmanager get-secret-value --secret-id "\$secret_name" --region "us-east-1" --output json)
echo "\$SECRET_JSON"
value=\$(echo "\$SECRET_JSON" | jq -r ".SecretString" 2>/dev/null)
echo "\$value"
echo "\$value" > "\$WORKSPACE/\$variable_name"
# Output the contents of the file to verify
cat "\$WORKSPACE/\$variable_name"
declare "\${variable_name%=*}=\${value}"
#declare "$variable_name=$secret_value"
#declare "$variable_name=\"$secret_value\""
#what brian said to do
#declare "\${variable_name%=*}=\${value}"
}
fetch_specific_key() {
local secret_name="$1"
local key="$2"
local secret_name="\$1"
local key="\$2"
local secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query "SecretString" --output text)
local extracted_value=$(extract_value_from_json "$secret_value" "$key")
declare "${key%=*}=${extracted_value}"
}
# Main script
if [[ "$1" == "-w" ]]; then
if [ $# -ne 3 ]; then
echo "Usage: $0 -w <name_of_file> <name_of_variable>"
exit 1
secret_script() {
echo "\$1"
echo "\$2"
echo "\$3"
if [[ "\$1" == "-w" ]]; then
if [ \$# -ne 3 ]; then
echo "Usage: $0 -w <name_of_file> <name_of_variable>"
exit 1
fi
fetch_whole_secret "\$2" "\$3"
else
if [ $# -ne 2 ]; then
echo "Usage: $0 <name_of_file> <name_of_key>"
exit 1
fi
fetch_specific_key "\$1" "\$2"
fi
fetch_whole_secret "$2" "$3"
else
if [ $# -ne 2 ]; then
echo "Usage: $0 <name_of_file> <name_of_key>"
exit 1
fi
fetch_specific_key "$1" "$2"
fi
}
EOF
28 changes: 18 additions & 10 deletions dataeng/resources/snowflake-refresh-snowpipe.sh
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
set -ex


# Creating Python virtual env
PYTHON_VENV="python_venv"
virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}"
Expand All @@ -10,16 +11,19 @@ source "${PYTHON_VENV}/bin/activate"
cd $WORKSPACE/analytics-tools/snowflake
make requirements

# Source the secrets-manager.sh script to make the function available
source $WORKSPACE/secrets-manager.sh
# Fetch the secrets from AWS
set +x
# Define the location of the script in the Jenkins workspace
SCRIPT_PATH="$WORKSPACE/secrets-manager.sh"


set +x

secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_snowpipe_user.p8
secrets-manager.sh -w analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS snowflake/rsa_key_passphrase_snowpipe_user
echo "Running secrets manager script"
python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowpipe_user.p8 -v rsa_key_snowpipe_user
python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user -v rsa_key_passphrase_snowpipe_user
#set -x

set -x
unset KEY_PATH
unset PASSPHRASE_PATH

python refresh_snowpipe.py \
--user 'SNOWPIPE' \
Expand All @@ -28,6 +32,10 @@ python refresh_snowpipe.py \
--pipe_name $PIPE_NAME \
--table_name $TABLE_NAME \
--delay $DELAY \
--limit $LIMIT
--key_file $KEY_PATH \
--passphrase_file $PASSPHRASE_PATH
--limit $LIMIT \
--key_file "$(cat "rsa_key_snowpipe_user")" \
--passphrase_file "$(cat "rsa_key_passphrase_snowpipe_user")"

rm rsa_key_snowpipe_user
rm rsa_key_passphrase_snowpipe_user

0 comments on commit 6f59269

Please sign in to comment.