test: test jenkins job changes

edx · Oct 12, 2023 · 7280679 · 7280679
1 parent b16ee36
commit 7280679
Show file tree

Hide file tree

Showing 3 changed files with 56 additions and 1 deletion.
diff --git a/dataeng/jobs/analytics/ModelTransfers.groovy b/dataeng/jobs/analytics/ModelTransfers.groovy
@@ -45,7 +45,6 @@ class ModelTransfers{
                 publishers common_publishers(allVars)
                 steps {
                     shell(dslFactory.readFileFromWorkspace('dataeng/resources/model-transfers.sh'))
-                    shell(dslFactory.readFileFromWorkspace('dataeng/resources/trigger-prefect-flow.sh'))
                 }
             }
         }

diff --git a/dataeng/jobs/analytics/ModelTransfersJenkinsTest.groovy b/dataeng/jobs/analytics/ModelTransfersJenkinsTest.groovy
@@ -0,0 +1,54 @@
+package analytics
+import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm
+import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator
+import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers
+import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers
+import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers
+import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters
+import static org.edx.jenkins.dsl.AnalyticsConstants.common_authorization
+
+class ModelTransfersJenkinsTest{
+    public static def job = { dslFactory, allVars ->
+        allVars.get('ENVIRONMENTS').each { environment, env_config ->
+            dslFactory.job("transfer-dbt-models-$environment"){
+                authorization common_authorization(env_config)
+                logRotator common_log_rotator(allVars)
+                parameters secure_scm_parameters(allVars)
+                parameters {
+                    stringParam('WAREHOUSE_TRANSFORMS_URL', allVars.get('WAREHOUSE_TRANSFORMS_URL'), 'URL for the Warehouse Transforms Repo.')
+                    stringParam('WAREHOUSE_TRANSFORMS_BRANCH', allVars.get('WAREHOUSE_TRANSFORMS_BRANCH'), 'Branch of Warehouse Transforms to use.')
+                    stringParam('DBT_PROJECT', env_config.get('DBT_PROJECT', allVars.get('DBT_PROJECT')), 'dbt project in warehouse-transforms to work on.')
+                    stringParam('DBT_PROFILE', env_config.get('DBT_PROFILE', allVars.get('DBT_PROFILE')), 'dbt profile from analytics-secure to work on.')
+                    stringParam('DBT_TARGET', env_config.get('DBT_TARGET', allVars.get('DBT_TARGET')), 'dbt target from analytics-secure to work on.')
+                    stringParam('MODELS_TO_TRANSFER', env_config.get('MODELS_TO_TRANSFER'), 'Name of DBT models which should be transferred to S3 via a Snowflake stage.')
+                    stringParam('NOTIFY', env_config.get('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY')), 'Space separated list of emails to send notifications to.')
+                }
+                multiscm secure_scm(allVars) << {
+                    git {
+                        remote {
+                            url('$WAREHOUSE_TRANSFORMS_URL')
+                            branch('$WAREHOUSE_TRANSFORMS_BRANCH')
+                            credentials('1')
+                        }
+                        extensions {
+                            relativeTargetDirectory('warehouse-transforms')
+                            pruneBranches()
+                            cleanAfterCheckout()
+                        }
+                    }
+                }
+                triggers common_triggers(allVars, env_config)
+                wrappers common_wrappers(allVars)
+                wrappers {
+                    colorizeOutput('xterm')
+                }
+                publishers common_publishers(allVars)
+                steps {
+                    shell(dslFactory.readFileFromWorkspace('dataeng/resources/model-transfers.sh'))
+                    shell(dslFactory.readFileFromWorkspace('dataeng/resources/trigger-prefect-flow.sh'))
+                }
+            }
+        }
+    }
+}
+
diff --git a/dataeng/jobs/createJobsNew.groovy b/dataeng/jobs/createJobsNew.groovy
@@ -4,6 +4,7 @@ import static analytics.DBTSourceFreshness.job as DBTSourceFreshnessJob
 import static analytics.DeployCluster.job as DeployClusterJob
 import static analytics.EmrCostReporter.job as EmrCostReporterJob
 import static analytics.ModelTransfers.job as ModelTransfersJob
+import static analytics.ModelTransfersJenkinsTest.job as ModelTransfersJenkinsTestJob
 import static analytics.RetirementJobEdxTriggers.job as RetirementJobEdxTriggersJob
 import static analytics.RetirementJobs.job as RetirementJobsJob
 import static analytics.SnowflakeCollectMetrics.job as SnowflakeCollectMetricsJob
@@ -46,6 +47,7 @@ def taskMap = [
     DEPLOY_CLUSTER_JOB: DeployClusterJob,
     EMR_COST_REPORTER_JOB: EmrCostReporterJob,
     MODEL_TRANSFERS_JOB: ModelTransfersJob,
+    MODEL_TRANSFERS_TEST_JOB: ModelTransfersJenkinsTestJob,
     RETIREMENT_JOB_EDX_TRIGGERS_JOB: RetirementJobEdxTriggersJob,
     RETIREMENT_JOBS_JOB: RetirementJobsJob,
     SNOWFLAKE_COLLECT_METRICS_JOB: SnowflakeCollectMetricsJob,