edx · mzakaria1 · Jan 24, 2023
diff --git a/dataeng/jobs/analytics/AggregateDailyTrackingLogs.groovy b/dataeng/jobs/analytics/AggregateDailyTrackingLogs.groovy
@@ -12,6 +12,7 @@ class AggregateDailyTrackingLogs {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("aggregate-daily-tracking-logs-$environment") {
+                description("This job aggregates tracking log files by grouping them by date, producing the aggregated output in the destination bucket.")
                 disabled(env_config.get('DISABLED', false))
                 logRotator common_log_rotator(allVars, env_config)
                 parameters to_date_interval_parameter(env_config)

diff --git a/dataeng/jobs/analytics/DBTDocs.groovy b/dataeng/jobs/analytics/DBTDocs.groovy
@@ -9,6 +9,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters
 class DBTDocs{
     public static def job = { dslFactory, allVars ->
         dslFactory.job("dbt-docs"){
+            description('Job that generates dbt documentation and uploads to S3.')
             logRotator common_log_rotator(allVars)
             parameters secure_scm_parameters(allVars)
             parameters {

diff --git a/dataeng/jobs/analytics/DBTSourceFreshness.groovy b/dataeng/jobs/analytics/DBTSourceFreshness.groovy
@@ -9,6 +9,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters
 class DBTSourceFreshness{
     public static def job = { dslFactory, allVars ->
         dslFactory.job("dbt-source-freshness"){
+            description('It checks DBT source freshness for a specified target.')
             logRotator common_log_rotator(allVars)
             parameters secure_scm_parameters(allVars)
             parameters {

diff --git a/dataeng/jobs/analytics/DatabaseExportCoursewareStudentmodule.groovy b/dataeng/jobs/analytics/DatabaseExportCoursewareStudentmodule.groovy
@@ -11,6 +11,7 @@ class DatabaseExportCoursewareStudentmodule {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("database-export-courseware-studentmodule-$environment") {
+                description('Job which generates a raw SQL dump of courseware_studentmodule table and separates it into a different tsv file for each course')
                 disabled(env_config.get('DISABLED', false))
                 logRotator common_log_rotator(allVars, env_config)
                 multiscm common_multiscm(allVars)

diff --git a/dataeng/jobs/analytics/DeployCluster.groovy b/dataeng/jobs/analytics/DeployCluster.groovy
@@ -7,6 +7,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers
 class DeployCluster {
     public static def job = { dslFactory, allVars ->
         dslFactory.job("deploy-cluster") {
+            description('Job to provision an EMR cluster, typically for adhoc usage.')
             logRotator common_log_rotator(allVars)
             parameters {
                 stringParam('CONFIG_REPO', 'git@github.com:edx/edx-analytics-configuration.git', '')

diff --git a/dataeng/jobs/analytics/EmrCostReporter.groovy b/dataeng/jobs/analytics/EmrCostReporter.groovy
@@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers
 class EmrCostReporter {
     public static def job = { dslFactory, allVars ->
         dslFactory.job('emr-cost-reporter') {
+            description('It analyzies EMR Cluster and generates its cost report weekly.')
             parameters {
                 stringParam('TOOLS_REPO', allVars.get('ANALYTICS_TOOLS_URL'), '')
                 stringParam('TOOLS_BRANCH', 'origin/master', '')

diff --git a/dataeng/jobs/analytics/Enrollment.groovy b/dataeng/jobs/analytics/Enrollment.groovy
@@ -14,6 +14,7 @@ class Enrollment {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("enrollment-$environment") {
+                description('Job which computes metrics related to user enrollments in courses')
                 // desupport-1271: temporarily disabling now the the dbt version is live.
                 // leave this for 2 weeks and delete the job when we are confident that
                 // this is no longer needed.

diff --git a/dataeng/jobs/analytics/EnrollmentValidationEvents.groovy b/dataeng/jobs/analytics/EnrollmentValidationEvents.groovy
@@ -12,6 +12,7 @@ class EnrollmentValidationEvents {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("enrollment-validation-events-$environment") {
+                description('It runs enrollment validation workflow and validates S3 file sizes with a threshold.')
                 disabled(env_config.get('DISABLED', false))
                 logRotator common_log_rotator(allVars)
                 parameters common_parameters(allVars, env_config)

diff --git a/dataeng/jobs/analytics/EventExportIncremental.groovy b/dataeng/jobs/analytics/EventExportIncremental.groovy
@@ -14,6 +14,7 @@ class EventExportIncremental {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("event-export-incremental-$environment") {
+                description('Job which groups tracking events by org and exports them to S3 for research purposes.')
                 disabled(env_config.get('DISABLED', false))
                 logRotator common_log_rotator(allVars, env_config)
                 parameters common_parameters(allVars, env_config)

diff --git a/dataeng/jobs/analytics/EventExportIncrementalLarge.groovy b/dataeng/jobs/analytics/EventExportIncrementalLarge.groovy
@@ -15,6 +15,7 @@ class EventExportIncrementalLarge {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("event-export-incremental-large-$environment") {
+                description('Job which groups tracking events by org and exports them to S3 for research purposes.')
                 disabled(env_config.get('DISABLED', false))
                 authorization common_authorization(allVars)
                 logRotator common_log_rotator(allVars, env_config)

diff --git a/dataeng/jobs/analytics/ExpireVerticaPassword.groovy b/dataeng/jobs/analytics/ExpireVerticaPassword.groovy
@@ -10,6 +10,7 @@ class ExpireVerticaPassword {
     public static def job = { dslFactory, allVars ->
 
         dslFactory.job('expire-vertica-password') {
+            description('It expires user passwords (excluds provided users) and notifies via space separated emails.')
             logRotator common_log_rotator(allVars)
             parameters {
                 stringParam('TOOLS_REPO', allVars.get('ANALYTICS_TOOLS_URL'), '')

diff --git a/dataeng/jobs/analytics/JenkinsBackup.groovy b/dataeng/jobs/analytics/JenkinsBackup.groovy
@@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers
 class JenkinsBackup {
     public static def job = { dslFactory, allVars ->
         dslFactory.job('jenkins-backup') {
+            description('It takes backup of jenkins jobs and config.yml (from jenkins_home) and pushes to S3 bucket')
             parameters {
                 stringParam('S3_BACKUP_BUCKET', allVars.get('S3_BACKUP_BUCKET'))
                 stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.')

diff --git a/dataeng/jobs/analytics/ModelTransfers.groovy b/dataeng/jobs/analytics/ModelTransfers.groovy
@@ -11,6 +11,7 @@ class ModelTransfers{
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("transfer-dbt-models-$environment"){
+                description('It transfers DBT models to S3 via a Snowflake stage.')
                 authorization common_authorization(env_config)
                 logRotator common_log_rotator(allVars)
                 parameters secure_scm_parameters(allVars)

diff --git a/dataeng/jobs/analytics/SnowflakeCollectMetrics.groovy b/dataeng/jobs/analytics/SnowflakeCollectMetrics.groovy
@@ -21,7 +21,7 @@ class SnowflakeCollectMetrics {
         jobConfigs.each { jobConfig ->
 
             dslFactory.job(jobConfig['NAME']){
-
+                description('This job collects Snowflake Metrics and notifies users via space separated emails.')
                 logRotator common_log_rotator(allVars)
                 parameters secure_scm_parameters(allVars)
                 parameters {

diff --git a/dataeng/jobs/analytics/SnowflakeDemographicsCleanup.groovy b/dataeng/jobs/analytics/SnowflakeDemographicsCleanup.groovy
@@ -10,6 +10,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm
 class SnowflakeDemographicsCleanup {
     public static def job = { dslFactory, allVars -> 
         dslFactory.job("snowflake-demographics-cleanup") {
+            description('This job cleans up Snowflake Demographics and notifies users via provided space separated emails.')
             logRotator common_log_rotator(allVars)
             authorization common_authorization(allVars)
             parameters secure_scm_parameters(allVars)

diff --git a/dataeng/jobs/analytics/SnowflakeExpirePasswords.groovy b/dataeng/jobs/analytics/SnowflakeExpirePasswords.groovy
@@ -29,7 +29,7 @@ class SnowflakeExpirePasswords {
         jobConfigs.each { jobConfig ->
 
             dslFactory.job(jobConfig['NAME']) {
-
+                description("Job to expire snowflake users' passwords, runs every 3 months.")
                 logRotator common_log_rotator(allVars)
                 parameters secure_scm_parameters(allVars)
                 parameters {

diff --git a/dataeng/jobs/analytics/SnowflakePublicGrantsCleaner.groovy b/dataeng/jobs/analytics/SnowflakePublicGrantsCleaner.groovy
@@ -14,6 +14,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm
 class SnowflakePublicGrantsCleaner {
     public static def job = { dslFactory, allVars ->
         dslFactory.job("snowflake-public-grants-cleaner") {
+            description('This job cleans up Snowflake public grants and notifies users via provided space separated emails.')
             logRotator common_log_rotator(allVars)
             parameters secure_scm_parameters(allVars)
             parameters {

diff --git a/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy b/dataeng/jobs/analytics/SnowflakeRefreshSnowpipe.groovy
@@ -41,7 +41,7 @@ class SnowflakeRefreshSnowpipe {
         jobConfigs.each { jobConfig ->
 
             dslFactory.job(jobConfig['NAME']) {
-
+                description('This job refreshes Snowflake snowpipe and notifies users via space separated emails.')
                 logRotator common_log_rotator(allVars)
                 parameters secure_scm_parameters(allVars)
                 parameters {

diff --git a/dataeng/jobs/analytics/SnowflakeReplicaImportFromS3.groovy b/dataeng/jobs/analytics/SnowflakeReplicaImportFromS3.groovy
@@ -45,6 +45,7 @@ class SnowflakeReplicaImportFromS3 {
 
         jobConfigs.each { appName, jobConfig ->
             dslFactory.job("snowflake-${appName.toLowerCase()}-read-replica-import-from-s3") {
+                description('This imports Mysql Database from S3 to Snowflake Schema.')
                 disabled(jobConfig.get('DISABLED', false))
                 logRotator common_log_rotator(allVars)
                 parameters common_parameters(allVars, jobConfig)

diff --git a/dataeng/jobs/analytics/SnowflakeSchemaBuilder.groovy b/dataeng/jobs/analytics/SnowflakeSchemaBuilder.groovy
@@ -8,6 +8,9 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters
 class SnowflakeSchemaBuilder {
     public static def job = { dslFactory, allVars ->
         dslFactory.job('snowflake-schema-builder') {
+            description('This job runs the dbt script to update schemas and sql, from the source project directory. ' +
+                'If any files are added, deleted, or changed it will make commit and create a PR.'
+            )
             logRotator common_log_rotator(allVars)
             parameters secure_scm_parameters(allVars)
             parameters {

diff --git a/dataeng/jobs/analytics/StitchSnowflakeLagMonitor.groovy b/dataeng/jobs/analytics/StitchSnowflakeLagMonitor.groovy
@@ -14,6 +14,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm
 class StitchSnowflakeLagMonitor {
     public static def job = { dslFactory, allVars ->
         dslFactory.job("stitch-snowflake-lag-monitor") {
+            description('This job checks for lag between Stitch and Snowflake.')
             logRotator common_log_rotator(allVars)
             parameters secure_scm_parameters(allVars)
             parameters {

diff --git a/dataeng/jobs/analytics/TerminateCluster.groovy b/dataeng/jobs/analytics/TerminateCluster.groovy
@@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers
 class TerminateCluster {
     public static def job = { dslFactory, allVars ->
         dslFactory.job("terminate-cluster") {
+            description('This job terminates provided Analytics EMR cluster.')
             logRotator common_log_rotator(allVars)
             parameters {
                 stringParam('CLUSTER_NAME', '', 'Name of the EMR cluster to terminate.')

diff --git a/dataeng/jobs/analytics/UpdateUsers.groovy b/dataeng/jobs/analytics/UpdateUsers.groovy
@@ -6,6 +6,7 @@ import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers
 class UpdateUsers {
     public static def job = { dslFactory, allVars ->
         dslFactory.job("update-users") {
+            description("This job updates the list of ssh users on a specified instance.")
             logRotator common_log_rotator(allVars)
             parameters {
                 textParam('EXTRA_VARS', allVars.get('EXTRA_VARS'), '')

diff --git a/dataeng/jobs/analytics/UserLocationByCourse.groovy b/dataeng/jobs/analytics/UserLocationByCourse.groovy
@@ -13,6 +13,7 @@ class UserLocationByCourse {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("user-location-by-course-$environment") {
+                description('Job which computes the number of users in each country that are enrolled in each course.')
                 disabled(env_config.get('DISABLED', false))
                 authorization common_authorization(env_config)
                 logRotator common_log_rotator(allVars, env_config)

diff --git a/dataeng/jobs/analytics/VideoTimeline.groovy b/dataeng/jobs/analytics/VideoTimeline.groovy
@@ -14,6 +14,7 @@ class VideoTimeline {
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("video-timeline-$environment") {
+                description('Job which aggregates statistics about video viewing.')
                 disabled(env_config.get('DISABLED', false))
                 authorization common_authorization(env_config)
                 logRotator common_log_rotator(allVars, env_config)

diff --git a/dataeng/jobs/analytics/WarehouseTransforms.groovy b/dataeng/jobs/analytics/WarehouseTransforms.groovy
@@ -12,6 +12,7 @@ class WarehouseTransforms{
     public static def job = { dslFactory, allVars ->
         allVars.get('ENVIRONMENTS').each { environment, env_config ->
             dslFactory.job("warehouse-transforms-$environment"){
+                description('Job to run dbt transformations.')
                 description(env_config.get('DESCRIPTION', ''))
                 disabled(env_config.get('DISABLED', false))
                 authorization common_authorization(env_config)