Merge branch 'master' into master

.github/requirements.txt

@@ -1,5 +1,5 @@
-# Packages required for the validation (update cache on the 2022-07-29)
-https://github.com/midas-network/covid19SMHvalidation/archive/refs/heads/main.zip
+# Packages required for the validation (update requirements (cache) on the 2023-05-26)
+https://github.com/midas-network/SMHvalidation/archive/refs/heads/main.zip # v0.0.1
 arrow
 covidcast
 cowplot

.github/workflows/install_cache_dependencies.yml

@@ -15,7 +15,7 @@ jobs:
           uses: actions/checkout@v2
 
         - name: Setup R
-          uses: r-lib/actions/setup-r@v1
+          uses: r-lib/actions/setup-r@v2
 
         - name: Install system dependencies
           run: sudo apt-get install libcurl4-openssl-dev libudunits2-dev libgdal-dev
@@ -32,4 +32,4 @@ jobs:
             R -e 'Sys.setenv("NOT_CRAN" = TRUE)
                   install.packages(c("arrow", "gh", "remotes"))
                   Sys.unsetenv("NOT_CRAN")
-                  remotes::install_url("https://github.com/midas-network/covid19SMHvalidation/archive/refs/heads/main.zip")'
+                  remotes::install_url("https://github.com/midas-network/SMHvalidation/archive/refs/heads/main.zip")'

.github/workflows/validation.yml

@@ -13,7 +13,7 @@ jobs:
           uses: actions/checkout@v2
 
         - name: Setup R
-          uses: r-lib/actions/setup-r@v1
+          uses: r-lib/actions/setup-r@v2
 
         - name: Install system dependencies
           run: sudo apt-get install libcurl4-openssl-dev libudunits2-dev libgdal-dev
@@ -32,7 +32,7 @@ jobs:
           run: |
             R -e 'Sys.setenv("NOT_CRAN" = TRUE)
                   install.packages(c("arrow", "remotes"))
-                  remotes::install_url("https://github.com/midas-network/covid19SMHvalidation/archive/refs/heads/main.zip")'
+                  remotes::install_url("https://github.com/midas-network/SMHvalidation/archive/refs/heads/main.zip")'
 
         - name: Run Validation
           run: |

code/validation/round_metadata/2022-10-30_metadata-sample.json

@@ -0,0 +1 @@
+{"column_names":["model_projection_date","scenario_name","scenario_id","target","target_end_date","location","sample","value"],"quantiles":{"required":[null],"optional":{}},"ages":{"age_min":{},"age_max":{}},"locations":["US","01","02","04","05","06","08","09","10","11","12","13","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","44","45","46","47","48","49","50","51","53","54","55","56","60","66","69","72","74","78"],"horizons":{"required":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26],"optional":{}},"scenarios":{"1":{"name":["lowBoo_highVar"],"id":["D-2022-10-29"]},"2":{"name":["lowBoo_modVar"],"id":["C-2022-10-29"]},"3":{"name":["highBoo_highVar"],"id":["B-2022-10-29"]},"4":{"name":["highBoo_modVar"],"id":["A-2022-10-29"]}},"first_week_ahead":["2022-11-05"],"targets":{"required":{"inc death":{"quantiles":{"required":[null],"optional":[null]},"type":{"required":["sample"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}},"inc hosp":{"quantiles":{"required":[null],"optional":[null]},"type":{"required":["sample"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}},"inc case":{"quantiles":{"required":[null],"optional":[null]},"type":{"required":["sample"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}}},"optional":{"inc inf":{"quantiles":{"required":[null],"optional":[null]},"type":{"required":["sample"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}}}}}

code/validation/round_metadata/2022-10-30_metadata.json

@@ -0,0 +1 @@
+{"column_names":["model_projection_date","scenario_name","scenario_id","target","target_end_date","location","type","quantile","value"],"quantiles":{"required":[0.01,0.025,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,0.7,0.75,0.8,0.85,0.9,0.95,0.975,0.99],"optional":[0,1]},"ages":{"age_min":{},"age_max":{}},"locations":["US","01","02","04","05","06","08","09","10","11","12","13","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","44","45","46","47","48","49","50","51","53","54","55","56","60","66","69","72","74","78"],"horizons":{"required":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26],"optional":{}},"scenarios":{"1":{"name":["lowBoo_highVar"],"id":["D-2022-10-29"]},"2":{"name":["lowBoo_modVar"],"id":["C-2022-10-29"]},"3":{"name":["highBoo_highVar"],"id":["B-2022-10-29"]},"4":{"name":["highBoo_modVar"],"id":["A-2022-10-29"]}},"first_week_ahead":["2022-11-05"],"targets":{"required":{"inc death":{"quantiles":{"required":["all"],"optional":["all"]},"type":{"required":["quantile","point"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}},"cum death":{"quantiles":{"required":["all"],"optional":["all"]},"type":{"required":["quantile","point"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}},"inc hosp":{"quantiles":{"required":["all"],"optional":["all"]},"type":{"required":["quantile","point"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}},"cum hosp":{"quantiles":{"required":["all"],"optional":["all"]},"type":{"required":["quantile","point"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}},"inc case":{"quantiles":{"required":["all"],"optional":["all"]},"type":{"required":["quantile","point"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}},"cum case":{"quantiles":{"required":["all"],"optional":["all"]},"type":{"required":["quantile","point"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}}},"optional":{"inc inf":{"quantiles":{"required":["all"],"optional":["all"]},"type":{"required":["quantile","point"],"optional":[null]},"location":{"required":["all"],"optional":[null]},"horizons":{"required":["all"],"optional":[null]},"scenarios":{"required":["all"],"optional":[null]},"value":[">= 0"],"agegroup":{"required":[null],"optional":[null]}}}}}

code/validation/validation.R

@@ -1,96 +1,72 @@
-library(covid19SMHvalidation)
+library(SMHvalidation)
 library(gh)
 
 # check if submissions file
-pr_files <- gh::gh(paste0("GET /repos/", 
+pr_files <- gh::gh(paste0("GET /repos/",
                           "midas-network/covid19-scenario-modeling-hub/", "pulls/",
                           Sys.getenv("GH_PR_NUMBER"),"/files"))
 
 pr_files_name <- purrr::map(pr_files, "filename")
 pr_sub_files <- grep(
-  "data-processed/.*/\\d{4}-\\d{2}-\\d{2}.+-.+(.csv|.zip|.gz)", pr_files_name,
-  value = TRUE)
+  "data-processed/.*/\\d{4}-\\d{2}-\\d{2}-.+(.pqt|.parquet)",
+  pr_files_name, value = TRUE)
+pop_path <- "data-locations/locations.csv"
+js_def_file <- "hub-config/tasks.json"
+lst_gs <- NULL
 
 # Run validation on file corresponding to the submission file format
-if (length(pr_sub_files) > 0) {  
-  # prepare observe data
-  lst_gs <- suppressWarnings(pull_gs_data())
-  # select submission files
-  pr_sub_files_lst <- pr_files[purrr::map(pr_files, "filename") %in% pr_sub_files]
-  # Prepare viz path if necessary
-  if (!(dir.exists(paste0(getwd(), "/proj_plot"))))
-    dir.create(paste0(getwd(), "/proj_plot"))
-  # run validation and visualization
-  test_tot <- lapply(seq_len(length(pr_sub_files_lst)), function(x) {
-    url_link <- URLdecode(pr_sub_files_lst[[x]]$raw_url)
-    if (grepl(".zip$|.gz$", url_link)) {
-      # download file
-      download.file(url_link, basename(url_link))
-      # generate visualization pdf
-      test_viz <- try(generate_validation_plots(path_proj = basename(url_link), lst_gs = lst_gs, 
-        save_path = paste0(getwd(), "/proj_plot"), y_sqrt = FALSE, plot_quantiles = c(0.025, 0.975)))
+if (length(pr_sub_files) > 0) {
+  team_name <- unique(basename(dirname(pr_sub_files)))
+  sub_file_date <- unique(stringr::str_extract(basename(pr_sub_files),
+                                               "\\d{4}-\\d{2}-\\d{2}"))
+  group_files <- paste0(sub_file_date, "-", team_name)
+  test_tot <- lapply(group_files, function(y) {
+    # select submission files
+    pr_sub_files_group <- grep(y, pr_sub_files, value = TRUE)
+    pr_sub_files_lst <- pr_files[purrr::map(pr_files, "filename") %in%
+                                   pr_sub_files_group]
+    if (length(pr_sub_files_lst) > 0) {
+      # run validation on all files
+      test_tot <- lapply(seq_len(length(pr_sub_files_lst)), function(x) {
+        # submission file
+        url_link <- URLdecode(pr_sub_files_lst[[x]]$raw_url)
+        # Run validation for Parquet and compressed file format
+        download.file(url_link, basename(url_link))
+      })
       # run validation
-      test <- capture.output(try(validate_submission(basename(url_link), lst_gs = lst_gs)))
+      test <- capture.output(try(
+        validate_submission(basename(pr_sub_files_group), js_def = js_def_file,
+                            lst_gs = lst_gs, pop_path = pop_path)))
+      # list of the viz and validation results
+      test_tot <- list(valid = test)
+    } else {
+      test_tot <-  list(
+        valid = paste0(
+          "No projection submission file in the standard SMH file ",
+          "format found in the Pull-Request for: ", y))
     }
-    if (grepl(".csv$", url_link)) {
-      # generate visualization pdf
-      test_viz <- try(generate_validation_plots(path_proj = url_link, lst_gs = lst_gs, 
-        save_path = paste0(getwd(), "/proj_plot"), y_sqrt = FALSE, plot_quantiles = c(0.025, 0.975)))
-      # run validation
-      test <- capture.output(try(validate_submission(url_link, lst_gs = lst_gs)))
-    }
-    # Remove visualization pdf if viz has an error
-    if (class(test_viz) == "try-error") 
-      file.remove(dir(paste0(getwd(), "/proj_plot"), full.names = TRUE))
-    # list of the viz and validation results
-    test_tot <- list(valid = test, viz = test_viz)
-    # returns all output
+      # returns all output
     return(test_tot)
-   })
+  })
 }  else {
-  test_tot <-  list(list(
-    valid = paste0(
-      "No projection submission file in the standard SMH file ",
-      "format found in the Pull-Request. No validation was run."),
-    viz = NA
-  ))
+    test_tot <-  list(list(
+      valid = paste0(
+        "No projection submission file in the standard SMH file ",
+        "format found in the Pull-Request. No validation was run.")
+    ))
 }
 
 # Post validation results as comment on the open PR
 test_valid <- purrr::map(test_tot, "valid")
 message <- purrr::map(test_valid, paste, collapse = "\n")
 
 lapply(seq_len(length(message)), function(x) {
-  gh::gh(paste0("POST /repos/", "midas-network/covid19-scenario-modeling-hub/", 
+  gh::gh(paste0("POST /repos/", "midas-network/covid19-scenario-modeling-hub/",
                 "issues/", Sys.getenv("GH_PR_NUMBER"),"/comments"),
          body = message[[x]],
          .token = Sys.getenv("GH_TOKEN"))
 })
 
-# Post visualization results as comment on the open PR
-test_viz <- purrr::map(test_tot, "viz")
-if (any(!is.na(test_viz))) {
-  message_plot <- paste0(
-    "If the submission contains a projection file, a pdf containing the ",
-    "visualization plots of the submission is available and downloadable ",
-    "in the GitHub actions. Please click on 'details' on the right of the ",
-    "'Validate submission' checks. The pdf is available in a ZIP file as ",
-    "an artifact of the GH Actions. For more information, please see ",
-    "[here](https://docs.github.com/en/actions/managing-workflow-runs/downloading-workflow-artifacts)")
-
-  if (any(unlist(purrr::map(test_viz, class)) == "try-error")) {
-    message_plot <- paste0(message_plot, "\n\n",
-      "The visualization encounters an issue and might not be available,",
-      " if the validation does not return any error, please feel free to ",
-      "tag `@LucieContamin` for any question.")
-  }
-
-  gh::gh(paste0("POST /repos/", "midas-network/covid19-scenario-modeling-hub/", 
-                "issues/", Sys.getenv("GH_PR_NUMBER"),"/comments"),
-         body = message_plot,
-         .token = Sys.getenv("GH_TOKEN"))
-}
-
 # Validate or stop the github actions
 if (any(grepl("\U000274c Error", test_valid))) {
   stop("The submission contains one or multiple issues")