Fixing Azone inupt file order dependence

See issue VisionEval/VisionEval-Dev#73

sources/framework/visioneval/DESCRIPTION

@@ -1,8 +1,8 @@
 Package: visioneval
 Type: Package
 Title: Software Framework for VisionEval Model System
-Version: 0.85
-Date: 2018-09-05
+Version: 0.86
+Date: 2019-09-18
 Author: Brian Gregor <gregorb@or-analytics.com>["aut", "cre"]
 Maintainer: Brian Gregor <gregorb@or-analytics.com>
 Copyright: AASHTO

sources/framework/visioneval/R/validation.R

@@ -1646,10 +1646,13 @@ parseInputFieldNames <-
 #' are no duplicates. Any file whose 'GROUP' specification is 'Global' or
 #' 'BaseYear' and whose 'TABLE' specification is a geographic specification
 #' other than 'Region' is checked to determine if it has a 'Geo' column and the
-#' entries are checked for completeness. The data in each column are checked
-#' against specifications to determine conformance. The function returns a list
-#' which contains a list of error messages and a list of the data inputs. The
-#' function also writes error messages and warnings to the log file.
+#' entries are checked for completeness. Entries for years other than the
+#' specified model run years are ignored. Data from input files is sorted by
+#' years and geography as needed to assure that it is consistent even when user
+#' input files are not consistent. The data in each column are checked against
+#' specifications to determine conformance. The function returns a list which
+#' contains a list of error messages and a list of the data inputs. The function
+#' also writes error messages and warnings to the log file.
 #'
 #' @param ModuleSpec_ls a list of module specifications that is consistent with
 #' the VisionEval requirements.
@@ -1705,7 +1708,6 @@ processModuleInputs <-
       }
       #Read in the data file and check that it is properly formatted
       Data_df <- try(read.csv(file.path(Dir, File), as.is = TRUE), silent = TRUE)
-
       if (class(Data_df) == "try-error") {
         Msg <-
           paste0(
@@ -1771,7 +1773,9 @@ processModuleInputs <-
       if (is.null(Data_ls[[Group]][[Table]])) {
         Data_ls[[Group]][[Table]] <- list()
       }
+
       #If Group is Year and Table is not Region, check Geo and Year fields
+      #-------------------------------------------------------------------
       if (Group  == "Year" & Table != "Region") {
         #Check that there are 'Year' and 'Geo' fields
         HasYearField <- "Year" %in% names(Data_df)
@@ -1789,13 +1793,32 @@ processModuleInputs <-
           FileErr_ls <- c(FileErr_ls, FileErr_)
           next()
         }
+        #Remove rows from Data_df for years other than model run years
+        Data_df <- Data_df[Data_df$Year %in% G$Years,]
         #Check that the file thas inputs for all years and geographic units
         #If so, save Year and Geo to table
         CorrectYearGeo <-
           checkInputYearGeo(Data_df$Year, Data_df$Geo, Group, Table)
         if (CorrectYearGeo$CompleteInput & !CorrectYearGeo$DupInput) {
-          Data_ls[[Group]][[Table]]$Year <- Data_df$Year
-          Data_ls[[Group]][[Table]]$Geo <- Data_df$Geo
+          #If Year and Geo data have not be added to the table then add them
+          if (is.null(Data_ls[[Group]][[Table]]$Year)) {
+            Data_ls[[Group]][[Table]]$Year <- Data_df$Year
+            Data_ls[[Group]][[Table]]$Geo <- Data_df$Geo
+            #Otherwise sort the input dataframe to be consistent
+          } else {
+            Data_df <- local({
+              #Year and Geo vector to match against
+              YearGeoToMatch_ <- paste(
+                Data_ls[[Group]][[Table]]$Year,
+                Data_ls[[Group]][[Table]]$Geo,
+                sep = "-"
+              )
+              #Year and Geo vector in Data_df
+              DataYearGeo_ <- paste(Data_df$Year, Data_df$Geo, sep = "-")
+              #Sort Data_df to match the Year and Geo order in the table
+              Data_df[match(YearGeoToMatch_, DataYearGeo_),]
+            })
+          }
         } else {
           if (!CorrectYearGeo$CompleteInput) {
             Msg <-
@@ -1821,8 +1844,10 @@ processModuleInputs <-
           next()
         }
       }
+
       #If Group is BaseYear or Global, and if Table is Azone, Bzone, Czone, or
       #Marea, check that geography is complete and correct
+      #-----------------------------------------------------------------------
       if (Group %in% c("BaseYear", "Global") &
           Table %in% c("Azone", "Bzone", "Czone", "Marea")) {
         #Check that there is a 'Geo' field
@@ -1842,6 +1867,7 @@ processModuleInputs <-
         #Check that the 'Geo' field is complete and not duplicated
         GeoDuplicated <- any(duplicated(Data_df$Geo))
         GeoIncomplete <- any(!(G$Geo_df[[Table]] %in% Data_df$Geo))
+        #If duplicated or incomplete print out errors
         if (GeoDuplicated | GeoIncomplete) {
           if (GeoDuplicated) {
             DupGeo_ <- unique(Data_df$Geo[GeoDuplicated])
@@ -1869,11 +1895,23 @@ processModuleInputs <-
           }
           FileErr_ls <- c(FileErr_ls, FileErr_)
           next()
+          #If no errors, process Geo
         } else {
-          Data_ls[[Group]][[Table]]$Geo <- Data_df$Geo
+          #If Geo not in table, add to table
+          if (is.null(Data_ls[[Group]][[Table]]$Geo)) {
+            Data_ls[[Group]][[Table]]$Geo <- Data_df$Geo
+            #Otherwise reorder Data_df to match Geo order in table
+          } else {
+            Data_df <- local({
+              GeoToMatch_ <- Data_ls[[Group]][[Table]]$Geo
+              Data_df[match(GeoToMatch_, Data_df$Geo),]
+            })
+          }
         }
       }
+
       #If Group is Year and Table is Region, check years are complete
+      #--------------------------------------------------------------
       if (Group  == "Year" & Table == "Region") {
         #Check that there is a 'Year' field
         HasYearField <- "Year" %in% names(Data_df)
@@ -1889,6 +1927,8 @@ processModuleInputs <-
           FileErr_ls <- c(FileErr_ls, FileErr_)
           next()
         }
+        #Remove rows from Data_df for years other than model run years
+        Data_df <- Data_df[Data_df$Year %in% G$Years,]
         #Check that the 'Year' field is complete and not duplicated
         YearDuplicated <- any(duplicated(Data_df$Year))
         YearIncomplete <- any(!(G$Years %in% Data_df$Year))
@@ -1919,12 +1959,21 @@ processModuleInputs <-
           }
           FileErr_ls <- c(FileErr_ls, FileErr_)
           next()
+          #Process Year input
         } else {
-          #Remove data for years that are not model run years
-          Data_ls[[Group]][[Table]]$Year <- Data_df$Year
+          #If table doesn't contain Year entry, add
+          if (is.null(Data_ls[[Group]][[Table]]$Year)) {
+            Data_ls[[Group]][[Table]]$Year <- Data_df$Year
+            #Otherwise sort Data_df to match Year order in table
+          } else {
+            YearToMatch_ <- Data_ls[[Group]][[Table]]$Year
+            Data_df[match(YearToMatch_, Data_df$Year),]
+          }
         }
       }
+
       #Check and load data into list
+      #-----------------------------
       DataErr_ls <- list(Errors = character(0), Warnings = character(0))
       for (Name in names(Spec_ls)) {
         ThisSpec_ls <- Spec_ls[[Name]]