v0.2.54..v0.2.55 changeset ConflateCommand.java
Garret Voltz edited this page Aug 14, 2020
·
1 revision
diff --git a/hoot-services/src/main/java/hoot/services/controllers/conflation/ConflateCommand.java b/hoot-services/src/main/java/hoot/services/controllers/conflation/ConflateCommand.java
index e0e8812..07d7a0d 100644
--- a/hoot-services/src/main/java/hoot/services/controllers/conflation/ConflateCommand.java
+++ b/hoot-services/src/main/java/hoot/services/controllers/conflation/ConflateCommand.java
@@ -22,20 +22,23 @@
* This will properly maintain the copyright information. DigitalGlobe
* copyrights will be updated automatically.
*
- * @copyright Copyright (C) 2016, 2017, 2018, 2019 DigitalGlobe (http://www.digitalglobe.com/)
+ * @copyright Copyright (C) 2016, 2017, 2018, 2019, 2020 DigitalGlobe (http://www.digitalglobe.com/)
*/
package hoot.services.controllers.conflation;
import static hoot.services.HootProperties.CONFLATION_TYPES_PATH;
import static hoot.services.HootProperties.HOME_FOLDER;
import static hoot.services.HootProperties.HOOTAPI_DB_URL;
+import static hoot.services.HootProperties.NETWORK_CONFLATION_PATH;
import static hoot.services.HootProperties.RPT_STORE_PATH;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -46,6 +49,9 @@ import org.apache.commons.io.FileUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
import hoot.services.command.CommandResult;
import hoot.services.command.ExternalCommand;
@@ -80,14 +86,6 @@ class ConflateCommand extends ExternalCommand {
}
}
- public static Map<String, Map<String, String>> getConfigOptions() {
- return configOptions;
- }
-
- public static Map<String, Map<String, Object>> getConflationFeatures() {
- return conflationFeatures;
- }
-
public static String getConfigKey(String key) {
String configKey = null;
@@ -167,7 +165,7 @@ class ConflateCommand extends ExternalCommand {
Arrays.stream(toOptionsList(params.getAdvancedOptions())).forEach((option) -> {
if (!option.isEmpty()) {
options.add(option.trim());
- };
+ }
});
}
@@ -183,7 +181,6 @@ class ConflateCommand extends ExternalCommand {
if (disabledFeatures != null && !disabledFeatures.isEmpty()) {
List<String> matchers = new ArrayList<>();
- List<String> mergers = new ArrayList<>();
String roadType = "Network".equals(conflationAlgorithm) ? "Roads" : "RoadsNetwork";
for (String feature: conflationFeatures.keySet()) {
@@ -193,13 +190,54 @@ class ConflateCommand extends ExternalCommand {
Map<String, Object> conflationFeature = conflationFeatures.get(feature);
String conflationMatcher = (String) conflationFeature.get("matcher");
String conflationMerger = (String) conflationFeature.get("merger");
- if (conflationMatcher != null && conflationMerger != null && !disabledFeatures.contains(feature)) {
+ if (conflationMatcher != null && !conflationMatcher.isEmpty()
+ && conflationMerger != null && !conflationMerger.isEmpty()
+ && !disabledFeatures.contains(feature)) {
matchers.add(conflationMatcher);
- mergers.add(conflationMerger);
}
}
- options.add("match.creators=" + String.join(";", matchers));
- options.add("merger.creators=" + String.join(";", mergers));
+
+ // These 2 lists will be in the proper order so we will us this to sort our matchers and mergers list
+ List<String> matchCreators;
+ List<String> mergerCreators;
+
+ // SPECIAL CASE: if network order networkAlgorithm.conf
+ if ("Network".equals(conflationType)) {
+ JSONParser parser = new JSONParser();
+ try (FileReader fileReader = new FileReader(new File(HOME_FOLDER, NETWORK_CONFLATION_PATH))) {
+ JSONObject networkConfigJson = (JSONObject) parser.parse(fileReader);
+ matchCreators = getCreatorsFromJson("match.creators", "MatchCreators", networkConfigJson);
+ mergerCreators = getCreatorsFromJson("merger.creators", "MergerCreators", networkConfigJson);
+ }
+ catch (IOException | ParseException ioe) {
+ throw new RuntimeException("Error reading NetworkAlgorithm.conf file", ioe);
+ }
+ } else {
+ // These 2 lists are in the proper order so we will us this to sort our matchers and mergers list
+ matchCreators = new ArrayList<>(Arrays.asList(configOptions.get("MatchCreators").get("default").split(";")));
+ mergerCreators = new ArrayList<>(Arrays.asList(configOptions.get("MergerCreators").get("default").split(";")));
+ }
+
+ List<String> sortedMatchers = new ArrayList<>();
+ List<String> sortedMergers = new ArrayList<>();
+ // since we are looping through the configOptions matchers list, we can check if our 'matchers' list we added items to contains the same strings.
+ // if it does, we just add those items from the configOptions matchers, along with the respective index item from configOptions mergers, to maintain proper order.
+ for (int i = 0; i < matchCreators.size(); i++) {
+ String current = matchCreators.get(i);
+
+ // Special case for network
+ if (current.equals("hoot::NetworkMatchCreator") && matchers.contains("hoot::HighwayMatchCreator")) {
+ sortedMatchers.add("hoot::NetworkMatchCreator");
+ sortedMergers.add("hoot::NetworkMergerCreator");
+ }
+ else if (matchers.contains(current)) {
+ sortedMatchers.add(current);
+ sortedMergers.add(mergerCreators.get(i));
+ }
+ }
+
+ options.add("match.creators=" + String.join(";", sortedMatchers));
+ options.add("merger.creators=" + String.join(";", sortedMergers));
}
Map<String, String> hoot2AdvOptions = params.getHoot2AdvOptions();
@@ -241,9 +279,9 @@ class ConflateCommand extends ExternalCommand {
substitutionMap.put("DIFF_TAGS", diffTags);
substitutionMap.put("STATS", stats);
- String command = null;
+ String command;
if (params.getHoot2() == null) { // hoot1
- command = "hoot ${CONFLATION_COMMAND} --${DEBUG_LEVEL} ${HOOT_OPTIONS} ${INPUT1} ${INPUT2} ${OUTPUT} ${DIFFERENTIAL} ${DIFF_TAGS} ${STATS}";
+ command = "hoot.bin ${CONFLATION_COMMAND} --${DEBUG_LEVEL} ${HOOT_OPTIONS} ${INPUT1} ${INPUT2} ${OUTPUT} ${DIFFERENTIAL} ${DIFF_TAGS} ${STATS}";
} else {
if (conflationType != null) {
@@ -273,7 +311,7 @@ class ConflateCommand extends ExternalCommand {
substitutionMap.put("CONFLATION_ALGORITHM", conflationAlgorithm + "Algorithm.conf");
}
- command = "hoot ${CONFLATION_COMMAND} --${DEBUG_LEVEL}"
+ command = "hoot.bin ${CONFLATION_COMMAND} --${DEBUG_LEVEL}"
+ (conflationType != null ? " -C ${CONFLATION_TYPE}" : "")
+ (conflationAlgorithm != null ? " -C ${CONFLATION_ALGORITHM}" : "")
+ " ${HOOT_OPTIONS} ${INPUT1} ${INPUT2} ${OUTPUT} ${DIFFERENTIAL} ${DIFF_TAGS} ${STATS}";
@@ -282,6 +320,58 @@ class ConflateCommand extends ExternalCommand {
super.configureCommand(command, substitutionMap, caller);
}
+ /*
+ * Returns a list of conflate match or merger creators given a configuration
+ *
+ * @param optionName core configuration option name; e.g. "match.creators"
+ * @param defaultOptionId configuration option name as referenced in the services default config; e.g. "MatchCreators"
+ * @param config the JSON configuration to parse
+ * @note Option replacement syntax is valid for any hoot config option, but in practice its only currently being used from the
+ * match/merger creator options, so the concept or parsing it may have to be abstracted here at a later time.
+ * @return a list of match/merger creator strings
+ */
+ private List<String> getCreatorsFromJson(String optionName, String defaultOptionId, JSONObject config) {
+ // hoot json config files support an option list value entry replacement syntax. For instance, to replace an instance of
+ // "hoot::HighwayMatchCreator", in the "match.creators" config option the json snippet would look like:
+ //
+ // "match.creators": "hoot::HighwayMatchCreator->hoot::NetworkMatchCreator",
+ //
+ // where '->' means replace the first item with the second item.
+
+ String creatorsStr = config.get(optionName).toString();
+
+ if (creatorsStr.contains("->")) {
+ // If our string contains the replacement operator, parse through each entry and perform the option value entry replacement
+ // on the default configuration. We're assuming that if one option val entry has '->', then they all do which is also what
+ // the core does when parsing CLI args.
+
+ List <String> modifiedCreators;
+ List <String> defaultCreators =
+ new ArrayList<>(Arrays.asList(configOptions.get(defaultOptionId).get("default").split(";")));
+ modifiedCreators = defaultCreators;
+ List <String> creatorReplacements = new ArrayList <>(Arrays.asList(creatorsStr.split(";")));
+ for (int i = 0; i < creatorReplacements.size(); i++) {
+ String creatorReplacement = creatorReplacements.get(i);
+ List <String> creatorReplacementParts = new ArrayList <>(Arrays.asList(creatorReplacement.split("->")));
+ if (creatorReplacementParts.size() != 2) {
+ throw new IllegalArgumentException(
+ String.format("Invalid replacement option value entry: \"%s\".", creatorReplacement));
+ }
+ String creatorToReplace = creatorReplacementParts.get(0);
+ String replacementCreator = creatorReplacementParts.get(1);
+ if (!modifiedCreators.contains(creatorToReplace)) {
+ throw new IllegalArgumentException(
+ String.format("Option value entry to replace does not exist: \"%s\".", creatorToReplace));
+ }
+ Collections.replaceAll(modifiedCreators, creatorToReplace, replacementCreator);
+ }
+ return modifiedCreators;
+ } else {
+ // Otherwise, just parse the creators as a literal option list value.
+ return new ArrayList<>(Arrays.asList(creatorsStr.split(";")));
+ }
+ }
+
private String[] toOptionsList(String optionsString) {
return optionsString.trim().split("-D ");
}